Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

More CPU kernels (#287)

Aleksandr Golokoze818f07b4fff
diff --git a/crates/uzu/src/backends/common/gpu_types/quantization.rs b/crates/uzu/src/backends/common/gpu_types/quantization.rs
index 416e9370f..3f29b4726 100644
--- a/crates/uzu/src/backends/common/gpu_types/quantization.rs
+++ b/crates/uzu/src/backends/common/gpu_types/quantization.rs
@@ -27,6 +27,10 @@ impl QuantizationMode {
QuantizationMode::UINT8 => DataType::U8,
}
}
+
+ pub fn to_u32(&self) -> u32 {
+ (*self).into()
+ }
}
impl From<QuantizationMode> for DataType {
@@ -39,6 +43,17 @@ impl From<QuantizationMode> for DataType {
}
}
+impl From<u32> for QuantizationMode {
+ fn from(val: u32) -> Self {
+ match val {
+ 0 => QuantizationMode::UINT4,
+ 1 => QuantizationMode::INT8,
+ 2 => QuantizationMode::UINT8,
+ _ => panic!("Invalid QuantizationMode value: {val}"),
+ }
+ }
+}
+
impl Into<u32> for QuantizationMode {
fn into(self) -> u32 {
match self {
diff --git a/crates/uzu/src/backends/cpu/kernel/activation/activation.rs b/crates/uzu/src/backends/cpu/kernel/activation/activation.rs
index 2b967ff48..58faee590 100644
--- a/crates/uzu/src/backends/cpu/kernel/activation/activation.rs
+++ b/crates/uzu/src/backends/cpu/kernel/activation/activation.rs
@@ -7,15 +7,19 @@ use crate::ArrayElement;
#[kernel(Activation)]
#[variants(T, f16, f32, bf16)]
pub fn activation<T: ArrayElement + Float>(
- #[allow(unused)]
- #[optional(!in_place)]
- input: Option<*const T>,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] n: u32,
- #[allow(unused)] act_type: crate::backends::common::gpu_types::activation_type::ActivationType,
- #[allow(unused)]
- #[specialize]
- in_place: bool,
+ #[optional(!in_place)] input: Option<*const T>,
+ output: *mut T,
+ n: u32,
+ act_type: crate::backends::common::gpu_types::activation_type::ActivationType,
+ #[specialize] in_place: bool,
) {
- todo!()
+ let input = match in_place {
+ true => output,
+ false => input.unwrap(),
+ };
+ unsafe {
+ for i in 0..n as usize {
+ *output.add(i) = act_type.activate(*input.add(i));
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
index 7afa88cad..3be696c10 100644
--- a/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
@@ -4,66 +4,179 @@ use num_traits::Float;
use crate::ArrayElement;
+const TOTAL_BLOCKS_COUNT: u32 = 32;
+const SEQUENCE_BLOCK_SIZE_1: u32 = 8;
+
#[kernel(AttentionTwoPass1)]
#[variants(T, f32, f16, bf16)]
#[variants(HEAD_DIM, 64, 128, 256)]
pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(
- #[allow(unused)] queries: *const T,
- #[allow(unused)] keys: *const T,
- #[allow(unused)] values: *const T,
- #[allow(unused)] out: *mut f32,
- #[allow(unused)] sums: *mut f32,
- #[allow(unused)] maxs: *mut f32,
- #[allow(unused)] gqa_factor: u32,
- #[allow(unused)] sequence_length: u32,
- #[allow(unused)] k_head_stride: u32,
- #[allow(unused)] k_seq_stride: u32,
- #[allow(unused)] v_head_stride: u32,
- #[allow(unused)] v_seq_stride: u32,
- #[allow(unused)] scale: f32,
- #[allow(unused)] num_heads: u32,
- #[allow(unused)] suffix_length: u32,
- #[allow(unused)]
- #[optional(float_mask)]
- fmask: Option<*const T>,
- #[allow(unused)]
- #[optional(has_mask)]
- mask_kv_seq_stride: Option<u32>,
- #[allow(unused)]
- #[optional(has_mask)]
- mask_q_seq_stride: Option<u32>,
- #[allow(unused)]
- #[optional(has_mask)]
- mask_head_stride: Option<u32>,
- #[allow(unused)]
- #[optional(has_sinks)]
- sinks: Option<*const f32>,
- #[allow(unused)]
- #[specialize]
- float_mask: bool,
- #[allow(unused)]
- #[specialize]
- has_mask: bool,
- #[allow(unused)]
- #[specialize]
- has_sinks: bool,
- #[allow(unused)]
- #[specialize]
- do_causal: bool,
+ queries: *const T,
+ keys: *const T,
+ values: *const T,
+ out: *mut f32,
+ sums: *mut f32,
+ maxs: *mut f32,
+ gqa_factor: u32,
+ sequence_length: u32,
+ k_head_stride: u32,
+ k_seq_stride: u32,
+ v_head_stride: u32,
+ v_seq_stride: u32,
+ scale: f32,
+ num_heads: u32,
+ suffix_length: u32,
+ #[optional(float_mask)] fmask: Option<*const T>,
+ #[optional(has_mask)] mask_kv_seq_stride: Option<u32>,
+ #[optional(has_mask)] mask_q_seq_stride: Option<u32>,
+ #[optional(has_mask)] mask_head_stride: Option<u32>,
+ #[optional(has_sinks)] sinks: Option<*const f32>,
+ #[specialize] float_mask: bool,
+ #[specialize] has_mask: bool,
+ #[specialize] has_sinks: bool,
+ #[specialize] do_causal: bool,
) {
- todo!()
+ let value_dim = HEAD_DIM;
+ let mut q = vec![0.0f32; HEAD_DIM as usize];
+ let mut o = vec![0.0f32; HEAD_DIM as usize];
+
+ for head_idx in 0..num_heads {
+ for q_seq_idx in 0..suffix_length {
+ for block_idx in 0..TOTAL_BLOCKS_COUNT {
+ let o_offset = q_seq_idx * num_heads + head_idx;
+ let q_offset = head_idx * suffix_length + q_seq_idx;
+ let kv_head_idx = head_idx / gqa_factor;
+
+ let queries_base: *const T = unsafe { queries.add((q_offset * HEAD_DIM) as usize) };
+ let keys_base: *const T = unsafe { keys.add((kv_head_idx * k_head_stride) as usize) };
+ let values_base: *const T = unsafe { values.add((kv_head_idx * v_head_stride) as usize) };
+ let out_base: *mut f32 =
+ unsafe { out.add((o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim) as usize) };
+
+ let fmask_base: Option<*const T> = if float_mask {
+ let offset = head_idx * mask_head_stride.unwrap_or(0) + q_seq_idx * mask_q_seq_stride.unwrap_or(0);
+ Some(unsafe { fmask.unwrap().add(offset as usize) })
+ } else {
+ None
+ };
+
+ // Read the query and scale
+ for j in 0..HEAD_DIM as usize {
+ q[j] = scale * unsafe { *queries_base.add(j) }.to_f32().unwrap();
+ }
+ o.fill(0f32);
+
+ let mut max_score = -1e9f32;
+ let mut sum_exp_score = 0.0f32;
+
+ if has_sinks && block_idx == 0 {
+ let q_head_idx = head_idx % num_heads;
+ max_score = unsafe { *sinks.unwrap().add(q_head_idx as usize) };
+ sum_exp_score = 1.0;
+ }
+
+ // Iterate over sequence positions assigned to this block
+ let mut chunk_start = block_idx * SEQUENCE_BLOCK_SIZE_1;
+ while chunk_start < sequence_length {
+ for t in 0..SEQUENCE_BLOCK_SIZE_1 {
+ let i = chunk_start + t;
+ if i >= sequence_length {
+ break;
+ }
+
+ let use_key = !do_causal || i <= (sequence_length - suffix_length + q_seq_idx);
+ if use_key {
+ let keys_ptr = unsafe { keys_base.add((i * k_seq_stride) as usize) };
+
+ // Compute the i-th score
+ let mut score = 0.0f32;
+ for j in 0..HEAD_DIM as usize {
+ score += q[j] * unsafe { *keys_ptr.add(j) }.to_f32().unwrap();
+ }
+ if float_mask {
+ let fmask_ptr =
+ unsafe { fmask_base.unwrap().add((i * mask_kv_seq_stride.unwrap()) as usize) };
+ score += f32::max(-1e9, unsafe { *fmask_ptr }.to_f32().unwrap());
+ }
+
+ // Update the accumulators
+ let new_max = f32::max(max_score, score);
+ let factor = (max_score - new_max).exp();
+ let exp_score = (score - new_max).exp();
+
+ max_score = new_max;
+ sum_exp_score = sum_exp_score * factor + exp_score;
+
+ // Update the output accumulator
+ let values_ptr = unsafe { values_base.add((i * v_seq_stride) as usize) };
+ for j in 0..HEAD_DIM as usize {
+ o[j] = o[j] * factor + exp_score * unsafe { *values_ptr.add(j) }.to_f32().unwrap();
+ }
+ }
+ }
+ chunk_start += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1;
+ }
+
+ // Write partial output, sum, and max for this block
+ for j in 0..HEAD_DIM as usize {
+ unsafe {
+ *out_base.add(j) = o[j];
+ }
+ }
+ unsafe {
+ *sums.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) = sum_exp_score;
+ *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) = max_score;
+ }
+ }
+ }
+ }
}
#[kernel(AttentionTwoPass2)]
#[variants(T, f32, f16, bf16)]
#[variants(HEAD_DIM, 64, 128, 256)]
pub fn attention_two_pass2<T: ArrayElement + Float, const HEAD_DIM: u32>(
- #[allow(unused)] partials: *const f32,
- #[allow(unused)] sums: *const f32,
- #[allow(unused)] maxs: *const f32,
- #[allow(unused)] out: *mut T,
- #[allow(unused)] num_heads: u32,
- #[allow(unused)] suffix_length: u32,
+ partials: *const f32,
+ sums: *const f32,
+ maxs: *const f32,
+ out: *mut T,
+ num_heads: u32,
+ suffix_length: u32,
) {
- todo!()
+ for head_idx in 0..num_heads {
+ for q_seq_idx in 0..suffix_length {
+ let o_offset = q_seq_idx * num_heads + head_idx;
+
+ // Find global max across all blocks
+ let mut global_max = f32::NEG_INFINITY;
+ for block_idx in 0..TOTAL_BLOCKS_COUNT {
+ let max_val = unsafe { *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };
+ global_max = f32::max(global_max, max_val);
+ }
+
+ // Compute global sum of exponentials with correction
+ let mut global_sum = 0.0f32;
+ for block_idx in 0..TOTAL_BLOCKS_COUNT {
+ let sum_val = unsafe { *sums.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };
+ let max_val = unsafe { *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };
+ global_sum += sum_val * (max_val - global_max).exp();
+ }
+
+ // Combine partial outputs
+ let out_ptr = unsafe { out.add((o_offset * HEAD_DIM) as usize) };
+ for j in 0..HEAD_DIM as usize {
+ let mut val = 0.0f32;
+ for block_idx in 0..TOTAL_BLOCKS_COUNT {
+ let partial_val = unsafe {
+ *partials.add((o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM + block_idx * HEAD_DIM) as usize + j)
+ };
+ let max_val = unsafe { *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };
+ val += partial_val * (max_val - global_max).exp();
+ }
+ unsafe {
+ *out_ptr.add(j) = T::from(val / global_sum).unwrap();
+ }
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/embedding_rows_sum.rs b/crates/uzu/src/backends/cpu/kernel/embedding/embedding_rows_sum.rs
index eccc8c7f1..24c7db766 100644
--- a/crates/uzu/src/backends/cpu/kernel/embedding/embedding_rows_sum.rs
+++ b/crates/uzu/src/backends/cpu/kernel/embedding/embedding_rows_sum.rs
@@ -7,13 +7,29 @@ use crate::ArrayElement;
#[kernel(EmbeddingRowsSum)]
#[variants(T, f32, f16, bf16)]
pub fn embedding_rows_sum<T: ArrayElement + Float>(
- #[allow(unused)] token_indices: *const u32,
- #[allow(unused)] weights: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] num_rows: u32,
- #[allow(unused)] total_rows: u32,
- #[allow(unused)] model_dim: u32,
- #[allow(unused)] codebook_stride: u32,
+ token_indices: *const u32,
+ weights: *const T,
+ output: *mut T,
+ num_rows: u32,
+ total_rows: u32,
+ model_dim: u32,
+ codebook_stride: u32,
) {
- todo!()
+ let codebook_stride = codebook_stride as usize;
+ let total_rows = total_rows as usize;
+ let model_dim = model_dim as usize;
+
+ unsafe {
+ for dim_idx in 0..model_dim {
+ let mut sum = 0f32;
+ for row_idx in 0..num_rows as usize {
+ let token = *token_indices.add(row_idx) as usize;
+ let row = row_idx * codebook_stride + token;
+ if row < total_rows {
+ sum += (*weights.add(row * model_dim + dim_idx)).to_f32().unwrap();
+ }
+ }
+ *output.add(dim_idx) = T::from(sum).unwrap();
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
index 7fa0e433a..363d9f644 100644
--- a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
+++ b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
@@ -2,7 +2,7 @@ use dsl::kernel;
use half::{bf16, f16};
use num_traits::Float;
-use crate::ArrayElement;
+use crate::{ArrayElement, backends::common::gpu_types::QuantizationMode};
#[kernel(QuantizedEmbeddingLookup)]
#[variants(T, f32, f16, bf16)]
@@ -23,5 +23,57 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
#[specialize]
quant_mode: u32,
) {
- todo!()
+ let quant_mode = QuantizationMode::from(quant_mode);
+
+ let packing_divisor: u32 = if quant_mode == QuantizationMode::UINT4 {
+ 2
+ } else {
+ 1
+ };
+ let weights_stride = model_dim / packing_divisor;
+ let num_groups = (model_dim + group_size - 1) / group_size;
+
+ unsafe {
+ for batch_idx in 0..batch_size {
+ let token_id = *token_ids.add(batch_idx as usize);
+
+ for dim_idx in 0..model_dim {
+ let out_idx = (batch_idx * model_dim + dim_idx) as usize;
+
+ if token_id >= vocab_size as u64 {
+ *output.add(out_idx) = T::zero();
+ continue;
+ }
+
+ let group_idx = dim_idx / group_size;
+ let scale = *scales.add((token_id as u32 * num_groups + group_idx) as usize);
+ let bias = *biases.add((token_id as u32 * num_groups + group_idx) as usize);
+
+ let quantized_value: i32 = match quant_mode {
+ QuantizationMode::UINT4 => {
+ let byte_idx = (token_id as u32 * weights_stride + dim_idx / 2) as usize;
+ let packed = *weights.add(byte_idx);
+ if (dim_idx & 1) == 0 {
+ (packed & 0x0F) as i32
+ } else {
+ ((packed >> 4) & 0x0F) as i32
+ }
+ },
+ QuantizationMode::INT8 => {
+ let elem_idx = (token_id as u32 * weights_stride + dim_idx) as usize;
+ let weights_i8 = weights as *const i8;
+ *weights_i8.add(elem_idx) as i32
+ },
+ QuantizationMode::UINT8 => {
+ let elem_idx = (token_id as u32 * weights_stride + dim_idx) as usize;
+ *weights.add(elem_idx) as i32
+ },
+ };
+
+ let out_f = scale.to_f32().unwrap() * quantized_value as f32 + bias.to_f32().unwrap();
+ let out_f = out_f * input_scale;
+ *output.add(out_idx) = T::from(out_f).unwrap();
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/kv_cache_update/kv_cache_update.rs b/crates/uzu/src/backends/cpu/kernel/kv_cache_update/kv_cache_update.rs
index 58e758c58..ebdaea8f3 100644
--- a/crates/uzu/src/backends/cpu/kernel/kv_cache_update/kv_cache_update.rs
+++ b/crates/uzu/src/backends/cpu/kernel/kv_cache_update/kv_cache_update.rs
@@ -7,13 +7,33 @@ use crate::ArrayElement;
#[kernel(KVCacheUpdate)]
#[variants(T, f32, bf16, f16)]
pub fn kv_cache_update<T: ArrayElement + Float>(
- #[allow(unused)] in_place_keys: *mut T,
- #[allow(unused)] in_place_values: *mut T,
- #[allow(unused)] swaps: &[crate::backends::common::gpu_types::kv_cache_update::Swap],
- #[allow(unused)] swap_count: u32,
- #[allow(unused)] num_heads: u32,
- #[allow(unused)] max_sequence_length: u32,
- #[allow(unused)] head_dim: u32,
+ in_place_keys: *mut T,
+ in_place_values: *mut T,
+ swaps: &[crate::backends::common::gpu_types::kv_cache_update::Swap],
+ swap_count: u32,
+ num_heads: u32,
+ max_sequence_length: u32,
+ head_dim: u32,
) {
- todo!()
+ let max_sequence_length = max_sequence_length as usize;
+ let head_dim = head_dim as usize;
+
+ for head_idx in 0..num_heads as usize {
+ for channel_idx in 0..head_dim {
+ for i in 0..swap_count as usize {
+ let head_offset = head_idx * max_sequence_length * head_dim;
+ let source_idx = head_offset + (swaps[i].source as usize) * head_dim + channel_idx;
+ let dest_idx = head_offset + (swaps[i].destination as usize) * head_dim + channel_idx;
+ unsafe {
+ let keys_src_ptr = in_place_keys.add(source_idx);
+ let mut keys_dst_ptr = in_place_keys.add(dest_idx);
+ keys_src_ptr.swap(keys_dst_ptr);
+
+ let values_src_ptr = in_place_values.add(source_idx);
+ let mut values_dst_ptr = in_place_values.add(dest_idx);
+ values_src_ptr.swap(values_dst_ptr);
+ }
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/layer_norm/layer_norm.rs b/crates/uzu/src/backends/cpu/kernel/layer_norm/layer_norm.rs
index 92e68ed43..79bd8a719 100644
--- a/crates/uzu/src/backends/cpu/kernel/layer_norm/layer_norm.rs
+++ b/crates/uzu/src/backends/cpu/kernel/layer_norm/layer_norm.rs
@@ -15,19 +15,62 @@ pub fn layer_norm<
OUT: ArrayElement + Float,
ACC: ArrayElement + Float,
>(
- #[allow(unused)]
- #[optional(!in_place)]
- input: Option<*const IN>,
- #[allow(unused)] scales: *const SC,
- #[allow(unused)] output: *mut OUT,
- #[allow(unused)] batch_size: u32,
- #[allow(unused)] model_dim: u32,
- #[allow(unused)] epsilon: f32,
- #[allow(unused)] scale_offset: f32,
- #[allow(unused)] full_layer: u32,
- #[allow(unused)]
- #[specialize]
- in_place: bool,
+ #[optional(!in_place)] input: Option<*const IN>,
+ scales: *const SC,
+ output: *mut OUT,
+ batch_size: u32,
+ model_dim: u32,
+ epsilon: f32,
+ scale_offset: f32,
+ full_layer: u32,
+ #[specialize] in_place: bool,
) {
- todo!()
+ let input = match in_place {
+ true => output as *const IN,
+ false => input.unwrap(),
+ };
+
+ let model_dim = model_dim as usize;
+ let epsilon = ACC::from(epsilon).unwrap();
+ let scale_offset = ACC::from(scale_offset).unwrap();
+ let element_count_accum = ACC::from(model_dim).unwrap();
+ let full_layer = full_layer != 0;
+
+ for batch in 0..(batch_size as usize) {
+ let batch_offset = batch * model_dim;
+
+ // Compute mean
+ let mut sum = ACC::zero();
+ for i in 0..model_dim {
+ let val = unsafe { ACC::from(*input.add(batch_offset + i)).unwrap() };
+ sum = sum + val;
+ }
+ let mean = sum / element_count_accum;
+
+ // Compute variance
+ let mut var_sum = ACC::zero();
+ for i in 0..model_dim {
+ let val = unsafe { ACC::from(*input.add(batch_offset + i)).unwrap() };
+ let centered = val - mean;
+ var_sum = var_sum + centered * centered;
+ }
+ let variance = var_sum / element_count_accum;
+ let inv_std = ACC::from((variance + epsilon).to_f32().unwrap().sqrt().recip()).unwrap();
+
+ // Normalize and scale
+ for i in 0..model_dim {
+ let input_val = unsafe { ACC::from(*input.add(batch_offset + i)).unwrap() };
+ let scale_val = unsafe { ACC::from(*scales.add(i)).unwrap() };
+ let normalized: ACC = (input_val - mean) * inv_std;
+ let result: OUT = if full_layer {
+ let scale_with_offset: ACC = scale_val + scale_offset;
+ OUT::from(normalized * scale_with_offset).unwrap()
+ } else {
+ let normalized_out = OUT::from(normalized).unwrap();
+ let scale_with_offset_out = OUT::from(scale_val + scale_offset).unwrap();
+ normalized_out * scale_with_offset_out
+ };
+ unsafe { *output.add(batch_offset + i) = result };
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs b/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs
index 97498c6e4..cb7733683 100644
--- a/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs
+++ b/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs
@@ -7,9 +7,14 @@ use crate::ArrayElement;
#[kernel(MaskUpdate)]
#[variants(T, f32, f16, bf16)]
pub fn mask_update<T: ArrayElement + Float>(
- #[allow(unused)] mask: *mut T,
- #[allow(unused)] unmask_col: i32,
- #[allow(unused)] mask_col: i32,
+ mask: *mut T,
+ unmask_col: i32,
+ mask_col: i32,
) {
- todo!()
+ if (unmask_col >= 0) {
+ unsafe { *mask.add(unmask_col as usize) = T::zero() };
+ }
+ if (mask_col >= 0) {
+ unsafe { *mask.add(mask_col as usize) = T::neg_infinity() };
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/pooling/pooling.rs b/crates/uzu/src/backends/cpu/kernel/pooling/pooling.rs
index 9daf2804b..90191d6c9 100644
--- a/crates/uzu/src/backends/cpu/kernel/pooling/pooling.rs
+++ b/crates/uzu/src/backends/cpu/kernel/pooling/pooling.rs
@@ -1,29 +1,56 @@
use dsl::kernel;
use half::{bf16, f16};
-use num_traits::Float;
+use num_traits::{Float, ToPrimitive};
use crate::ArrayElement;
#[kernel(PoolingCls)]
#[variants(T, f32, f16, bf16)]
pub fn pooling_cls<T: ArrayElement + Float>(
- #[allow(unused)] input: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] seq_len: u32,
- #[allow(unused)] hidden_dim: u32,
- #[allow(unused)] batch_size: u32,
+ input: *const T,
+ output: *mut T,
+ seq_len: u32,
+ hidden_dim: u32,
+ batch_size: u32,
) {
- todo!()
+ let seq_len = seq_len as usize;
+ let hidden_dim = hidden_dim as usize;
+ let batch_size = batch_size as usize;
+
+ for dim_idx in 0..hidden_dim {
+ for batch_idx in 0..batch_size {
+ unsafe {
+ *output.add(batch_idx * hidden_dim + dim_idx) = *input.add(batch_idx * hidden_dim * seq_len + dim_idx);
+ }
+ }
+ }
}
#[kernel(PoolingMean)]
#[variants(T, f32, f16, bf16)]
pub fn pooling_mean<T: ArrayElement + Float>(
- #[allow(unused)] input: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] seq_len: u32,
- #[allow(unused)] hidden_dim: u32,
- #[allow(unused)] batch_size: u32,
+ input: *const T,
+ output: *mut T,
+ seq_len: u32,
+ hidden_dim: u32,
+ batch_size: u32,
) {
- todo!()
+ let seq_len = seq_len as usize;
+ let hidden_dim = hidden_dim as usize;
+ let batch_size = batch_size as usize;
+
+ for dim_idx in 0..hidden_dim {
+ for batch_idx in 0..batch_size {
+ let mut sum = 0f32;
+ for seq_idx in 0..seq_len {
+ unsafe {
+ let position = batch_idx * seq_len * hidden_dim + seq_idx * hidden_dim + dim_idx;
+ sum += (*input.add(position)).to_f32().unwrap();
+ }
+ }
+ unsafe {
+ *output.add(batch_idx * hidden_dim + dim_idx) = T::from(sum / seq_len.to_f32().unwrap()).unwrap();
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/bitmask.rs b/crates/uzu/src/backends/cpu/kernel/sampling/bitmask.rs
index e99000ba7..4fac35672 100644
--- a/crates/uzu/src/backends/cpu/kernel/sampling/bitmask.rs
+++ b/crates/uzu/src/backends/cpu/kernel/sampling/bitmask.rs
@@ -7,16 +7,33 @@ use crate::ArrayElement;
#[kernel(Bitmask)]
#[variants(T, f32, f16, bf16)]
pub fn bitmask<T: ArrayElement + Float>(
- #[allow(unused)]
- #[optional(!in_place)]
- logits: Option<*const T>,
- #[allow(unused)] bitmask: *const u32,
- #[allow(unused)] processed_logits: *mut T,
- #[allow(unused)] batch_size: u32,
- #[allow(unused)] vocab_size: u32,
- #[allow(unused)]
- #[specialize]
- in_place: bool,
+ #[optional(!in_place)] logits: Option<*const T>,
+ bitmask: *const u32,
+ processed_logits: *mut T,
+ batch_size: u32,
+ vocab_size: u32,
+ #[specialize] in_place: bool,
) {
- todo!()
+ let logits: *const T = match in_place {
+ true => processed_logits,
+ false => logits.unwrap(),
+ };
+
+ let batch_size = batch_size as usize;
+ let vocab_size = vocab_size as usize;
+ let bitmask_size = vocab_size.div_ceil(32);
+
+ for batch_idx in 0..batch_size {
+ for token_idx in 0..vocab_size {
+ let global_idx = batch_idx * vocab_size + token_idx;
+ let bitmask_idx = batch_idx * bitmask_size + token_idx / 32;
+ unsafe {
+ let mask = (*bitmask.add(bitmask_idx) >> (token_idx % 32)) & 1;
+ *processed_logits.add(global_idx) = match mask != 0 {
+ true => *logits.add(global_idx),
+ false => T::neg_infinity(),
+ };
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/min_p.rs b/crates/uzu/src/backends/cpu/kernel/sampling/min_p.rs
index c32c6c23a..71af24405 100644
--- a/crates/uzu/src/backends/cpu/kernel/sampling/min_p.rs
+++ b/crates/uzu/src/backends/cpu/kernel/sampling/min_p.rs
@@ -7,16 +7,40 @@ use crate::ArrayElement;
#[kernel(MinP)]
#[variants(T, f32, f16, bf16)]
pub fn min_p<T: ArrayElement + Float>(
- #[allow(unused)]
- #[optional(!in_place)]
- logits: Option<*const T>,
- #[allow(unused)] processed_logits: *mut T,
- #[allow(unused)] batch_size: u32,
- #[allow(unused)] vocab_size: u32,
- #[allow(unused)] min_p: f32,
- #[allow(unused)]
- #[specialize]
- in_place: bool,
+ #[optional(!in_place)] logits: Option<*const T>,
+ processed_logits: *mut T,
+ batch_size: u32,
+ vocab_size: u32,
+ min_p: f32,
+ #[specialize] in_place: bool,
) {
- todo!()
+ let logits = match in_place {
+ true => processed_logits,
+ false => logits.unwrap(),
+ };
+
+ let batch_size = batch_size as usize;
+ let vocab_size = vocab_size as usize;
+
+ for batch_idx in 0..batch_size {
+ // find maximum logit
+ let mut max_logit = f32::NEG_INFINITY;
+ for i in 0..vocab_size {
+ let logit = unsafe { *logits.add(batch_idx * vocab_size + i) }.to_f32().unwrap();
+ max_logit = max_logit.max(logit);
+ }
+
+ // then the threshold is just max_logit + log(min_p), mask everything strictly below it
+ let threshold: T = T::from(max_logit + min_p.ln()).unwrap();
+ for i in 0..vocab_size {
+ let position = batch_idx * vocab_size + i;
+ unsafe {
+ let logit = *logits.add(position);
+ *processed_logits.add(position) = match logit >= threshold {
+ true => logit,
+ false => T::neg_infinity(),
+ }
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs b/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs
index dc3aacc56..ba1ad5b59 100644
--- a/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs
+++ b/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs
@@ -7,9 +7,15 @@ use crate::ArrayElement;
#[kernel(Sigmoid)]
#[variants(T, f32, f16, bf16)]
pub fn sigmoid<T: ArrayElement + Float>(
- #[allow(unused)] input: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] total_elements: u32,
+ input: *const T,
+ output: *mut T,
+ total_elements: u32,
) {
- todo!()
+ for i in 0..total_elements as usize {
+ unsafe {
+ let input_value = (*input.add(i)).to_f32().unwrap();
+ let value = 1.0 / (1.0 + (-input_value).exp());
+ *output.add(i) = T::from(value).unwrap();
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs
index e0d2289b5..f61c5f917 100644
--- a/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs
+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs
@@ -7,12 +7,20 @@ use crate::ArrayElement;
#[kernel(TensorAddScale)]
#[variants(T, f32, f16, bf16)]
pub fn tensor_add_scale<T: ArrayElement + Float>(
- #[allow(unused)] input: *const T,
- #[allow(unused)] bias: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] num_cols: u32,
- #[allow(unused)] length: u32,
- #[allow(unused)] scale: f32,
+ input: *const T,
+ bias: *const T,
+ output: *mut T,
+ num_cols: u32,
+ length: u32,
+ scale: f32,
) {
- todo!()
+ let num_cols = num_cols as usize;
+ for position in 0..length as usize {
+ let col = position % num_cols;
+ unsafe {
+ let input_value = (*input.add(position)).to_f32().unwrap();
+ let bias_value = (*bias.add(col)).to_f32().unwrap();
+ *output.add(position) = T::from((input_value + bias_value) * scale).unwrap();
+ }
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
index 4ddc4c8b7..e4adb869f 100644
--- a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
+++ b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
@@ -9,8 +9,6 @@ PUBLIC KERNEL(Sigmoid) (
const constant uint& total_elements,
uint tid AXIS(total_elements, 256)
) {
- if (tid >= total_elements)
- return;
float x = float(input[tid]);
output[tid] = T(1.0f / (1.0f + exp(-x)));
}
\ No newline at end of file
diff --git a/crates/uzu/tests/common/assert.rs b/crates/uzu/tests/common/assert.rs
index 5cf17e43f..ec9711815 100644
--- a/crates/uzu/tests/common/assert.rs
+++ b/crates/uzu/tests/common/assert.rs
@@ -18,6 +18,10 @@ pub fn assert_eq_float<T: ArrayElement + Float + Display>(
);
for i in 0..expected.len() {
+ if expected[i] == actual[i] {
+ continue;
+ }
+
let diff = (expected[i] - actual[i]).to_f32().unwrap().abs();
assert!(
diff < eps,
diff --git a/crates/uzu/tests/unit/kernel/activation_test.rs b/crates/uzu/tests/unit/kernel/activation_test.rs
new file mode 100644
index 000000000..a06798046
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/activation_test.rs
@@ -0,0 +1,260 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, gpu_types::ActivationType, kernel::ActivationKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ data: Box<[T]>,
+ act_type: ActivationType,
+ in_place: bool,
+}
+
+fn make_data<T: ArrayElement + Float>(n: usize) -> Vec<T> {
+ let mut data = Vec::with_capacity(n);
+ for i in 0..n {
+ // Use a range of values including negative to exercise both branches of activations
+ data.push(T::from((i as f32 * 0.3) - 2.0).unwrap());
+ }
+ data
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+ let kernel = <<B as Backend>::Kernels as Kernels>::ActivationKernel::new(&context, T::data_type(), input.in_place)
+ .expect("Failed to create ActivationKernel");
+
+ let n = input.data.len();
+
+ if input.in_place {
+ let output_array = context.create_array_from(&[n], &input.data, "");
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");
+ kernel.encode(
+ Option::<&B::Buffer>::None,
+ output_array.buffer().borrow_mut().deref_mut(),
+ n as u32,
+ input.act_type,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ output_array.as_slice().to_vec()
+ } else {
+ let input_array = context.create_array_from(&[n], &input.data, "");
+ let output_array = context.create_array_uninitialized(&[n], T::data_type(), "");
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");
+ kernel.encode(
+ Some(input_array.buffer().borrow().deref()),
+ output_array.buffer().borrow_mut().deref_mut(),
+ n as u32,
+ input.act_type,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ output_array.as_slice().to_vec()
+ }
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ act_type: ActivationType,
+ in_place: bool,
+) -> (Input<T>, Vec<T>) {
+ let n = 128;
+ let data = make_data::<T>(n);
+
+ let input = Input {
+ data: data.into_boxed_slice(),
+ act_type,
+ in_place,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+/// Large input to exercise more threads on GPU.
+fn get_test_data_large<T: ArrayElement + Float>(
+ act_type: ActivationType,
+ in_place: bool,
+) -> (Input<T>, Vec<T>) {
+ let n = 4096;
+ let data = make_data::<T>(n);
+
+ let input = Input {
+ data: data.into_boxed_slice(),
+ act_type,
+ in_place,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &Input<T>,
+ expected: &[T],
+ test_name: &str,
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.02f32
+ } else {
+ 1e-5
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let actual = get_output::<T, B>(input);
+ let msg = format!("Activation {} failed with backend={}", test_name, std::any::type_name::<B>(),);
+ assert_eq_float::<T>(expected, &actual, eps, &msg);
+ });
+}
+
+fn test_activation<T: ArrayElement + Float + Debug + Display>(
+ act_type: ActivationType,
+ in_place: bool,
+) {
+ let label = format!(
+ "{:?}_{}",
+ act_type,
+ if in_place {
+ "in_place"
+ } else {
+ "out_of_place"
+ }
+ );
+ let (input, expected) = get_test_data::<T>(act_type, in_place);
+ test_internal::<T>(&input, &expected, &label);
+}
+
+fn test_activation_large<T: ArrayElement + Float + Debug + Display>(
+ act_type: ActivationType,
+ in_place: bool,
+) {
+ let label = format!(
+ "{:?}_{}_large",
+ act_type,
+ if in_place {
+ "in_place"
+ } else {
+ "out_of_place"
+ }
+ );
+ let (input, expected) = get_test_data_large::<T>(act_type, in_place);
+ test_internal::<T>(&input, &expected, &label);
+}
+
+// SILU out-of-place tests
+#[test]
+fn test_silu_f32() {
+ test_activation::<f32>(ActivationType::SILU, false);
+}
+
+#[test]
+fn test_silu_f16() {
+ test_activation::<f16>(ActivationType::SILU, false);
+}
+
+#[test]
+fn test_silu_bf16() {
+ test_activation::<bf16>(ActivationType::SILU, false);
+}
+
+// SILU in-place tests
+#[test]
+fn test_silu_in_place_f32() {
+ test_activation::<f32>(ActivationType::SILU, true);
+}
+
+#[test]
+fn test_silu_in_place_f16() {
+ test_activation::<f16>(ActivationType::SILU, true);
+}
+
+#[test]
+fn test_silu_in_place_bf16() {
+ test_activation::<bf16>(ActivationType::SILU, true);
+}
+
+// GELU out-of-place tests
+#[test]
+fn test_gelu_f32() {
+ test_activation::<f32>(ActivationType::GELU, false);
+}
+
+#[test]
+fn test_gelu_f16() {
+ test_activation::<f16>(ActivationType::GELU, false);
+}
+
+#[test]
+fn test_gelu_bf16() {
+ test_activation::<bf16>(ActivationType::GELU, false);
+}
+
+// GELU in-place tests
+#[test]
+fn test_gelu_in_place_f32() {
+ test_activation::<f32>(ActivationType::GELU, true);
+}
+
+#[test]
+fn test_gelu_in_place_f16() {
+ test_activation::<f16>(ActivationType::GELU, true);
+}
+
+#[test]
+fn test_gelu_in_place_bf16() {
+ test_activation::<bf16>(ActivationType::GELU, true);
+}
+
+// Large SILU tests
+#[test]
+fn test_silu_large_f32() {
+ test_activation_large::<f32>(ActivationType::SILU, false);
+}
+
+#[test]
+fn test_silu_large_f16() {
+ test_activation_large::<f16>(ActivationType::SILU, false);
+}
+
+#[test]
+fn test_silu_large_bf16() {
+ test_activation_large::<bf16>(ActivationType::SILU, false);
+}
+
+// Large GELU tests
+#[test]
+fn test_gelu_large_f32() {
+ test_activation_large::<f32>(ActivationType::GELU, false);
+}
+
+#[test]
+fn test_gelu_large_f16() {
+ test_activation_large::<f16>(ActivationType::GELU, false);
+}
+
+#[test]
+fn test_gelu_large_bf16() {
+ test_activation_large::<bf16>(ActivationType::GELU, false);
+}
+
+// Large in-place tests
+#[test]
+fn test_silu_in_place_large_f32() {
+ test_activation_large::<f32>(ActivationType::SILU, true);
+}
+
+#[test]
+fn test_gelu_in_place_large_f32() {
+ test_activation_large::<f32>(ActivationType::GELU, true);
+}
diff --git a/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs
new file mode 100644
index 000000000..f9b2936fa
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs
@@ -0,0 +1,480 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{
+ Backend, Context, Encoder, Kernels,
+ kernel::{AttentionTwoPass1Kernel, AttentionTwoPass2Kernel},
+ },
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+const TOTAL_BLOCKS_COUNT: u32 = 32;
+
+// --- First pass types and helpers ---
+
+struct FirstPassInput<T: ArrayElement + Float> {
+ queries: Box<[T]>,
+ keys: Box<[T]>,
+ values: Box<[T]>,
+ mask: Option<Box<[T]>>,
+ num_heads: u32,
+ gqa_factor: u32,
+ sequence_length: u32,
+ suffix_length: u32,
+ head_dim: u32,
+ scale: f32,
+ do_causal: bool,
+ has_mask: bool,
+}
+
+struct FirstPassOutput {
+ partials: Vec<f32>,
+ sums: Vec<f32>,
+ maxs: Vec<f32>,
+}
+
+fn get_first_pass_input<T: ArrayElement + Float>(
+ num_heads: u32,
+ num_kv_heads: u32,
+ sequence_length: u32,
+ suffix_length: u32,
+ head_dim: u32,
+ do_causal: bool,
+ with_mask: bool,
+) -> FirstPassInput<T> {
+ let gqa_factor = num_heads / num_kv_heads;
+
+ let q_size = (num_heads * suffix_length * head_dim) as usize;
+ let mut queries = vec![T::zero(); q_size];
+ for i in 0..q_size {
+ queries[i] = T::from((i as f32 * 0.13 + 0.5).sin() * 0.5).unwrap();
+ }
+
+ let k_size = (num_kv_heads * sequence_length * head_dim) as usize;
+ let mut keys = vec![T::zero(); k_size];
+ for i in 0..k_size {
+ keys[i] = T::from((i as f32 * 0.07 + 1.0).cos() * 0.5).unwrap();
+ }
+
+ let v_size = (num_kv_heads * sequence_length * head_dim) as usize;
+ let mut values = vec![T::zero(); v_size];
+ for i in 0..v_size {
+ values[i] = T::from((i as f32 * 0.11 + 2.0).sin() * 0.5).unwrap();
+ }
+
+ let mask = if with_mask {
+ let m_size = (suffix_length * sequence_length) as usize;
+ let mut m = vec![T::zero(); m_size];
+ for q in 0..suffix_length {
+ for k in 0..sequence_length {
+ let idx = (q * sequence_length + k) as usize;
+ let diff = (k as i32) - (sequence_length as i32 - suffix_length as i32 + q as i32);
+ if diff.abs() > 3 {
+ m[idx] = T::from(-1e9f32).unwrap();
+ }
+ }
+ }
+ Some(m.into_boxed_slice())
+ } else {
+ None
+ };
+
+ let scale = 1.0 / (head_dim as f32).sqrt();
+
+ FirstPassInput {
+ queries: queries.into_boxed_slice(),
+ keys: keys.into_boxed_slice(),
+ values: values.into_boxed_slice(),
+ mask,
+ num_heads,
+ gqa_factor,
+ sequence_length,
+ suffix_length,
+ head_dim,
+ scale,
+ do_causal,
+ has_mask: with_mask,
+ }
+}
+
+fn get_first_pass_output<T: ArrayElement + Float, B: Backend>(input: &FirstPassInput<T>) -> FirstPassOutput {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let float_mask = input.has_mask;
+ let kernel = <<B as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
+ &context,
+ T::data_type(),
+ input.head_dim,
+ float_mask,
+ input.has_mask,
+ false,
+ input.do_causal,
+ )
+ .expect("Failed to create AttentionTwoPass1Kernel");
+
+ let queries_array = context.create_array_from(&[input.queries.len()], &input.queries, "");
+ let keys_array = context.create_array_from(&[input.keys.len()], &input.keys, "");
+ let values_array = context.create_array_from(&[input.values.len()], &input.values, "");
+
+ let total_offsets = (input.suffix_length * input.num_heads) as usize;
+ let partials_size = total_offsets * TOTAL_BLOCKS_COUNT as usize * input.head_dim as usize;
+ let sums_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;
+ let maxs_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;
+
+ let partials_array = context.create_array_uninitialized(&[partials_size], DataType::F32, "");
+ let sums_array = context.create_array_uninitialized(&[sums_size], DataType::F32, "");
+ let maxs_array = context.create_array_uninitialized(&[maxs_size], DataType::F32, "");
+
+ let mask_array = input.mask.as_ref().map(|m| context.create_array_from(&[m.len()], m, ""));
+ let mask_buf_rc = mask_array.as_ref().map(|a| a.buffer());
+ let mask_buf_borrow = mask_buf_rc.as_ref().map(|rc| rc.borrow());
+ let mask_buffer: Option<&B::Buffer> = mask_buf_borrow.as_ref().map(|b| b.deref());
+
+ let mask_kv_seq_stride: Option<u32> = input.has_mask.then_some(1);
+ let mask_q_seq_stride: Option<u32> = input.has_mask.then_some(input.sequence_length);
+ let mask_head_stride: Option<u32> = input.has_mask.then_some(0);
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ queries_array.buffer().borrow().deref(),
+ keys_array.buffer().borrow().deref(),
+ values_array.buffer().borrow().deref(),
+ partials_array.buffer().borrow_mut().deref_mut(),
+ sums_array.buffer().borrow_mut().deref_mut(),
+ maxs_array.buffer().borrow_mut().deref_mut(),
+ input.gqa_factor,
+ input.sequence_length,
+ input.sequence_length * input.head_dim,
+ input.head_dim,
+ input.sequence_length * input.head_dim,
+ input.head_dim,
+ input.scale,
+ input.num_heads,
+ input.suffix_length,
+ mask_buffer,
+ mask_kv_seq_stride,
+ mask_q_seq_stride,
+ mask_head_stride,
+ None::<&B::Buffer>,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ FirstPassOutput {
+ partials: partials_array.as_slice().to_vec(),
+ sums: sums_array.as_slice().to_vec(),
+ maxs: maxs_array.as_slice().to_vec(),
+ }
+}
+
+// --- Second pass types and helpers ---
+
+struct SecondPassInput {
+ partials: Box<[f32]>,
+ sums: Box<[f32]>,
+ maxs: Box<[f32]>,
+ num_heads: u32,
+ suffix_length: u32,
+ head_dim: u32,
+}
+
+fn get_second_pass_input(
+ num_heads: u32,
+ suffix_length: u32,
+ head_dim: u32,
+) -> SecondPassInput {
+ let total_offsets = (suffix_length * num_heads) as usize;
+ let partials_size = total_offsets * TOTAL_BLOCKS_COUNT as usize * head_dim as usize;
+ let sums_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;
+ let maxs_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;
+
+ let mut partials = vec![0.0f32; partials_size];
+ for i in 0..partials_size {
+ partials[i] = (i as f32 * 0.03 + 0.1).sin() * 0.5;
+ }
+
+ let mut sums = vec![0.0f32; sums_size];
+ for i in 0..sums_size {
+ // Ensure sums are positive (they represent sum of exp scores)
+ sums[i] = (i as f32 * 0.07 + 1.0).cos().abs() + 0.1;
+ }
+
+ let mut maxs = vec![0.0f32; maxs_size];
+ for i in 0..maxs_size {
+ maxs[i] = (i as f32 * 0.05 + 0.5).sin() * 2.0;
+ }
+
+ SecondPassInput {
+ partials: partials.into_boxed_slice(),
+ sums: sums.into_boxed_slice(),
+ maxs: maxs.into_boxed_slice(),
+ num_heads,
+ suffix_length,
+ head_dim,
+ }
+}
+
+fn get_second_pass_output<T: ArrayElement + Float, B: Backend>(input: &SecondPassInput) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel =
+ <<B as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(&context, T::data_type(), input.head_dim)
+ .expect("Failed to create AttentionTwoPass2Kernel");
+
+ let partials_array = context.create_array_from(&[input.partials.len()], &input.partials, "");
+ let sums_array = context.create_array_from(&[input.sums.len()], &input.sums, "");
+ let maxs_array = context.create_array_from(&[input.maxs.len()], &input.maxs, "");
+
+ let output_size = (input.suffix_length * input.num_heads * input.head_dim) as usize;
+ let output_array = context.create_array_uninitialized(&[output_size], T::data_type(), "");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ partials_array.buffer().borrow().deref(),
+ sums_array.buffer().borrow().deref(),
+ maxs_array.buffer().borrow().deref(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.num_heads,
+ input.suffix_length,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+// --- First pass tests ---
+
+fn test_first_pass_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &FirstPassInput<T>,
+ expected: &FirstPassOutput,
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-2
+ } else {
+ 1e-5
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_first_pass_output::<T, B>(input);
+ let msg = format!(
+ "AttentionTwoPass1 failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={}, mask={})",
+ std::any::type_name::<B>(),
+ input.num_heads,
+ input.sequence_length,
+ input.suffix_length,
+ input.head_dim,
+ input.do_causal,
+ input.has_mask,
+ );
+ assert_eq_float::<f32>(&expected.partials, &output.partials, eps, &format!("{msg} [partials]"));
+ assert_eq_float::<f32>(&expected.sums, &output.sums, eps, &format!("{msg} [sums]"));
+ assert_eq_float::<f32>(&expected.maxs, &output.maxs, eps, &format!("{msg} [maxs]"));
+ });
+}
+
+fn test_first_pass_basic<T: ArrayElement + Float + Debug + Display>() {
+ let input = get_first_pass_input::<T>(4, 4, 8, 1, 64, false, false);
+ let expected = get_first_pass_output::<T, Cpu>(&input);
+ test_first_pass_internal(&input, &expected);
+
+ let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, false, false);
+ let expected = get_first_pass_output::<T, Cpu>(&input);
+ test_first_pass_internal(&input, &expected);
+}
+
+fn test_first_pass_causal<T: ArrayElement + Float + Debug + Display>() {
+ let input = get_first_pass_input::<T>(4, 4, 16, 1, 64, true, false);
+ let expected = get_first_pass_output::<T, Cpu>(&input);
+ test_first_pass_internal(&input, &expected);
+
+ let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, true, false);
+ let expected = get_first_pass_output::<T, Cpu>(&input);
+ test_first_pass_internal(&input, &expected);
+}
+
+fn test_first_pass_gqa<T: ArrayElement + Float + Debug + Display>() {
+ let input = get_first_pass_input::<T>(8, 2, 8, 1, 64, false, false);
+ let expected = get_first_pass_output::<T, Cpu>(&input);
+ test_first_pass_internal(&input, &expected);
+
+ let input = get_first_pass_input::<T>(8, 2, 8, 4, 64, true, false);
+ let expected = get_first_pass_output::<T, Cpu>(&input);
+ test_first_pass_internal(&input, &expected);
+}
+
+fn test_first_pass_mask<T: ArrayElement + Float + Debug + Display>() {
+ let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, false, true);
+ let expected = get_first_pass_output::<T, Cpu>(&input);
+ test_first_pass_internal(&input, &expected);
+
+ let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, true, true);
+ let expected = get_first_pass_output::<T, Cpu>(&input);
+ test_first_pass_internal(&input, &expected);
+}
+
+fn test_first_pass_head_dim_128<T: ArrayElement + Float + Debug + Display>() {
+ let input = get_first_pass_input::<T>(4, 4, 8, 2, 128, true, false);
+ let expected = get_first_pass_output::<T, Cpu>(&input);
+ test_first_pass_internal(&input, &expected);
+}
+
+// --- Second pass tests ---
+
+fn test_second_pass_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &SecondPassInput,
+ expected: &[T],
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-2
+ } else {
+ 1e-5
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_second_pass_output::<T, B>(input);
+ let msg = format!(
+ "AttentionTwoPass2 failed (backend={}, heads={}, suffix={}, head_dim={})",
+ std::any::type_name::<B>(),
+ input.num_heads,
+ input.suffix_length,
+ input.head_dim,
+ );
+ assert_eq_float::<T>(expected, &output, eps, &msg);
+ });
+}
+
+fn test_second_pass_basic<T: ArrayElement + Float + Debug + Display>() {
+ let input = get_second_pass_input(4, 1, 64);
+ let expected = get_second_pass_output::<T, Cpu>(&input);
+ test_second_pass_internal::<T>(&input, &expected);
+
+ let input = get_second_pass_input(4, 4, 64);
+ let expected = get_second_pass_output::<T, Cpu>(&input);
+ test_second_pass_internal::<T>(&input, &expected);
+}
+
+fn test_second_pass_head_dim_128<T: ArrayElement + Float + Debug + Display>() {
+ let input = get_second_pass_input(4, 2, 128);
+ let expected = get_second_pass_output::<T, Cpu>(&input);
+ test_second_pass_internal::<T>(&input, &expected);
+}
+
+// --- First pass test entries ---
+
+#[test]
+fn test_first_pass_basic_f32() {
+ test_first_pass_basic::<f32>();
+}
+
+#[test]
+fn test_first_pass_basic_f16() {
+ test_first_pass_basic::<f16>();
+}
+
+#[test]
+fn test_first_pass_basic_bf16() {
+ test_first_pass_basic::<bf16>();
+}
+
+#[test]
+fn test_first_pass_causal_f32() {
+ test_first_pass_causal::<f32>();
+}
+
+#[test]
+fn test_first_pass_causal_f16() {
+ test_first_pass_causal::<f16>();
+}
+
+#[test]
+fn test_first_pass_causal_bf16() {
+ test_first_pass_causal::<bf16>();
+}
+
+#[test]
+fn test_first_pass_gqa_f32() {
+ test_first_pass_gqa::<f32>();
+}
+
+#[test]
+fn test_first_pass_gqa_f16() {
+ test_first_pass_gqa::<f16>();
+}
+
+#[test]
+fn test_first_pass_gqa_bf16() {
+ test_first_pass_gqa::<bf16>();
+}
+
+#[test]
+fn test_first_pass_mask_f32() {
+ test_first_pass_mask::<f32>();
+}
+
+#[test]
+fn test_first_pass_mask_f16() {
+ test_first_pass_mask::<f16>();
+}
+
+#[test]
+fn test_first_pass_mask_bf16() {
+ test_first_pass_mask::<bf16>();
+}
+
+#[test]
+fn test_first_pass_head_dim_128_f32() {
+ test_first_pass_head_dim_128::<f32>();
+}
+
+#[test]
+fn test_first_pass_head_dim_128_f16() {
+ test_first_pass_head_dim_128::<f16>();
+}
+
+#[test]
+fn test_first_pass_head_dim_128_bf16() {
+ test_first_pass_head_dim_128::<bf16>();
+}
+
+// --- Second pass test entries ---
+
+#[test]
+fn test_second_pass_basic_f32() {
+ test_second_pass_basic::<f32>();
+}
+
+#[test]
+fn test_second_pass_basic_f16() {
+ test_second_pass_basic::<f16>();
+}
+
+#[test]
+fn test_second_pass_basic_bf16() {
+ test_second_pass_basic::<bf16>();
+}
+
+#[test]
+fn test_second_pass_head_dim_128_f32() {
+ test_second_pass_head_dim_128::<f32>();
+}
+
+#[test]
+fn test_second_pass_head_dim_128_f16() {
+ test_second_pass_head_dim_128::<f16>();
+}
+
+#[test]
+fn test_second_pass_head_dim_128_bf16() {
+ test_second_pass_head_dim_128::<bf16>();
+}
diff --git a/crates/uzu/tests/unit/kernel/attention/mod.rs b/crates/uzu/tests/unit/kernel/attention/mod.rs
index 7f8328e4b..661eb7fb1 100644
--- a/crates/uzu/tests/unit/kernel/attention/mod.rs
+++ b/crates/uzu/tests/unit/kernel/attention/mod.rs
@@ -1,4 +1,5 @@
mod attention_gemm_test;
mod attention_single_pass_test;
+mod attention_two_pass_test;
mod attention_update_kv_cache_test;
mod sigmoid_gate_test;
diff --git a/crates/uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs b/crates/uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs
new file mode 100644
index 000000000..d3cb57570
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs
@@ -0,0 +1,229 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::EmbeddingRowsSumKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ token_indices: Box<[u32]>,
+ weights: Box<[T]>,
+ num_rows: u32,
+ total_rows: u32,
+ model_dim: u32,
+ codebook_stride: u32,
+}
+
+fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let num_rows = 4u32;
+ let codebook_size = 8u32;
+ let codebook_stride = codebook_size;
+ let total_rows = num_rows * codebook_size;
+ let model_dim = 513u32;
+
+ let token_indices: Box<[u32]> = Box::new([3, 7, 1, 5]);
+ let weights: Vec<T> = (0..total_rows as usize * model_dim as usize)
+ .map(|i| T::from((i as f32 * 0.1).sin() * 10.0).unwrap())
+ .collect();
+
+ let input = Input {
+ token_indices,
+ weights: weights.into_boxed_slice(),
+ num_rows,
+ total_rows,
+ model_dim,
+ codebook_stride,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_test_data_edge<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ // Single row, small dim
+ let num_rows = 1u32;
+ let codebook_stride = 4u32;
+ let total_rows = 4u32;
+ let model_dim = 4u32;
+
+ let token_indices: Box<[u32]> = Box::new([2]);
+ let weights: Vec<T> =
+ (0..total_rows as usize * model_dim as usize).map(|i| T::from(i as f32 + 1.0).unwrap()).collect();
+
+ let input = Input {
+ token_indices,
+ weights: weights.into_boxed_slice(),
+ num_rows,
+ total_rows,
+ model_dim,
+ codebook_stride,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_test_data_oob<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ // Token index causes row to exceed total_rows, should be skipped
+ let num_rows = 2u32;
+ let codebook_stride = 4u32;
+ let total_rows = 4u32;
+ let model_dim = 4u32;
+
+ // row_idx=0: row = 0*4+1 = 1 (valid)
+ // row_idx=1: row = 1*4+3 = 7 (out of bounds, total_rows=4)
+ let token_indices: Box<[u32]> = Box::new([1, 3]);
+ let weights: Vec<T> =
+ (0..total_rows as usize * model_dim as usize).map(|i| T::from(i as f32 + 1.0).unwrap()).collect();
+
+ let input = Input {
+ token_indices,
+ weights: weights.into_boxed_slice(),
+ num_rows,
+ total_rows,
+ model_dim,
+ codebook_stride,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::EmbeddingRowsSumKernel::new(&context, T::data_type())
+ .expect("Failed to create EmbeddingRowsSumKernel");
+
+ let token_indices_array = context.create_array_from(&[input.num_rows as usize], &input.token_indices, "");
+ let weights_array =
+ context.create_array_from(&[input.total_rows as usize, input.model_dim as usize], &input.weights, "");
+ let output_array = context.create_array_uninitialized(&[input.model_dim as usize], T::data_type(), "");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ token_indices_array.buffer().borrow().deref(),
+ weights_array.buffer().borrow().deref(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.num_rows,
+ input.total_rows,
+ input.model_dim,
+ input.codebook_stride,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.1f32
+ } else {
+ 1e-4
+ };
+ let (input, expected) = get_test_data_basic::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!("EmbeddingRowsSum basic test failed for backend {}", std::any::type_name::<B>()),
+ );
+ });
+}
+
+fn test_edge<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.05f32
+ } else {
+ 1e-5
+ };
+ let (input, expected) = get_test_data_edge::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!("EmbeddingRowsSum edge test failed for backend {}", std::any::type_name::<B>()),
+ );
+ });
+}
+
+fn test_oob<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.05f32
+ } else {
+ 1e-5
+ };
+ let (input, expected) = get_test_data_oob::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!("EmbeddingRowsSum oob test failed for backend {}", std::any::type_name::<B>()),
+ );
+ });
+}
+
+// basic tests
+#[test]
+fn test_basic_f32() {
+ test_basic::<f32>();
+}
+
+#[test]
+fn test_basic_f16() {
+ test_basic::<f16>();
+}
+
+#[test]
+fn test_basic_bf16() {
+ test_basic::<bf16>();
+}
+
+// edge tests
+#[test]
+fn test_edge_f32() {
+ test_edge::<f32>();
+}
+
+#[test]
+fn test_edge_f16() {
+ test_edge::<f16>();
+}
+
+#[test]
+fn test_edge_bf16() {
+ test_edge::<bf16>();
+}
+
+// out-of-bounds tests
+#[test]
+fn test_oob_f32() {
+ test_oob::<f32>();
+}
+
+#[test]
+fn test_oob_f16() {
+ test_oob::<f16>();
+}
+
+#[test]
+fn test_oob_bf16() {
+ test_oob::<bf16>();
+}
diff --git a/crates/uzu/tests/unit/kernel/full_precision_embedding_test.rs b/crates/uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rs
similarity index 100%
rename from crates/uzu/tests/unit/kernel/full_precision_embedding_test.rs
rename to crates/uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rs
diff --git a/crates/uzu/tests/unit/kernel/embedding/mod.rs b/crates/uzu/tests/unit/kernel/embedding/mod.rs
new file mode 100644
index 000000000..1f2dc0e7d
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/embedding/mod.rs
@@ -0,0 +1,3 @@
+mod embedding_rows_sum_test;
+mod full_precision_embedding_test;
+mod quant_embedding_test;
diff --git a/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs b/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
new file mode 100644
index 000000000..40c667400
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
@@ -0,0 +1,246 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{
+ Backend, Context, Encoder, Kernels, gpu_types::QuantizationMode, kernel::QuantizedEmbeddingLookupKernel,
+ },
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ token_ids: Box<[u64]>,
+ weights: Box<[u8]>,
+ scales: Box<[T]>,
+ biases: Box<[T]>,
+ batch_size: u32,
+ vocab_size: u32,
+ model_dim: u32,
+ input_scale: f32,
+ group_size: u32,
+ quant_mode: QuantizationMode,
+}
+
+fn get_test_data<T: ArrayElement + Float>(quant_mode: QuantizationMode) -> (Input<T>, Vec<T>) {
+ let batch_size = 3u32;
+ let vocab_size = 8u32;
+ let model_dim = 64u32;
+ let group_size = 32u32;
+ let input_scale = 1.5f32;
+
+ let token_ids: Box<[u64]> = Box::new([2, 5, 0]);
+
+ let packing_divisor = quant_mode.packing_divisor() as u32;
+ let weights_stride = model_dim / packing_divisor;
+ let weights: Vec<u8> = (0..vocab_size as usize * weights_stride as usize)
+ .map(|i| ((i % 16) as u8) | ((((i + 3) % 16) as u8) << 4))
+ .collect();
+
+ let num_groups = (model_dim + group_size - 1) / group_size;
+ let scales: Vec<T> = (0..vocab_size as usize * num_groups as usize)
+ .map(|i| T::from(0.5 + (i as f32 * 0.1).sin() * 0.3).unwrap())
+ .collect();
+ let biases: Vec<T> = (0..vocab_size as usize * num_groups as usize)
+ .map(|i| T::from((i as f32 * 0.2).cos() * 0.1).unwrap())
+ .collect();
+
+ let input = Input {
+ token_ids,
+ weights: weights.into_boxed_slice(),
+ scales: scales.into_boxed_slice(),
+ biases: biases.into_boxed_slice(),
+ batch_size,
+ vocab_size,
+ model_dim,
+ input_scale,
+ group_size,
+ quant_mode,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_test_data_oob<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let batch_size = 2u32;
+ let vocab_size = 4u32;
+ let model_dim = 32u32;
+ let group_size = 32u32;
+ let input_scale = 1.0f32;
+
+ // token_id 1 is valid, token_id 10 is out of bounds
+ let token_ids: Box<[u64]> = Box::new([1, 10]);
+
+ let weights: Vec<u8> = (0..vocab_size as usize * model_dim as usize).map(|i| (i % 200) as u8).collect();
+
+ let num_groups = (model_dim + group_size - 1) / group_size;
+ let scales: Vec<T> =
+ (0..vocab_size as usize * num_groups as usize).map(|i| T::from(0.5 + i as f32 * 0.1).unwrap()).collect();
+ let biases: Vec<T> =
+ (0..vocab_size as usize * num_groups as usize).map(|i| T::from(i as f32 * 0.01).unwrap()).collect();
+
+ let input = Input {
+ token_ids,
+ weights: weights.into_boxed_slice(),
+ scales: scales.into_boxed_slice(),
+ biases: biases.into_boxed_slice(),
+ batch_size,
+ vocab_size,
+ model_dim,
+ input_scale,
+ group_size,
+ quant_mode: QuantizationMode::UINT8,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
+ &context,
+ T::data_type(),
+ input.group_size,
+ input.quant_mode.to_u32(),
+ )
+ .expect("Failed to create QuantizedEmbeddingLookupKernel");
+
+ let packing_divisor = input.quant_mode.packing_divisor();
+ let weights_stride = input.model_dim as usize / packing_divisor;
+ let num_groups = (input.model_dim as usize + input.group_size as usize - 1) / input.group_size as usize;
+
+ let token_ids_array = context.create_array_from(&[input.batch_size as usize], &input.token_ids, "");
+ let weights_array = context.create_array_from(&[input.vocab_size as usize, weights_stride], &input.weights, "");
+ let scales_array = context.create_array_from(&[input.vocab_size as usize, num_groups], &input.scales, "");
+ let biases_array = context.create_array_from(&[input.vocab_size as usize, num_groups], &input.biases, "");
+ let output_array =
+ context.create_array_uninitialized(&[input.batch_size as usize, input.model_dim as usize], T::data_type(), "");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ token_ids_array.buffer().borrow().deref(),
+ weights_array.buffer().borrow().deref(),
+ scales_array.buffer().borrow().deref(),
+ biases_array.buffer().borrow().deref(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.batch_size,
+ input.vocab_size,
+ input.model_dim,
+ input.input_scale,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+fn test_quant_mode<T: ArrayElement + Float + Debug + Display>(quant_mode: QuantizationMode) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.5f32
+ } else {
+ 1e-4
+ };
+ let (input, expected) = get_test_data::<T>(quant_mode);
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!("QuantizedEmbeddingLookup {quant_mode:?} test failed for backend {}", std::any::type_name::<B>()),
+ );
+ });
+}
+
+fn test_oob<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.5f32
+ } else {
+ 1e-4
+ };
+ let (input, expected) = get_test_data_oob::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!("QuantizedEmbeddingLookup OOB test failed for backend {}", std::any::type_name::<B>()),
+ );
+ });
+}
+
+// UINT4 tests
+#[test]
+fn test_uint4_f32() {
+ test_quant_mode::<f32>(QuantizationMode::UINT4);
+}
+
+#[test]
+fn test_uint4_f16() {
+ test_quant_mode::<f16>(QuantizationMode::UINT4);
+}
+
+#[test]
+fn test_uint4_bf16() {
+ test_quant_mode::<bf16>(QuantizationMode::UINT4);
+}
+
+// INT8 tests
+#[test]
+fn test_int8_f32() {
+ test_quant_mode::<f32>(QuantizationMode::INT8);
+}
+
+#[test]
+fn test_int8_f16() {
+ test_quant_mode::<f16>(QuantizationMode::INT8);
+}
+
+#[test]
+fn test_int8_bf16() {
+ test_quant_mode::<bf16>(QuantizationMode::INT8);
+}
+
+// UINT8 tests
+#[test]
+fn test_uint8_f32() {
+ test_quant_mode::<f32>(QuantizationMode::UINT8);
+}
+
+#[test]
+fn test_uint8_f16() {
+ test_quant_mode::<f16>(QuantizationMode::UINT8);
+}
+
+#[test]
+fn test_uint8_bf16() {
+ test_quant_mode::<bf16>(QuantizationMode::UINT8);
+}
+
+// OOB tests
+#[test]
+fn test_oob_f32() {
+ test_oob::<f32>();
+}
+
+#[test]
+fn test_oob_f16() {
+ test_oob::<f16>();
+}
+
+#[test]
+fn test_oob_bf16() {
+ test_oob::<bf16>();
+}
diff --git a/crates/uzu/tests/unit/kernel/kv_cache_update_test.rs b/crates/uzu/tests/unit/kernel/kv_cache_update_test.rs
new file mode 100644
index 000000000..670344b39
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/kv_cache_update_test.rs
@@ -0,0 +1,282 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::DerefMut,
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ keys: Box<[T]>,
+ values: Box<[T]>,
+ swaps: Vec<Swap>,
+ num_heads: u32,
+ max_sequence_length: u32,
+ head_dim: u32,
+}
+
+fn make_data<T: ArrayElement + Float>(
+ num_heads: usize,
+ max_sequence_length: usize,
+ head_dim: usize,
+) -> (Vec<T>, Vec<T>) {
+ let total = num_heads * max_sequence_length * head_dim;
+ let mut keys = Vec::with_capacity(total);
+ let mut values = Vec::with_capacity(total);
+ for i in 0..total {
+ keys.push(T::from(1.0 + i as f32 * 0.1).unwrap());
+ values.push(T::from(100.0 + i as f32 * 0.1).unwrap());
+ }
+ (keys, values)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {
+ let context = B::Context::new().expect("Failed to create Context");
+ let kernel = <<B as Backend>::Kernels as Kernels>::KVCacheUpdateKernel::new(&context, T::data_type())
+ .expect("Failed to create KVCacheUpdateKernel");
+
+ let total = input.num_heads as usize * input.max_sequence_length as usize * input.head_dim as usize;
+ let keys_array = context.create_array_from(&[total], &input.keys, "");
+ let values_array = context.create_array_from(&[total], &input.values, "");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");
+ kernel.encode(
+ keys_array.buffer().borrow_mut().deref_mut(),
+ values_array.buffer().borrow_mut().deref_mut(),
+ &input.swaps,
+ input.swaps.len() as u32,
+ input.num_heads,
+ input.max_sequence_length,
+ input.head_dim,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ (keys_array.as_slice().to_vec(), values_array.as_slice().to_vec())
+}
+
+/// Single swap between two different positions.
+fn get_test_data_single_swap<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {
+ let num_heads = 2usize;
+ let max_sequence_length = 4usize;
+ let head_dim = 3usize;
+
+ let (keys, values) = make_data::<T>(num_heads, max_sequence_length, head_dim);
+
+ let swaps = vec![Swap {
+ source: 0,
+ destination: 2,
+ }];
+
+ let input = Input {
+ keys: keys.into_boxed_slice(),
+ values: values.into_boxed_slice(),
+ swaps: swaps.clone(),
+ num_heads: num_heads as u32,
+ max_sequence_length: max_sequence_length as u32,
+ head_dim: head_dim as u32,
+ };
+
+ let (expected_keys, expected_values) = get_output::<T, Cpu>(&input);
+ (input, expected_keys, expected_values)
+}
+
+/// Multiple swaps.
+fn get_test_data_multi_swap<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {
+ let num_heads = 2usize;
+ let max_sequence_length = 8usize;
+ let head_dim = 4usize;
+
+ let (keys, values) = make_data::<T>(num_heads, max_sequence_length, head_dim);
+
+ let swaps = vec![
+ Swap {
+ source: 1,
+ destination: 3,
+ },
+ Swap {
+ source: 0,
+ destination: 5,
+ },
+ ];
+
+ let input = Input {
+ keys: keys.into_boxed_slice(),
+ values: values.into_boxed_slice(),
+ swaps: swaps.clone(),
+ num_heads: num_heads as u32,
+ max_sequence_length: max_sequence_length as u32,
+ head_dim: head_dim as u32,
+ };
+
+ let (expected_keys, expected_values) = get_output::<T, Cpu>(&input);
+ (input, expected_keys, expected_values)
+}
+
+/// No swaps — data should remain unchanged.
+fn get_test_data_no_swap<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {
+ let num_heads = 2usize;
+ let max_sequence_length = 4usize;
+ let head_dim = 3usize;
+
+ let (keys, values) = make_data::<T>(num_heads, max_sequence_length, head_dim);
+
+ let expected_keys = keys.clone();
+ let expected_values = values.clone();
+
+ let input = Input {
+ keys: keys.into_boxed_slice(),
+ values: values.into_boxed_slice(),
+ swaps: vec![],
+ num_heads: num_heads as u32,
+ max_sequence_length: max_sequence_length as u32,
+ head_dim: head_dim as u32,
+ };
+
+ (input, expected_keys, expected_values)
+}
+
+/// Larger dimensions to exercise more threads.
+fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {
+ let num_heads = 8usize;
+ let max_sequence_length = 64usize;
+ let head_dim = 128usize;
+
+ let (keys, values) = make_data::<T>(num_heads, max_sequence_length, head_dim);
+
+ let swaps = vec![
+ Swap {
+ source: 0,
+ destination: 63,
+ },
+ Swap {
+ source: 10,
+ destination: 30,
+ },
+ Swap {
+ source: 5,
+ destination: 20,
+ },
+ ];
+
+ let input = Input {
+ keys: keys.into_boxed_slice(),
+ values: values.into_boxed_slice(),
+ swaps: swaps.clone(),
+ num_heads: num_heads as u32,
+ max_sequence_length: max_sequence_length as u32,
+ head_dim: head_dim as u32,
+ };
+
+ let (expected_keys, expected_values) = get_output::<T, Cpu>(&input);
+ (input, expected_keys, expected_values)
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &Input<T>,
+ expected_keys: &[T],
+ expected_values: &[T],
+ test_name: &str,
+) {
+ let eps = 1e-5;
+ for_each_non_cpu_backend!(|B| {
+ let (actual_keys, actual_values) = get_output::<T, B>(input);
+ let msg = format!("KVCacheUpdate {} keys failed with backend={}", test_name, std::any::type_name::<B>(),);
+ assert_eq_float::<T>(expected_keys, &actual_keys, eps, &msg);
+ let msg = format!("KVCacheUpdate {} values failed with backend={}", test_name, std::any::type_name::<B>(),);
+ assert_eq_float::<T>(expected_values, &actual_values, eps, &msg);
+ });
+}
+
+fn test_single_swap_internal<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected_keys, expected_values) = get_test_data_single_swap::<T>();
+ test_internal::<T>(&input, &expected_keys, &expected_values, "single_swap");
+}
+
+fn test_multi_swap_internal<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected_keys, expected_values) = get_test_data_multi_swap::<T>();
+ test_internal::<T>(&input, &expected_keys, &expected_values, "multi_swap");
+}
+
+fn test_no_swap_internal<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected_keys, expected_values) = get_test_data_no_swap::<T>();
+ test_internal::<T>(&input, &expected_keys, &expected_values, "no_swap");
+}
+
+fn test_large_internal<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected_keys, expected_values) = get_test_data_large::<T>();
+ test_internal::<T>(&input, &expected_keys, &expected_values, "large");
+}
+
+// Single swap tests
+#[test]
+fn test_single_swap_f32() {
+ test_single_swap_internal::<f32>();
+}
+
+#[test]
+fn test_single_swap_f16() {
+ test_single_swap_internal::<f16>();
+}
+
+#[test]
+fn test_single_swap_bf16() {
+ test_single_swap_internal::<bf16>();
+}
+
+// Multi swap tests
+#[test]
+fn test_multi_swap_f32() {
+ test_multi_swap_internal::<f32>();
+}
+
+#[test]
+fn test_multi_swap_f16() {
+ test_multi_swap_internal::<f16>();
+}
+
+#[test]
+fn test_multi_swap_bf16() {
+ test_multi_swap_internal::<bf16>();
+}
+
+// No swap tests
+#[test]
+fn test_no_swap_f32() {
+ test_no_swap_internal::<f32>();
+}
+
+#[test]
+fn test_no_swap_f16() {
+ test_no_swap_internal::<f16>();
+}
+
+#[test]
+fn test_no_swap_bf16() {
+ test_no_swap_internal::<bf16>();
+}
+
+// Large dimension tests
+#[test]
+fn test_large_f32() {
+ test_large_internal::<f32>();
+}
+
+#[test]
+fn test_large_f16() {
+ test_large_internal::<f16>();
+}
+
+#[test]
+fn test_large_bf16() {
+ test_large_internal::<bf16>();
+}
diff --git a/crates/uzu/tests/unit/kernel/layer_norm_test.rs b/crates/uzu/tests/unit/kernel/layer_norm_test.rs
new file mode 100644
index 000000000..f0af5edfb
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/layer_norm_test.rs
@@ -0,0 +1,314 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::LayerNormKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+static BOOL_ALL: &[bool] = &[true, false];
+static BOOL_FALSE: &[bool] = &[false];
+
+struct Input<IN: ArrayElement + Float, SC: ArrayElement + Float, OUT: ArrayElement + Float> {
+ input: Box<[IN]>,
+ scales: Box<[SC]>,
+ output: Box<[OUT]>,
+ batch_size: u32,
+ model_dim: u32,
+ epsilon: f32,
+ scale_offset: f32,
+ full_layer: u32,
+ in_place: bool,
+}
+
+fn get_test_data_basic<
+ IN: ArrayElement + Float,
+ SC: ArrayElement + Float,
+ OUT: ArrayElement + Float,
+ ACC: ArrayElement + Float,
+>(
+ full_layer: u32,
+ in_place: bool,
+) -> (Input<IN, SC, OUT>, Vec<OUT>) {
+ let batch_size = 2usize;
+ let model_dim = 4096usize;
+ let epsilon = 1e-5f32;
+ let scale_offset = 0.123f32;
+
+ let total = batch_size * model_dim;
+ let mut input_data: Vec<IN> = vec![IN::zero(); total];
+ let mut output_data: Vec<OUT> = vec![OUT::zero(); total];
+ let scales_data: Vec<SC> = (0..model_dim).map(|i| SC::from(1.0 + 0.01 * (i as f32).sin()).unwrap()).collect();
+
+ for i in 0..total {
+ let val = 2.0 * f32::sin(i as f32 * 0.01) + 0.1 * (i as f32 / total as f32);
+ input_data[i] = IN::from(val).unwrap();
+ output_data[i] = OUT::from(val).unwrap();
+ }
+
+ let input = Input::<IN, SC, OUT> {
+ input: input_data.into_boxed_slice(),
+ scales: scales_data.into_boxed_slice(),
+ output: output_data.into_boxed_slice(),
+ batch_size: batch_size as u32,
+ model_dim: model_dim as u32,
+ epsilon,
+ scale_offset,
+ full_layer,
+ in_place,
+ };
+
+ let output = get_output::<Cpu, IN, SC, OUT, ACC>(&input);
+ (input, output)
+}
+
+fn get_test_data_edge<
+ IN: ArrayElement + Float,
+ SC: ArrayElement + Float,
+ OUT: ArrayElement + Float,
+ ACC: ArrayElement + Float,
+>(
+ full_layer: u32,
+ in_place: bool,
+) -> (Input<IN, SC, OUT>, Vec<OUT>) {
+ let batch_size = 1usize;
+ let model_dim = 4usize;
+ let epsilon = 1e-6f32;
+ let scale_offset = 0.0f32;
+
+ let input_data_f32: Vec<f32> = vec![1.0, -2.0, 3.0, -4.0];
+ let scales_data_f32: Vec<f32> = vec![2.0, 0.5, 1.5, 0.8];
+ let input_data: Vec<IN> = input_data_f32.iter().map(|&x| IN::from(x).unwrap()).collect();
+ let output_data: Vec<OUT> = input_data_f32.iter().map(|&x| OUT::from(x).unwrap()).collect();
+ let scales_data: Vec<SC> = scales_data_f32.iter().map(|&x| SC::from(x).unwrap()).collect();
+
+ let input = Input::<IN, SC, OUT> {
+ input: input_data.into_boxed_slice(),
+ scales: scales_data.into_boxed_slice(),
+ output: output_data.into_boxed_slice(),
+ batch_size: batch_size as u32,
+ model_dim: model_dim as u32,
+ epsilon,
+ scale_offset,
+ full_layer,
+ in_place,
+ };
+
+ let output = get_output::<Cpu, IN, SC, OUT, ACC>(&input);
+ (input, output)
+}
+
+fn get_output<
+ B: Backend,
+ IN: ArrayElement + Float,
+ SC: ArrayElement + Float,
+ OUT: ArrayElement + Float,
+ ACC: ArrayElement + Float,
+>(
+ input: &Input<IN, SC, OUT>
+) -> Vec<OUT> {
+ let context = B::Context::new().expect("Failed to create Context");
+ let kernel = <<B as Backend>::Kernels as Kernels>::LayerNormKernel::new(
+ &context,
+ IN::data_type(),
+ SC::data_type(),
+ OUT::data_type(),
+ ACC::data_type(),
+ input.in_place,
+ )
+ .expect("Failed to create LayerNormKernel");
+
+ let input_size = input.input.len();
+ let input_array = context.create_array_from(&[input_size], &input.input, "");
+ let input_array_buffer_rc = input_array.buffer();
+ let input_array_borrow = input_array_buffer_rc.borrow();
+ let input_array_deref = input_array_borrow.deref();
+ let input_buffer = (!input.in_place).then(|| input_array_deref);
+
+ let scales_array = context.create_array_from(&[input.scales.len()], &input.scales, "");
+ let output_array = match input.in_place {
+ true => context.create_array_from(&[input_size], &input.output, ""),
+ false => context.create_array_uninitialized(&[input_size], OUT::data_type(), ""),
+ };
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ input_buffer,
+ scales_array.buffer().borrow().deref(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.batch_size,
+ input.model_dim,
+ input.epsilon,
+ input.scale_offset,
+ input.full_layer,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+fn test_internal<
+ IN: ArrayElement + Float,
+ SC: ArrayElement + Float,
+ OUT: ArrayElement + Float + Debug + Display,
+ ACC: ArrayElement + Float,
+>(
+ input: &Input<IN, SC, OUT>,
+ expected: &[OUT],
+) {
+ let eps = if matches!(IN::data_type(), DataType::F16 | DataType::BF16)
+ || matches!(SC::data_type(), DataType::F16 | DataType::BF16)
+ || matches!(OUT::data_type(), DataType::F16 | DataType::BF16)
+ {
+ 0.012f32
+ } else {
+ 1e-5
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<B, IN, SC, OUT, ACC>(input);
+ let msg = format!(
+ "LayerNorm kernel test failed with backend={}, full_layer={}, in_place={}",
+ std::any::type_name::<B>(),
+ input.full_layer,
+ input.in_place,
+ );
+ assert_eq_float::<OUT>(expected, &output, eps, &msg);
+ });
+}
+
+fn test_basic<
+ IN: ArrayElement + Float,
+ SC: ArrayElement + Float,
+ OUT: ArrayElement + Float + Debug + Display,
+ ACC: ArrayElement + Float,
+>() {
+ let in_place_values: &[bool] = if IN::data_type() == OUT::data_type() {
+ &BOOL_ALL
+ } else {
+ &BOOL_FALSE
+ };
+
+ for in_place in in_place_values {
+ for full_layer in [1u32, 0u32] {
+ let (input, expected) = get_test_data_basic::<IN, SC, OUT, ACC>(full_layer, *in_place);
+ test_internal::<IN, SC, OUT, ACC>(&input, expected.as_slice())
+ }
+ }
+}
+
+fn test_edge<
+ IN: ArrayElement + Float,
+ SC: ArrayElement + Float,
+ OUT: ArrayElement + Float + Debug + Display,
+ ACC: ArrayElement + Float,
+>() {
+ let in_place_values: &[bool] = if IN::data_type() == OUT::data_type() {
+ &BOOL_ALL
+ } else {
+ &BOOL_FALSE
+ };
+
+ for in_place in in_place_values {
+ let (input, expected) = get_test_data_edge::<IN, SC, OUT, ACC>(0, *in_place);
+ test_internal::<IN, SC, OUT, ACC>(&input, expected.as_slice())
+ }
+}
+
+// basic tests - AccumT f32
+#[test]
+fn test_f32_f32_f32_f32() {
+ test_basic::<f32, f32, f32, f32>();
+}
+
+#[test]
+fn test_f16_f32_f32_f32() {
+ test_basic::<f16, f32, f32, f32>();
+}
+
+#[test]
+fn test_f32_f16_f32_f32() {
+ test_basic::<f32, f16, f32, f32>();
+}
+
+#[test]
+fn test_f32_f32_f16_f32() {
+ test_basic::<f32, f32, f16, f32>();
+}
+
+#[test]
+fn test_f16_f16_f32_f32() {
+ test_basic::<f16, f16, f32, f32>();
+}
+
+#[test]
+fn test_f16_f32_f16_f32() {
+ test_basic::<f16, f32, f16, f32>();
+}
+
+#[test]
+fn test_f32_f16_f16_f32() {
+ test_basic::<f32, f16, f16, f32>();
+}
+
+#[test]
+fn test_f16_f16_f16_f32() {
+ test_basic::<f16, f16, f16, f32>();
+}
+
+// basic tests - AccumT f32 + bf16
+#[test]
+fn test_bf16_f32_f32_f32() {
+ test_basic::<bf16, f32, f32, f32>();
+}
+
+#[test]
+fn test_f32_bf16_f32_f32() {
+ test_basic::<f32, bf16, f32, f32>();
+}
+
+#[test]
+fn test_f32_f32_bf16_f32() {
+ test_basic::<f32, f32, bf16, f32>();
+}
+
+#[test]
+fn test_bf16_bf16_f32_f32() {
+ test_basic::<bf16, bf16, f32, f32>();
+}
+
+#[test]
+fn test_bf16_f32_bf16_f32() {
+ test_basic::<bf16, f32, bf16, f32>();
+}
+
+#[test]
+fn test_bf16_bf16_bf16_f32() {
+ test_basic::<bf16, bf16, bf16, f32>();
+}
+
+// edge tests
+#[test]
+fn test_edge_f32_f32_f32_f32() {
+ test_edge::<f32, f32, f32, f32>();
+}
+
+#[test]
+fn test_edge_f32_f16_f16_f32() {
+ test_edge::<f32, f16, f16, f32>();
+}
+
+#[test]
+fn test_edge_f16_f16_f16_f32() {
+ test_edge::<f16, f16, f16, f32>();
+}
diff --git a/crates/uzu/tests/unit/kernel/mask_update_test.rs b/crates/uzu/tests/unit/kernel/mask_update_test.rs
new file mode 100644
index 000000000..f4113d5ab
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/mask_update_test.rs
@@ -0,0 +1,240 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::DerefMut,
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::MaskUpdateKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ mask: Box<[T]>,
+ unmask_col: i32,
+ mask_col: i32,
+}
+
+fn make_mask<T: ArrayElement + Float>(size: usize) -> Vec<T> {
+ let mut mask = Vec::with_capacity(size);
+ for i in 0..size {
+ mask.push(T::from(i as f32 * 0.5).unwrap());
+ }
+ mask
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+ let kernel = <<B as Backend>::Kernels as Kernels>::MaskUpdateKernel::new(&context, T::data_type())
+ .expect("Failed to create MaskUpdateKernel");
+
+ let len = input.mask.len();
+ let mask_array = context.create_array_from(&[len], &input.mask, "");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");
+ kernel.encode(mask_array.buffer().borrow_mut().deref_mut(), input.unmask_col, input.mask_col, &mut encoder);
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ mask_array.as_slice().to_vec()
+}
+
+/// Unmask one column and mask another.
+fn get_test_data_unmask_and_mask<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let size = 8;
+ let mask = make_mask::<T>(size);
+
+ let input = Input {
+ mask: mask.into_boxed_slice(),
+ unmask_col: 2,
+ mask_col: 5,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+/// Only unmask (mask_col negative).
+fn get_test_data_unmask_only<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let size = 8;
+ let mask = make_mask::<T>(size);
+
+ let input = Input {
+ mask: mask.into_boxed_slice(),
+ unmask_col: 3,
+ mask_col: -1,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+/// Only mask (unmask_col negative).
+fn get_test_data_mask_only<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let size = 8;
+ let mask = make_mask::<T>(size);
+
+ let input = Input {
+ mask: mask.into_boxed_slice(),
+ unmask_col: -1,
+ mask_col: 4,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+/// Both columns negative — data should remain unchanged.
+fn get_test_data_no_op<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let size = 8;
+ let mask = make_mask::<T>(size);
+
+ let expected = mask.clone();
+
+ let input = Input {
+ mask: mask.into_boxed_slice(),
+ unmask_col: -1,
+ mask_col: -1,
+ };
+
+ (input, expected)
+}
+
+/// Larger mask to exercise more threads.
+fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let size = 1024;
+ let mask = make_mask::<T>(size);
+
+ let input = Input {
+ mask: mask.into_boxed_slice(),
+ unmask_col: 100,
+ mask_col: 900,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &Input<T>,
+ expected: &[T],
+ test_name: &str,
+) {
+ let eps = 1e-5;
+ let expected_norm = expected.to_vec();
+ for_each_non_cpu_backend!(|B| {
+ let actual = get_output::<T, B>(input);
+ let msg = format!("MaskUpdate {} failed with backend={}", test_name, std::any::type_name::<B>());
+ assert_eq_float::<T>(&expected_norm, &actual, eps, &msg);
+ });
+}
+
+fn test_unmask_and_mask_internal<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data_unmask_and_mask::<T>();
+ test_internal::<T>(&input, &expected, "unmask_and_mask");
+}
+
+fn test_unmask_only_internal<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data_unmask_only::<T>();
+ test_internal::<T>(&input, &expected, "unmask_only");
+}
+
+fn test_mask_only_internal<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data_mask_only::<T>();
+ test_internal::<T>(&input, &expected, "mask_only");
+}
+
+fn test_no_op_internal<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data_no_op::<T>();
+ test_internal::<T>(&input, &expected, "no_op");
+}
+
+fn test_large_internal<T: ArrayElement + Float + Debug + Display>() {
+ let (input, expected) = get_test_data_large::<T>();
+ test_internal::<T>(&input, &expected, "large");
+}
+
+// Unmask and mask tests
+#[test]
+fn test_unmask_and_mask_f32() {
+ test_unmask_and_mask_internal::<f32>();
+}
+
+#[test]
+fn test_unmask_and_mask_f16() {
+ test_unmask_and_mask_internal::<f16>();
+}
+
+#[test]
+fn test_unmask_and_mask_bf16() {
+ test_unmask_and_mask_internal::<bf16>();
+}
+
+// Unmask only tests
+#[test]
+fn test_unmask_only_f32() {
+ test_unmask_only_internal::<f32>();
+}
+
+#[test]
+fn test_unmask_only_f16() {
+ test_unmask_only_internal::<f16>();
+}
+
+#[test]
+fn test_unmask_only_bf16() {
+ test_unmask_only_internal::<bf16>();
+}
+
+// Mask only tests
+#[test]
+fn test_mask_only_f32() {
+ test_mask_only_internal::<f32>();
+}
+
+#[test]
+fn test_mask_only_f16() {
+ test_mask_only_internal::<f16>();
+}
+
+#[test]
+fn test_mask_only_bf16() {
+ test_mask_only_internal::<bf16>();
+}
+
+// No-op tests
+#[test]
+fn test_no_op_f32() {
+ test_no_op_internal::<f32>();
+}
+
+#[test]
+fn test_no_op_f16() {
+ test_no_op_internal::<f16>();
+}
+
+#[test]
+fn test_no_op_bf16() {
+ test_no_op_internal::<bf16>();
+}
+
+// Large dimension tests
+#[test]
+fn test_large_f32() {
+ test_large_internal::<f32>();
+}
+
+#[test]
+fn test_large_f16() {
+ test_large_internal::<f16>();
+}
+
+#[test]
+fn test_large_bf16() {
+ test_large_internal::<bf16>();
+}
diff --git a/crates/uzu/tests/unit/kernel/mod.rs b/crates/uzu/tests/unit/kernel/mod.rs
index ad863e541..34e21cfb0 100644
--- a/crates/uzu/tests/unit/kernel/mod.rs
+++ b/crates/uzu/tests/unit/kernel/mod.rs
@@ -3,16 +3,23 @@
mod common;
mod activation_mul_test;
+mod activation_test;
mod attention;
mod audio;
-mod full_precision_embedding_test;
+mod embedding;
+mod kv_cache_update_test;
+mod layer_norm_test;
+mod mask_update_test;
mod matmul;
mod moe;
+mod pooling;
mod qk_norm_test;
mod rms_norm_test;
mod rope_test;
mod sampling;
+mod sigmoid_test;
mod tensor_add_bias_test;
+mod tensor_add_scale_test;
mod tensor_add_swap_test;
mod tensor_copy_test;
mod token_copy_test;
diff --git a/crates/uzu/tests/unit/kernel/pooling/mod.rs b/crates/uzu/tests/unit/kernel/pooling/mod.rs
new file mode 100644
index 000000000..22da4b449
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/pooling/mod.rs
@@ -0,0 +1,2 @@
+mod pooling_cls_test;
+mod pooling_mean_test;
diff --git a/crates/uzu/tests/unit/kernel/pooling/pooling_cls_test.rs b/crates/uzu/tests/unit/kernel/pooling/pooling_cls_test.rs
new file mode 100644
index 000000000..248d9c87f
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/pooling/pooling_cls_test.rs
@@ -0,0 +1,157 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::PoolingClsKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ input: Box<[T]>,
+ seq_len: u32,
+ hidden_dim: u32,
+ batch_size: u32,
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ batch_size: u32,
+ seq_len: u32,
+ hidden_dim: u32,
+) -> (Input<T>, Vec<T>) {
+ let len = (batch_size * seq_len * hidden_dim) as usize;
+ let input: Vec<T> = (0..len).map(|i| T::from((i as f32 * 0.1).sin() * 2.0).unwrap()).collect();
+
+ let input = Input {
+ input: input.into_boxed_slice(),
+ seq_len,
+ hidden_dim,
+ batch_size,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::PoolingClsKernel::new(&context, T::data_type())
+ .expect("Failed to create PoolingClsKernel");
+
+ let input_len = (input.batch_size * input.seq_len * input.hidden_dim) as usize;
+ let output_len = (input.batch_size * input.hidden_dim) as usize;
+
+ let input_array = context.create_array_from(&[input_len], &input.input, "");
+ let output_array = context.create_array_uninitialized(&[output_len], T::data_type(), "");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ input_array.buffer().borrow().deref(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.seq_len,
+ input.hidden_dim,
+ input.batch_size,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ batch_size: u32,
+ seq_len: u32,
+ hidden_dim: u32,
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-3
+ } else {
+ 1e-5
+ };
+
+ let (input, expected) = get_test_data::<T>(batch_size, seq_len, hidden_dim);
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ let msg = format!("PoolingCls test failed for backend {}", std::any::type_name::<B>());
+ assert_eq_float::<T>(&expected, &output, eps, &msg);
+ });
+}
+
+// Basic tests
+#[test]
+fn test_basic_f32() {
+ test_internal::<f32>(2, 16, 64);
+}
+
+#[test]
+fn test_basic_f16() {
+ test_internal::<f16>(2, 16, 64);
+}
+
+#[test]
+fn test_basic_bf16() {
+ test_internal::<bf16>(2, 16, 64);
+}
+
+// Single batch
+#[test]
+fn test_single_batch_f32() {
+ test_internal::<f32>(1, 32, 128);
+}
+
+#[test]
+fn test_single_batch_f16() {
+ test_internal::<f16>(1, 32, 128);
+}
+
+#[test]
+fn test_single_batch_bf16() {
+ test_internal::<bf16>(1, 32, 128);
+}
+
+// Single token sequence
+#[test]
+fn test_single_token_f32() {
+ test_internal::<f32>(2, 1, 64);
+}
+
+#[test]
+fn test_single_token_f16() {
+ test_internal::<f16>(2, 1, 64);
+}
+
+#[test]
+fn test_single_token_bf16() {
+ test_internal::<bf16>(2, 1, 64);
+}
+
+// Large hidden dim
+#[test]
+fn test_large_hidden_dim_f32() {
+ test_internal::<f32>(2, 8, 512);
+}
+
+// Non-aligned hidden dim
+#[test]
+fn test_non_aligned_f32() {
+ test_internal::<f32>(3, 10, 100);
+}
+
+#[test]
+fn test_non_aligned_f16() {
+ test_internal::<f16>(3, 10, 100);
+}
+
+#[test]
+fn test_non_aligned_bf16() {
+ test_internal::<bf16>(3, 10, 100);
+}
diff --git a/crates/uzu/tests/unit/kernel/pooling/pooling_mean_test.rs b/crates/uzu/tests/unit/kernel/pooling/pooling_mean_test.rs
new file mode 100644
index 000000000..526045fca
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/pooling/pooling_mean_test.rs
@@ -0,0 +1,163 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::PoolingMeanKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ input: Box<[T]>,
+ seq_len: u32,
+ hidden_dim: u32,
+ batch_size: u32,
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ batch_size: u32,
+ seq_len: u32,
+ hidden_dim: u32,
+) -> (Input<T>, Vec<T>) {
+ let len = (batch_size * seq_len * hidden_dim) as usize;
+ let input: Vec<T> = (0..len).map(|i| T::from((i as f32 * 0.1).sin() * 2.0).unwrap()).collect();
+
+ let input = Input {
+ input: input.into_boxed_slice(),
+ seq_len,
+ hidden_dim,
+ batch_size,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::PoolingMeanKernel::new(&context, T::data_type())
+ .expect("Failed to create PoolingMeanKernel");
+
+ let input_len = (input.batch_size * input.seq_len * input.hidden_dim) as usize;
+ let output_len = (input.batch_size * input.hidden_dim) as usize;
+
+ let input_array = context.create_array_from(&[input_len], &input.input, "");
+ let output_array = context.create_array_uninitialized(&[output_len], T::data_type(), "");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ input_array.buffer().borrow().deref(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.seq_len,
+ input.hidden_dim,
+ input.batch_size,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ batch_size: u32,
+ seq_len: u32,
+ hidden_dim: u32,
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-2
+ } else {
+ 1e-5
+ };
+
+ let (input, expected) = get_test_data::<T>(batch_size, seq_len, hidden_dim);
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ let msg = format!("PoolingMean test failed for backend {}", std::any::type_name::<B>());
+ assert_eq_float::<T>(&expected, &output, eps, &msg);
+ });
+}
+
+// Basic tests
+#[test]
+fn test_basic_f32() {
+ test_internal::<f32>(2, 16, 64);
+}
+
+#[test]
+fn test_basic_f16() {
+ test_internal::<f16>(2, 16, 64);
+}
+
+#[test]
+fn test_basic_bf16() {
+ test_internal::<bf16>(2, 16, 64);
+}
+
+// Single batch
+#[test]
+fn test_single_batch_f32() {
+ test_internal::<f32>(1, 32, 128);
+}
+
+#[test]
+fn test_single_batch_f16() {
+ test_internal::<f16>(1, 32, 128);
+}
+
+#[test]
+fn test_single_batch_bf16() {
+ test_internal::<bf16>(1, 32, 128);
+}
+
+// Single token sequence — mean of one element equals that element
+#[test]
+fn test_single_token_f32() {
+ test_internal::<f32>(2, 1, 64);
+}
+
+#[test]
+fn test_single_token_f16() {
+ test_internal::<f16>(2, 1, 64);
+}
+
+#[test]
+fn test_single_token_bf16() {
+ test_internal::<bf16>(2, 1, 64);
+}
+
+// Large hidden dim
+#[test]
+fn test_large_hidden_dim_f32() {
+ test_internal::<f32>(2, 8, 512);
+}
+
+// Long sequence — more accumulation, tests precision
+#[test]
+fn test_long_seq_f32() {
+ test_internal::<f32>(1, 256, 64);
+}
+
+// Non-aligned hidden dim
+#[test]
+fn test_non_aligned_f32() {
+ test_internal::<f32>(3, 10, 100);
+}
+
+#[test]
+fn test_non_aligned_f16() {
+ test_internal::<f16>(3, 10, 100);
+}
+
+#[test]
+fn test_non_aligned_bf16() {
+ test_internal::<bf16>(3, 10, 100);
+}
diff --git a/crates/uzu/tests/unit/kernel/sampling/bitmask_test.rs b/crates/uzu/tests/unit/kernel/sampling/bitmask_test.rs
new file mode 100644
index 000000000..71de2c260
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/sampling/bitmask_test.rs
@@ -0,0 +1,299 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::BitmaskKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ logits: Box<[T]>,
+ bitmask: Box<[u32]>,
+ batch_size: u32,
+ vocab_size: u32,
+ in_place: bool,
+}
+
+fn get_test_data_basic<T: ArrayElement + Float>(in_place: bool) -> (Input<T>, Vec<T>) {
+ let batch_size = 2u32;
+ let vocab_size = 128u32;
+
+ let len = (batch_size * vocab_size) as usize;
+ let logits: Vec<T> = (0..len).map(|i| T::from((i as f32 * 0.1).sin() * 2.0).unwrap()).collect();
+
+ let bitmask_size = ((vocab_size + 31) / 32) as usize;
+ let mut bitmask = vec![0u32; batch_size as usize * bitmask_size];
+ // Allow every other token
+ for b in 0..batch_size as usize {
+ for word in 0..bitmask_size {
+ bitmask[b * bitmask_size + word] = 0x55555555; // bits 0,2,4,...
+ }
+ }
+
+ let input = Input {
+ logits: logits.into_boxed_slice(),
+ bitmask: bitmask.into_boxed_slice(),
+ batch_size,
+ vocab_size,
+ in_place,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_test_data_all_allowed<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let batch_size = 1u32;
+ let vocab_size = 64u32;
+
+ let len = (batch_size * vocab_size) as usize;
+ let logits: Vec<T> = (0..len).map(|i| T::from(i as f32 * 0.5).unwrap()).collect();
+
+ let bitmask_size = ((vocab_size + 31) / 32) as usize;
+ let bitmask = vec![0xFFFFFFFFu32; batch_size as usize * bitmask_size];
+
+ let input = Input {
+ logits: logits.into_boxed_slice(),
+ bitmask: bitmask.into_boxed_slice(),
+ batch_size,
+ vocab_size,
+ in_place: false,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_test_data_none_allowed<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let batch_size = 1u32;
+ let vocab_size = 64u32;
+
+ let len = (batch_size * vocab_size) as usize;
+ let logits: Vec<T> = (0..len).map(|i| T::from(i as f32 * 0.5).unwrap()).collect();
+
+ let bitmask_size = ((vocab_size + 31) / 32) as usize;
+ let bitmask = vec![0u32; batch_size as usize * bitmask_size];
+
+ let input = Input {
+ logits: logits.into_boxed_slice(),
+ bitmask: bitmask.into_boxed_slice(),
+ batch_size,
+ vocab_size,
+ in_place: false,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_test_data_non_aligned<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ // vocab_size not a multiple of 32
+ let batch_size = 2u32;
+ let vocab_size = 100u32;
+
+ let len = (batch_size * vocab_size) as usize;
+ let logits: Vec<T> = (0..len).map(|i| T::from((i as f32 * 0.3).cos() * 3.0).unwrap()).collect();
+
+ let bitmask_size = ((vocab_size + 31) / 32) as usize;
+ let mut bitmask = vec![0u32; batch_size as usize * bitmask_size];
+ // Set specific bits
+ for b in 0..batch_size as usize {
+ for word in 0..bitmask_size {
+ bitmask[b * bitmask_size + word] = 0xAAAAAAAA; // bits 1,3,5,...
+ }
+ }
+
+ let input = Input {
+ logits: logits.into_boxed_slice(),
+ bitmask: bitmask.into_boxed_slice(),
+ batch_size,
+ vocab_size,
+ in_place: false,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::BitmaskKernel::new(&context, T::data_type(), input.in_place)
+ .expect("Failed to create BitmaskKernel");
+
+ let len = (input.batch_size * input.vocab_size) as usize;
+ let bitmask_len = input.bitmask.len();
+
+ let logits_array = context.create_array_from(&[len], &input.logits, "");
+ let bitmask_array = context.create_array_from(&[bitmask_len], &input.bitmask, "");
+ let logits_array_buffer_rc = logits_array.buffer();
+ let logits_array_borrow = logits_array_buffer_rc.borrow();
+ let logits_array_deref = logits_array_borrow.deref();
+ let logits_buffer = (!input.in_place).then(|| logits_array_deref);
+ let output_array = match input.in_place {
+ true => context.create_array_from(&[len], &input.logits, ""),
+ false => context.create_array_uninitialized(&[len], T::data_type(), ""),
+ };
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ logits_buffer,
+ bitmask_array.buffer().borrow().deref(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.batch_size,
+ input.vocab_size,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+fn test_basic<T: ArrayElement + Float + Debug + Display>(in_place: bool) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-3
+ } else {
+ 1e-5
+ };
+
+ let (input, expected) = get_test_data_basic::<T>(in_place);
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ let msg = format!("Bitmask basic test failed for backend {}", std::any::type_name::<B>());
+ assert_eq_float::<T>(&expected, &output, eps, &msg);
+ });
+}
+
+fn test_all_allowed<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-3
+ } else {
+ 1e-5
+ };
+
+ let (input, expected) = get_test_data_all_allowed::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ let msg = format!("Bitmask all-allowed test failed for backend {}", std::any::type_name::<B>());
+ assert_eq_float::<T>(&expected, &output, eps, &msg);
+ });
+}
+
+fn test_none_allowed<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-3
+ } else {
+ 1e-5
+ };
+
+ let (input, expected) = get_test_data_none_allowed::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ let msg = format!("Bitmask none-allowed test failed for backend {}", std::any::type_name::<B>());
+ assert_eq_float::<T>(&expected, &output, eps, &msg);
+ });
+}
+
+fn test_non_aligned<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-3
+ } else {
+ 1e-5
+ };
+
+ let (input, expected) = get_test_data_non_aligned::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ let msg = format!("Bitmask non-aligned test failed for backend {}", std::any::type_name::<B>());
+ assert_eq_float::<T>(&expected, &output, eps, &msg);
+ });
+}
+
+// Out-of-place basic tests
+#[test]
+fn test_basic_f32() {
+ test_basic::<f32>(false);
+}
+
+#[test]
+fn test_basic_f16() {
+ test_basic::<f16>(false);
+}
+
+#[test]
+fn test_basic_bf16() {
+ test_basic::<bf16>(false);
+}
+
+// In-place basic tests
+#[test]
+fn test_in_place_f32() {
+ test_basic::<f32>(true);
+}
+
+#[test]
+fn test_in_place_f16() {
+ test_basic::<f16>(true);
+}
+
+#[test]
+fn test_in_place_bf16() {
+ test_basic::<bf16>(true);
+}
+
+// All-allowed tests
+#[test]
+fn test_all_allowed_f32() {
+ test_all_allowed::<f32>();
+}
+
+#[test]
+fn test_all_allowed_f16() {
+ test_all_allowed::<f16>();
+}
+
+#[test]
+fn test_all_allowed_bf16() {
+ test_all_allowed::<bf16>();
+}
+
+// None-allowed tests
+#[test]
+fn test_none_allowed_f32() {
+ test_none_allowed::<f32>();
+}
+
+#[test]
+fn test_none_allowed_f16() {
+ test_none_allowed::<f16>();
+}
+
+#[test]
+fn test_none_allowed_bf16() {
+ test_none_allowed::<bf16>();
+}
+
+// Non-aligned vocab_size tests
+#[test]
+fn test_non_aligned_f32() {
+ test_non_aligned::<f32>();
+}
+
+#[test]
+fn test_non_aligned_f16() {
+ test_non_aligned::<f16>();
+}
+
+#[test]
+fn test_non_aligned_bf16() {
+ test_non_aligned::<bf16>();
+}
diff --git a/crates/uzu/tests/unit/kernel/sampling/min_p_test.rs b/crates/uzu/tests/unit/kernel/sampling/min_p_test.rs
new file mode 100644
index 000000000..1ad9a74aa
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/sampling/min_p_test.rs
@@ -0,0 +1,172 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::MinPKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ logits: Box<[T]>,
+ batch_size: u32,
+ vocab_size: u32,
+ min_p: f32,
+ in_place: bool,
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ batch_size: u32,
+ vocab_size: u32,
+ min_p: f32,
+ in_place: bool,
+) -> (Input<T>, Vec<T>) {
+ let len = (batch_size * vocab_size) as usize;
+ let logits: Vec<T> = (0..len).map(|i| T::from((i as f32 * 0.1).sin() * 2.0).unwrap()).collect();
+
+ let input = Input {
+ logits: logits.into_boxed_slice(),
+ batch_size,
+ vocab_size,
+ min_p,
+ in_place,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::MinPKernel::new(&context, T::data_type(), input.in_place)
+ .expect("Failed to create MinPKernel");
+
+ let len = (input.batch_size * input.vocab_size) as usize;
+ let logits_array = context.create_array_from(&[len], &input.logits, "");
+ let logits_array_buffer_rc = logits_array.buffer();
+ let logits_array_borrow = logits_array_buffer_rc.borrow();
+ let logits_array_deref = logits_array_borrow.deref();
+ let logits_buffer = (!input.in_place).then(|| logits_array_deref);
+ let output_array = match input.in_place {
+ true => context.create_array_from(&[len], &input.logits, ""),
+ false => context.create_array_uninitialized(&[len], T::data_type(), ""),
+ };
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ logits_buffer,
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.batch_size,
+ input.vocab_size,
+ input.min_p,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ batch_size: u32,
+ vocab_size: u32,
+ min_p: f32,
+ in_place: bool,
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-3
+ } else {
+ 1e-5
+ };
+
+ let (input, expected) = get_test_data::<T>(batch_size, vocab_size, min_p, in_place);
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ let msg = format!("MinP test failed for backend {}", std::any::type_name::<B>());
+ assert_eq_float::<T>(&expected, &output, eps, &msg);
+ });
+}
+
+// Out-of-place tests
+#[test]
+fn test_f32() {
+ test_internal::<f32>(4, 128, 0.1, false);
+}
+
+#[test]
+fn test_f16() {
+ test_internal::<f16>(4, 128, 0.1, false);
+}
+
+#[test]
+fn test_bf16() {
+ test_internal::<bf16>(4, 128, 0.1, false);
+}
+
+// In-place tests
+#[test]
+fn test_in_place_f32() {
+ test_internal::<f32>(4, 128, 0.1, true);
+}
+
+#[test]
+fn test_in_place_f16() {
+ test_internal::<f16>(4, 128, 0.1, true);
+}
+
+#[test]
+fn test_in_place_bf16() {
+ test_internal::<bf16>(4, 128, 0.1, true);
+}
+
+// High min_p — filters most tokens
+#[test]
+fn test_high_min_p_f32() {
+ test_internal::<f32>(2, 128, 0.9, false);
+}
+
+#[test]
+fn test_high_min_p_f16() {
+ test_internal::<f16>(2, 128, 0.9, false);
+}
+
+#[test]
+fn test_high_min_p_bf16() {
+ test_internal::<bf16>(2, 128, 0.9, false);
+}
+
+// Low min_p — keeps most tokens
+#[test]
+fn test_low_min_p_f32() {
+ test_internal::<f32>(2, 128, 0.01, false);
+}
+
+#[test]
+fn test_low_min_p_f16() {
+ test_internal::<f16>(2, 128, 0.01, false);
+}
+
+#[test]
+fn test_low_min_p_bf16() {
+ test_internal::<bf16>(2, 128, 0.01, false);
+}
+
+// Single batch
+#[test]
+fn test_single_batch_f32() {
+ test_internal::<f32>(1, 256, 0.1, false);
+}
+
+// Large vocab
+#[test]
+fn test_large_vocab_f32() {
+ test_internal::<f32>(2, 2048, 0.1, false);
+}
diff --git a/crates/uzu/tests/unit/kernel/sampling/mod.rs b/crates/uzu/tests/unit/kernel/sampling/mod.rs
index e87b9abd0..1331ba3df 100644
--- a/crates/uzu/tests/unit/kernel/sampling/mod.rs
+++ b/crates/uzu/tests/unit/kernel/sampling/mod.rs
@@ -1,5 +1,7 @@
mod argmax_test_old;
+mod bitmask_test;
mod gumbel_test;
+mod min_p_test;
mod temperature_test;
mod test_argmax;
mod top_k_test;
diff --git a/crates/uzu/tests/unit/kernel/sigmoid_test.rs b/crates/uzu/tests/unit/kernel/sigmoid_test.rs
new file mode 100644
index 000000000..ca4f467a9
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/sigmoid_test.rs
@@ -0,0 +1,176 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::SigmoidKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ data: Box<[T]>,
+}
+
+fn get_input<T: ArrayElement + Float>(n: usize) -> Vec<T> {
+ (0..n).map(|i| T::from((i as f32 * 0.3) - 5.0).unwrap()).collect()
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::SigmoidKernel::new(&context, T::data_type())
+ .expect("Failed to create SigmoidKernel");
+
+ let n = input.data.len();
+ let input_array = context.create_array_from(&[n], &input.data, "");
+ let output_array = context.create_array_uninitialized(&[n], T::data_type(), "");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ input_array.buffer().borrow().deref(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ n as u32,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let data = get_input::<T>(128);
+ let input = Input {
+ data: data.into_boxed_slice(),
+ };
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let data = get_input::<T>(4096);
+ let input = Input {
+ data: data.into_boxed_slice(),
+ };
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_test_data_edge<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ // Values at sigmoid boundaries: very negative, zero, very positive
+ let values: Vec<T> = [-100.0f32, -10.0, -1.0, 0.0, 1.0, 10.0, 100.0].iter().map(|&v| T::from(v).unwrap()).collect();
+ let input = Input {
+ data: values.into_boxed_slice(),
+ };
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.02f32
+ } else {
+ 1e-5
+ };
+ let (input, expected) = get_test_data_basic::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!("Sigmoid basic test failed for backend {}", std::any::type_name::<B>()),
+ );
+ });
+}
+
+fn test_large<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.02f32
+ } else {
+ 1e-5
+ };
+ let (input, expected) = get_test_data_large::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!("Sigmoid large test failed for backend {}", std::any::type_name::<B>()),
+ );
+ });
+}
+
+fn test_edge<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.02f32
+ } else {
+ 1e-5
+ };
+ let (input, expected) = get_test_data_edge::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!("Sigmoid edge test failed for backend {}", std::any::type_name::<B>()),
+ );
+ });
+}
+
+// basic tests
+#[test]
+fn test_basic_f32() {
+ test_basic::<f32>();
+}
+
+#[test]
+fn test_basic_f16() {
+ test_basic::<f16>();
+}
+
+#[test]
+fn test_basic_bf16() {
+ test_basic::<bf16>();
+}
+
+// large tests
+#[test]
+fn test_large_f32() {
+ test_large::<f32>();
+}
+
+#[test]
+fn test_large_f16() {
+ test_large::<f16>();
+}
+
+#[test]
+fn test_large_bf16() {
+ test_large::<bf16>();
+}
+
+// edge tests
+#[test]
+fn test_edge_f32() {
+ test_edge::<f32>();
+}
+
+#[test]
+fn test_edge_f16() {
+ test_edge::<f16>();
+}
+
+#[test]
+fn test_edge_bf16() {
+ test_edge::<bf16>();
+}
diff --git a/crates/uzu/tests/unit/kernel/tensor_add_scale_test.rs b/crates/uzu/tests/unit/kernel/tensor_add_scale_test.rs
new file mode 100644
index 000000000..9ebcbe6eb
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/tensor_add_scale_test.rs
@@ -0,0 +1,216 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement, DataType,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::TensorAddScaleKernel},
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ input: Box<[T]>,
+ bias: Box<[T]>,
+ num_cols: u32,
+ length: u32,
+ scale: f32,
+}
+
+fn get_input<T: ArrayElement + Float>(
+ num_rows: usize,
+ num_cols: usize,
+) -> (Vec<T>, Vec<T>) {
+ let length = num_rows * num_cols;
+ let input: Vec<T> = (0..length).map(|i| T::from((i as f32 * 0.3) - 5.0).unwrap()).collect();
+ let bias: Vec<T> = (0..num_cols).map(|i| T::from((i as f32 * 0.7) - 2.0).unwrap()).collect();
+ (input, bias)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::TensorAddScaleKernel::new(&context, T::data_type())
+ .expect("Failed to create TensorAddScaleKernel");
+
+ let length = input.length as usize;
+ let num_cols = input.num_cols as usize;
+ let input_array = context.create_array_from(&[length], &input.input, "");
+ let bias_array = context.create_array_from(&[num_cols], &input.bias, "");
+ let output_array = context.create_array_uninitialized(&[length], T::data_type(), "");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ input_array.buffer().borrow().deref(),
+ bias_array.buffer().borrow().deref(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.num_cols,
+ input.length,
+ input.scale,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let num_rows = 4usize;
+ let num_cols = 32usize;
+ let (data, bias) = get_input::<T>(num_rows, num_cols);
+
+ let input = Input {
+ input: data.into_boxed_slice(),
+ bias: bias.into_boxed_slice(),
+ num_cols: num_cols as u32,
+ length: (num_rows * num_cols) as u32,
+ scale: 0.5,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let num_rows = 64usize;
+ let num_cols = 128usize;
+ let (data, bias) = get_input::<T>(num_rows, num_cols);
+
+ let input = Input {
+ input: data.into_boxed_slice(),
+ bias: bias.into_boxed_slice(),
+ num_cols: num_cols as u32,
+ length: (num_rows * num_cols) as u32,
+ scale: 2.0,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn get_test_data_edge<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ // Single row, small dim, scale = 1.0
+ let num_rows = 1usize;
+ let num_cols = 4usize;
+ let data: Vec<T> = [1.0f32, -1.0, 0.0, 3.5].iter().map(|&v| T::from(v).unwrap()).collect();
+ let bias: Vec<T> = [0.5f32, -0.5, 1.0, 0.0].iter().map(|&v| T::from(v).unwrap()).collect();
+
+ let input = Input {
+ input: data.into_boxed_slice(),
+ bias: bias.into_boxed_slice(),
+ num_cols: num_cols as u32,
+ length: (num_rows * num_cols) as u32,
+ scale: 1.0,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+ (input, expected)
+}
+
+fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.05f32
+ } else {
+ 1e-5
+ };
+ let (input, expected) = get_test_data_basic::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!("TensorAddScale basic test failed for backend {}", std::any::type_name::<B>()),
+ );
+ });
+}
+
+fn test_large<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.05f32
+ } else {
+ 1e-5
+ };
+ let (input, expected) = get_test_data_large::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!("TensorAddScale large test failed for backend {}", std::any::type_name::<B>()),
+ );
+ });
+}
+
+fn test_edge<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.05f32
+ } else {
+ 1e-5
+ };
+ let (input, expected) = get_test_data_edge::<T>();
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!("TensorAddScale edge test failed for backend {}", std::any::type_name::<B>()),
+ );
+ });
+}
+
+// basic tests
+#[test]
+fn test_basic_f32() {
+ test_basic::<f32>();
+}
+
+#[test]
+fn test_basic_f16() {
+ test_basic::<f16>();
+}
+
+#[test]
+fn test_basic_bf16() {
+ test_basic::<bf16>();
+}
+
+// large tests
+#[test]
+fn test_large_f32() {
+ test_large::<f32>();
+}
+
+#[test]
+fn test_large_f16() {
+ test_large::<f16>();
+}
+
+#[test]
+fn test_large_bf16() {
+ test_large::<bf16>();
+}
+
+// edge tests
+#[test]
+fn test_edge_f32() {
+ test_edge::<f32>();
+}
+
+#[test]
+fn test_edge_f16() {
+ test_edge::<f16>();
+}
+
+#[test]
+fn test_edge_bf16() {
+ test_edge::<bf16>();
+}