A high-performance inference engine for AI models
More CPU kernels (#287)
Aleksandr Golokoze818f07b4fff
diff --git a/crates/uzu/src/backends/common/gpu_types/quantization.rs b/crates/uzu/src/backends/common/gpu_types/quantization.rsindex 416e9370f..3f29b4726 100644--- a/crates/uzu/src/backends/common/gpu_types/quantization.rs+++ b/crates/uzu/src/backends/common/gpu_types/quantization.rs@@ -27,6 +27,10 @@ impl QuantizationMode {QuantizationMode::UINT8 => DataType::U8,}}++ pub fn to_u32(&self) -> u32 {+ (*self).into()+ }}impl From<QuantizationMode> for DataType {@@ -39,6 +43,17 @@ impl From<QuantizationMode> for DataType {}}+impl From<u32> for QuantizationMode {+ fn from(val: u32) -> Self {+ match val {+ 0 => QuantizationMode::UINT4,+ 1 => QuantizationMode::INT8,+ 2 => QuantizationMode::UINT8,+ _ => panic!("Invalid QuantizationMode value: {val}"),+ }+ }+}+impl Into<u32> for QuantizationMode {fn into(self) -> u32 {match self {diff --git a/crates/uzu/src/backends/cpu/kernel/activation/activation.rs b/crates/uzu/src/backends/cpu/kernel/activation/activation.rsindex 2b967ff48..58faee590 100644--- a/crates/uzu/src/backends/cpu/kernel/activation/activation.rs+++ b/crates/uzu/src/backends/cpu/kernel/activation/activation.rs@@ -7,15 +7,19 @@ use crate::ArrayElement;#[kernel(Activation)]#[variants(T, f16, f32, bf16)]pub fn activation<T: ArrayElement + Float>(- #[allow(unused)]- #[optional(!in_place)]- input: Option<*const T>,- #[allow(unused)] output: *mut T,- #[allow(unused)] n: u32,- #[allow(unused)] act_type: crate::backends::common::gpu_types::activation_type::ActivationType,- #[allow(unused)]- #[specialize]- in_place: bool,+ #[optional(!in_place)] input: Option<*const T>,+ output: *mut T,+ n: u32,+ act_type: crate::backends::common::gpu_types::activation_type::ActivationType,+ #[specialize] in_place: bool,) {- todo!()+ let input = match in_place {+ true => output,+ false => input.unwrap(),+ };+ unsafe {+ for i in 0..n as usize {+ *output.add(i) = act_type.activate(*input.add(i));+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rsindex 7afa88cad..3be696c10 100644--- a/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs@@ -4,66 +4,179 @@ use num_traits::Float;use crate::ArrayElement;+const TOTAL_BLOCKS_COUNT: u32 = 32;+const SEQUENCE_BLOCK_SIZE_1: u32 = 8;+#[kernel(AttentionTwoPass1)]#[variants(T, f32, f16, bf16)]#[variants(HEAD_DIM, 64, 128, 256)]pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(- #[allow(unused)] queries: *const T,- #[allow(unused)] keys: *const T,- #[allow(unused)] values: *const T,- #[allow(unused)] out: *mut f32,- #[allow(unused)] sums: *mut f32,- #[allow(unused)] maxs: *mut f32,- #[allow(unused)] gqa_factor: u32,- #[allow(unused)] sequence_length: u32,- #[allow(unused)] k_head_stride: u32,- #[allow(unused)] k_seq_stride: u32,- #[allow(unused)] v_head_stride: u32,- #[allow(unused)] v_seq_stride: u32,- #[allow(unused)] scale: f32,- #[allow(unused)] num_heads: u32,- #[allow(unused)] suffix_length: u32,- #[allow(unused)]- #[optional(float_mask)]- fmask: Option<*const T>,- #[allow(unused)]- #[optional(has_mask)]- mask_kv_seq_stride: Option<u32>,- #[allow(unused)]- #[optional(has_mask)]- mask_q_seq_stride: Option<u32>,- #[allow(unused)]- #[optional(has_mask)]- mask_head_stride: Option<u32>,- #[allow(unused)]- #[optional(has_sinks)]- sinks: Option<*const f32>,- #[allow(unused)]- #[specialize]- float_mask: bool,- #[allow(unused)]- #[specialize]- has_mask: bool,- #[allow(unused)]- #[specialize]- has_sinks: bool,- #[allow(unused)]- #[specialize]- do_causal: bool,+ queries: *const T,+ keys: *const T,+ values: *const T,+ out: *mut f32,+ sums: *mut f32,+ maxs: *mut f32,+ gqa_factor: u32,+ sequence_length: u32,+ k_head_stride: u32,+ k_seq_stride: u32,+ v_head_stride: u32,+ v_seq_stride: u32,+ scale: f32,+ num_heads: u32,+ suffix_length: u32,+ #[optional(float_mask)] fmask: Option<*const T>,+ #[optional(has_mask)] mask_kv_seq_stride: Option<u32>,+ #[optional(has_mask)] mask_q_seq_stride: Option<u32>,+ #[optional(has_mask)] mask_head_stride: Option<u32>,+ #[optional(has_sinks)] sinks: Option<*const f32>,+ #[specialize] float_mask: bool,+ #[specialize] has_mask: bool,+ #[specialize] has_sinks: bool,+ #[specialize] do_causal: bool,) {- todo!()+ let value_dim = HEAD_DIM;+ let mut q = vec![0.0f32; HEAD_DIM as usize];+ let mut o = vec![0.0f32; HEAD_DIM as usize];++ for head_idx in 0..num_heads {+ for q_seq_idx in 0..suffix_length {+ for block_idx in 0..TOTAL_BLOCKS_COUNT {+ let o_offset = q_seq_idx * num_heads + head_idx;+ let q_offset = head_idx * suffix_length + q_seq_idx;+ let kv_head_idx = head_idx / gqa_factor;++ let queries_base: *const T = unsafe { queries.add((q_offset * HEAD_DIM) as usize) };+ let keys_base: *const T = unsafe { keys.add((kv_head_idx * k_head_stride) as usize) };+ let values_base: *const T = unsafe { values.add((kv_head_idx * v_head_stride) as usize) };+ let out_base: *mut f32 =+ unsafe { out.add((o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim) as usize) };++ let fmask_base: Option<*const T> = if float_mask {+ let offset = head_idx * mask_head_stride.unwrap_or(0) + q_seq_idx * mask_q_seq_stride.unwrap_or(0);+ Some(unsafe { fmask.unwrap().add(offset as usize) })+ } else {+ None+ };++ // Read the query and scale+ for j in 0..HEAD_DIM as usize {+ q[j] = scale * unsafe { *queries_base.add(j) }.to_f32().unwrap();+ }+ o.fill(0f32);++ let mut max_score = -1e9f32;+ let mut sum_exp_score = 0.0f32;++ if has_sinks && block_idx == 0 {+ let q_head_idx = head_idx % num_heads;+ max_score = unsafe { *sinks.unwrap().add(q_head_idx as usize) };+ sum_exp_score = 1.0;+ }++ // Iterate over sequence positions assigned to this block+ let mut chunk_start = block_idx * SEQUENCE_BLOCK_SIZE_1;+ while chunk_start < sequence_length {+ for t in 0..SEQUENCE_BLOCK_SIZE_1 {+ let i = chunk_start + t;+ if i >= sequence_length {+ break;+ }++ let use_key = !do_causal || i <= (sequence_length - suffix_length + q_seq_idx);+ if use_key {+ let keys_ptr = unsafe { keys_base.add((i * k_seq_stride) as usize) };++ // Compute the i-th score+ let mut score = 0.0f32;+ for j in 0..HEAD_DIM as usize {+ score += q[j] * unsafe { *keys_ptr.add(j) }.to_f32().unwrap();+ }+ if float_mask {+ let fmask_ptr =+ unsafe { fmask_base.unwrap().add((i * mask_kv_seq_stride.unwrap()) as usize) };+ score += f32::max(-1e9, unsafe { *fmask_ptr }.to_f32().unwrap());+ }++ // Update the accumulators+ let new_max = f32::max(max_score, score);+ let factor = (max_score - new_max).exp();+ let exp_score = (score - new_max).exp();++ max_score = new_max;+ sum_exp_score = sum_exp_score * factor + exp_score;++ // Update the output accumulator+ let values_ptr = unsafe { values_base.add((i * v_seq_stride) as usize) };+ for j in 0..HEAD_DIM as usize {+ o[j] = o[j] * factor + exp_score * unsafe { *values_ptr.add(j) }.to_f32().unwrap();+ }+ }+ }+ chunk_start += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1;+ }++ // Write partial output, sum, and max for this block+ for j in 0..HEAD_DIM as usize {+ unsafe {+ *out_base.add(j) = o[j];+ }+ }+ unsafe {+ *sums.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) = sum_exp_score;+ *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) = max_score;+ }+ }+ }+ }}#[kernel(AttentionTwoPass2)]#[variants(T, f32, f16, bf16)]#[variants(HEAD_DIM, 64, 128, 256)]pub fn attention_two_pass2<T: ArrayElement + Float, const HEAD_DIM: u32>(- #[allow(unused)] partials: *const f32,- #[allow(unused)] sums: *const f32,- #[allow(unused)] maxs: *const f32,- #[allow(unused)] out: *mut T,- #[allow(unused)] num_heads: u32,- #[allow(unused)] suffix_length: u32,+ partials: *const f32,+ sums: *const f32,+ maxs: *const f32,+ out: *mut T,+ num_heads: u32,+ suffix_length: u32,) {- todo!()+ for head_idx in 0..num_heads {+ for q_seq_idx in 0..suffix_length {+ let o_offset = q_seq_idx * num_heads + head_idx;++ // Find global max across all blocks+ let mut global_max = f32::NEG_INFINITY;+ for block_idx in 0..TOTAL_BLOCKS_COUNT {+ let max_val = unsafe { *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };+ global_max = f32::max(global_max, max_val);+ }++ // Compute global sum of exponentials with correction+ let mut global_sum = 0.0f32;+ for block_idx in 0..TOTAL_BLOCKS_COUNT {+ let sum_val = unsafe { *sums.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };+ let max_val = unsafe { *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };+ global_sum += sum_val * (max_val - global_max).exp();+ }++ // Combine partial outputs+ let out_ptr = unsafe { out.add((o_offset * HEAD_DIM) as usize) };+ for j in 0..HEAD_DIM as usize {+ let mut val = 0.0f32;+ for block_idx in 0..TOTAL_BLOCKS_COUNT {+ let partial_val = unsafe {+ *partials.add((o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM + block_idx * HEAD_DIM) as usize + j)+ };+ let max_val = unsafe { *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };+ val += partial_val * (max_val - global_max).exp();+ }+ unsafe {+ *out_ptr.add(j) = T::from(val / global_sum).unwrap();+ }+ }+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/embedding_rows_sum.rs b/crates/uzu/src/backends/cpu/kernel/embedding/embedding_rows_sum.rsindex eccc8c7f1..24c7db766 100644--- a/crates/uzu/src/backends/cpu/kernel/embedding/embedding_rows_sum.rs+++ b/crates/uzu/src/backends/cpu/kernel/embedding/embedding_rows_sum.rs@@ -7,13 +7,29 @@ use crate::ArrayElement;#[kernel(EmbeddingRowsSum)]#[variants(T, f32, f16, bf16)]pub fn embedding_rows_sum<T: ArrayElement + Float>(- #[allow(unused)] token_indices: *const u32,- #[allow(unused)] weights: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] num_rows: u32,- #[allow(unused)] total_rows: u32,- #[allow(unused)] model_dim: u32,- #[allow(unused)] codebook_stride: u32,+ token_indices: *const u32,+ weights: *const T,+ output: *mut T,+ num_rows: u32,+ total_rows: u32,+ model_dim: u32,+ codebook_stride: u32,) {- todo!()+ let codebook_stride = codebook_stride as usize;+ let total_rows = total_rows as usize;+ let model_dim = model_dim as usize;++ unsafe {+ for dim_idx in 0..model_dim {+ let mut sum = 0f32;+ for row_idx in 0..num_rows as usize {+ let token = *token_indices.add(row_idx) as usize;+ let row = row_idx * codebook_stride + token;+ if row < total_rows {+ sum += (*weights.add(row * model_dim + dim_idx)).to_f32().unwrap();+ }+ }+ *output.add(dim_idx) = T::from(sum).unwrap();+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rsindex 7fa0e433a..363d9f644 100644--- a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs+++ b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs@@ -2,7 +2,7 @@ use dsl::kernel;use half::{bf16, f16};use num_traits::Float;-use crate::ArrayElement;+use crate::{ArrayElement, backends::common::gpu_types::QuantizationMode};#[kernel(QuantizedEmbeddingLookup)]#[variants(T, f32, f16, bf16)]@@ -23,5 +23,57 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(#[specialize]quant_mode: u32,) {- todo!()+ let quant_mode = QuantizationMode::from(quant_mode);++ let packing_divisor: u32 = if quant_mode == QuantizationMode::UINT4 {+ 2+ } else {+ 1+ };+ let weights_stride = model_dim / packing_divisor;+ let num_groups = (model_dim + group_size - 1) / group_size;++ unsafe {+ for batch_idx in 0..batch_size {+ let token_id = *token_ids.add(batch_idx as usize);++ for dim_idx in 0..model_dim {+ let out_idx = (batch_idx * model_dim + dim_idx) as usize;++ if token_id >= vocab_size as u64 {+ *output.add(out_idx) = T::zero();+ continue;+ }++ let group_idx = dim_idx / group_size;+ let scale = *scales.add((token_id as u32 * num_groups + group_idx) as usize);+ let bias = *biases.add((token_id as u32 * num_groups + group_idx) as usize);++ let quantized_value: i32 = match quant_mode {+ QuantizationMode::UINT4 => {+ let byte_idx = (token_id as u32 * weights_stride + dim_idx / 2) as usize;+ let packed = *weights.add(byte_idx);+ if (dim_idx & 1) == 0 {+ (packed & 0x0F) as i32+ } else {+ ((packed >> 4) & 0x0F) as i32+ }+ },+ QuantizationMode::INT8 => {+ let elem_idx = (token_id as u32 * weights_stride + dim_idx) as usize;+ let weights_i8 = weights as *const i8;+ *weights_i8.add(elem_idx) as i32+ },+ QuantizationMode::UINT8 => {+ let elem_idx = (token_id as u32 * weights_stride + dim_idx) as usize;+ *weights.add(elem_idx) as i32+ },+ };++ let out_f = scale.to_f32().unwrap() * quantized_value as f32 + bias.to_f32().unwrap();+ let out_f = out_f * input_scale;+ *output.add(out_idx) = T::from(out_f).unwrap();+ }+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/kv_cache_update/kv_cache_update.rs b/crates/uzu/src/backends/cpu/kernel/kv_cache_update/kv_cache_update.rsindex 58e758c58..ebdaea8f3 100644--- a/crates/uzu/src/backends/cpu/kernel/kv_cache_update/kv_cache_update.rs+++ b/crates/uzu/src/backends/cpu/kernel/kv_cache_update/kv_cache_update.rs@@ -7,13 +7,33 @@ use crate::ArrayElement;#[kernel(KVCacheUpdate)]#[variants(T, f32, bf16, f16)]pub fn kv_cache_update<T: ArrayElement + Float>(- #[allow(unused)] in_place_keys: *mut T,- #[allow(unused)] in_place_values: *mut T,- #[allow(unused)] swaps: &[crate::backends::common::gpu_types::kv_cache_update::Swap],- #[allow(unused)] swap_count: u32,- #[allow(unused)] num_heads: u32,- #[allow(unused)] max_sequence_length: u32,- #[allow(unused)] head_dim: u32,+ in_place_keys: *mut T,+ in_place_values: *mut T,+ swaps: &[crate::backends::common::gpu_types::kv_cache_update::Swap],+ swap_count: u32,+ num_heads: u32,+ max_sequence_length: u32,+ head_dim: u32,) {- todo!()+ let max_sequence_length = max_sequence_length as usize;+ let head_dim = head_dim as usize;++ for head_idx in 0..num_heads as usize {+ for channel_idx in 0..head_dim {+ for i in 0..swap_count as usize {+ let head_offset = head_idx * max_sequence_length * head_dim;+ let source_idx = head_offset + (swaps[i].source as usize) * head_dim + channel_idx;+ let dest_idx = head_offset + (swaps[i].destination as usize) * head_dim + channel_idx;+ unsafe {+ let keys_src_ptr = in_place_keys.add(source_idx);+ let mut keys_dst_ptr = in_place_keys.add(dest_idx);+ keys_src_ptr.swap(keys_dst_ptr);++ let values_src_ptr = in_place_values.add(source_idx);+ let mut values_dst_ptr = in_place_values.add(dest_idx);+ values_src_ptr.swap(values_dst_ptr);+ }+ }+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/layer_norm/layer_norm.rs b/crates/uzu/src/backends/cpu/kernel/layer_norm/layer_norm.rsindex 92e68ed43..79bd8a719 100644--- a/crates/uzu/src/backends/cpu/kernel/layer_norm/layer_norm.rs+++ b/crates/uzu/src/backends/cpu/kernel/layer_norm/layer_norm.rs@@ -15,19 +15,62 @@ pub fn layer_norm<OUT: ArrayElement + Float,ACC: ArrayElement + Float,>(- #[allow(unused)]- #[optional(!in_place)]- input: Option<*const IN>,- #[allow(unused)] scales: *const SC,- #[allow(unused)] output: *mut OUT,- #[allow(unused)] batch_size: u32,- #[allow(unused)] model_dim: u32,- #[allow(unused)] epsilon: f32,- #[allow(unused)] scale_offset: f32,- #[allow(unused)] full_layer: u32,- #[allow(unused)]- #[specialize]- in_place: bool,+ #[optional(!in_place)] input: Option<*const IN>,+ scales: *const SC,+ output: *mut OUT,+ batch_size: u32,+ model_dim: u32,+ epsilon: f32,+ scale_offset: f32,+ full_layer: u32,+ #[specialize] in_place: bool,) {- todo!()+ let input = match in_place {+ true => output as *const IN,+ false => input.unwrap(),+ };++ let model_dim = model_dim as usize;+ let epsilon = ACC::from(epsilon).unwrap();+ let scale_offset = ACC::from(scale_offset).unwrap();+ let element_count_accum = ACC::from(model_dim).unwrap();+ let full_layer = full_layer != 0;++ for batch in 0..(batch_size as usize) {+ let batch_offset = batch * model_dim;++ // Compute mean+ let mut sum = ACC::zero();+ for i in 0..model_dim {+ let val = unsafe { ACC::from(*input.add(batch_offset + i)).unwrap() };+ sum = sum + val;+ }+ let mean = sum / element_count_accum;++ // Compute variance+ let mut var_sum = ACC::zero();+ for i in 0..model_dim {+ let val = unsafe { ACC::from(*input.add(batch_offset + i)).unwrap() };+ let centered = val - mean;+ var_sum = var_sum + centered * centered;+ }+ let variance = var_sum / element_count_accum;+ let inv_std = ACC::from((variance + epsilon).to_f32().unwrap().sqrt().recip()).unwrap();++ // Normalize and scale+ for i in 0..model_dim {+ let input_val = unsafe { ACC::from(*input.add(batch_offset + i)).unwrap() };+ let scale_val = unsafe { ACC::from(*scales.add(i)).unwrap() };+ let normalized: ACC = (input_val - mean) * inv_std;+ let result: OUT = if full_layer {+ let scale_with_offset: ACC = scale_val + scale_offset;+ OUT::from(normalized * scale_with_offset).unwrap()+ } else {+ let normalized_out = OUT::from(normalized).unwrap();+ let scale_with_offset_out = OUT::from(scale_val + scale_offset).unwrap();+ normalized_out * scale_with_offset_out+ };+ unsafe { *output.add(batch_offset + i) = result };+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs b/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rsindex 97498c6e4..cb7733683 100644--- a/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs+++ b/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs@@ -7,9 +7,14 @@ use crate::ArrayElement;#[kernel(MaskUpdate)]#[variants(T, f32, f16, bf16)]pub fn mask_update<T: ArrayElement + Float>(- #[allow(unused)] mask: *mut T,- #[allow(unused)] unmask_col: i32,- #[allow(unused)] mask_col: i32,+ mask: *mut T,+ unmask_col: i32,+ mask_col: i32,) {- todo!()+ if (unmask_col >= 0) {+ unsafe { *mask.add(unmask_col as usize) = T::zero() };+ }+ if (mask_col >= 0) {+ unsafe { *mask.add(mask_col as usize) = T::neg_infinity() };+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/pooling/pooling.rs b/crates/uzu/src/backends/cpu/kernel/pooling/pooling.rsindex 9daf2804b..90191d6c9 100644--- a/crates/uzu/src/backends/cpu/kernel/pooling/pooling.rs+++ b/crates/uzu/src/backends/cpu/kernel/pooling/pooling.rs@@ -1,29 +1,56 @@use dsl::kernel;use half::{bf16, f16};-use num_traits::Float;+use num_traits::{Float, ToPrimitive};use crate::ArrayElement;#[kernel(PoolingCls)]#[variants(T, f32, f16, bf16)]pub fn pooling_cls<T: ArrayElement + Float>(- #[allow(unused)] input: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] seq_len: u32,- #[allow(unused)] hidden_dim: u32,- #[allow(unused)] batch_size: u32,+ input: *const T,+ output: *mut T,+ seq_len: u32,+ hidden_dim: u32,+ batch_size: u32,) {- todo!()+ let seq_len = seq_len as usize;+ let hidden_dim = hidden_dim as usize;+ let batch_size = batch_size as usize;++ for dim_idx in 0..hidden_dim {+ for batch_idx in 0..batch_size {+ unsafe {+ *output.add(batch_idx * hidden_dim + dim_idx) = *input.add(batch_idx * hidden_dim * seq_len + dim_idx);+ }+ }+ }}#[kernel(PoolingMean)]#[variants(T, f32, f16, bf16)]pub fn pooling_mean<T: ArrayElement + Float>(- #[allow(unused)] input: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] seq_len: u32,- #[allow(unused)] hidden_dim: u32,- #[allow(unused)] batch_size: u32,+ input: *const T,+ output: *mut T,+ seq_len: u32,+ hidden_dim: u32,+ batch_size: u32,) {- todo!()+ let seq_len = seq_len as usize;+ let hidden_dim = hidden_dim as usize;+ let batch_size = batch_size as usize;++ for dim_idx in 0..hidden_dim {+ for batch_idx in 0..batch_size {+ let mut sum = 0f32;+ for seq_idx in 0..seq_len {+ unsafe {+ let position = batch_idx * seq_len * hidden_dim + seq_idx * hidden_dim + dim_idx;+ sum += (*input.add(position)).to_f32().unwrap();+ }+ }+ unsafe {+ *output.add(batch_idx * hidden_dim + dim_idx) = T::from(sum / seq_len.to_f32().unwrap()).unwrap();+ }+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/bitmask.rs b/crates/uzu/src/backends/cpu/kernel/sampling/bitmask.rsindex e99000ba7..4fac35672 100644--- a/crates/uzu/src/backends/cpu/kernel/sampling/bitmask.rs+++ b/crates/uzu/src/backends/cpu/kernel/sampling/bitmask.rs@@ -7,16 +7,33 @@ use crate::ArrayElement;#[kernel(Bitmask)]#[variants(T, f32, f16, bf16)]pub fn bitmask<T: ArrayElement + Float>(- #[allow(unused)]- #[optional(!in_place)]- logits: Option<*const T>,- #[allow(unused)] bitmask: *const u32,- #[allow(unused)] processed_logits: *mut T,- #[allow(unused)] batch_size: u32,- #[allow(unused)] vocab_size: u32,- #[allow(unused)]- #[specialize]- in_place: bool,+ #[optional(!in_place)] logits: Option<*const T>,+ bitmask: *const u32,+ processed_logits: *mut T,+ batch_size: u32,+ vocab_size: u32,+ #[specialize] in_place: bool,) {- todo!()+ let logits: *const T = match in_place {+ true => processed_logits,+ false => logits.unwrap(),+ };++ let batch_size = batch_size as usize;+ let vocab_size = vocab_size as usize;+ let bitmask_size = vocab_size.div_ceil(32);++ for batch_idx in 0..batch_size {+ for token_idx in 0..vocab_size {+ let global_idx = batch_idx * vocab_size + token_idx;+ let bitmask_idx = batch_idx * bitmask_size + token_idx / 32;+ unsafe {+ let mask = (*bitmask.add(bitmask_idx) >> (token_idx % 32)) & 1;+ *processed_logits.add(global_idx) = match mask != 0 {+ true => *logits.add(global_idx),+ false => T::neg_infinity(),+ };+ }+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/min_p.rs b/crates/uzu/src/backends/cpu/kernel/sampling/min_p.rsindex c32c6c23a..71af24405 100644--- a/crates/uzu/src/backends/cpu/kernel/sampling/min_p.rs+++ b/crates/uzu/src/backends/cpu/kernel/sampling/min_p.rs@@ -7,16 +7,40 @@ use crate::ArrayElement;#[kernel(MinP)]#[variants(T, f32, f16, bf16)]pub fn min_p<T: ArrayElement + Float>(- #[allow(unused)]- #[optional(!in_place)]- logits: Option<*const T>,- #[allow(unused)] processed_logits: *mut T,- #[allow(unused)] batch_size: u32,- #[allow(unused)] vocab_size: u32,- #[allow(unused)] min_p: f32,- #[allow(unused)]- #[specialize]- in_place: bool,+ #[optional(!in_place)] logits: Option<*const T>,+ processed_logits: *mut T,+ batch_size: u32,+ vocab_size: u32,+ min_p: f32,+ #[specialize] in_place: bool,) {- todo!()+ let logits = match in_place {+ true => processed_logits,+ false => logits.unwrap(),+ };++ let batch_size = batch_size as usize;+ let vocab_size = vocab_size as usize;++ for batch_idx in 0..batch_size {+ // find maximum logit+ let mut max_logit = f32::NEG_INFINITY;+ for i in 0..vocab_size {+ let logit = unsafe { *logits.add(batch_idx * vocab_size + i) }.to_f32().unwrap();+ max_logit = max_logit.max(logit);+ }++ // then the threshold is just max_logit + log(min_p), mask everything strictly below it+ let threshold: T = T::from(max_logit + min_p.ln()).unwrap();+ for i in 0..vocab_size {+ let position = batch_idx * vocab_size + i;+ unsafe {+ let logit = *logits.add(position);+ *processed_logits.add(position) = match logit >= threshold {+ true => logit,+ false => T::neg_infinity(),+ }+ }+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs b/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rsindex dc3aacc56..ba1ad5b59 100644--- a/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs+++ b/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs@@ -7,9 +7,15 @@ use crate::ArrayElement;#[kernel(Sigmoid)]#[variants(T, f32, f16, bf16)]pub fn sigmoid<T: ArrayElement + Float>(- #[allow(unused)] input: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] total_elements: u32,+ input: *const T,+ output: *mut T,+ total_elements: u32,) {- todo!()+ for i in 0..total_elements as usize {+ unsafe {+ let input_value = (*input.add(i)).to_f32().unwrap();+ let value = 1.0 / (1.0 + (-input_value).exp());+ *output.add(i) = T::from(value).unwrap();+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rsindex e0d2289b5..f61c5f917 100644--- a/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs@@ -7,12 +7,20 @@ use crate::ArrayElement;#[kernel(TensorAddScale)]#[variants(T, f32, f16, bf16)]pub fn tensor_add_scale<T: ArrayElement + Float>(- #[allow(unused)] input: *const T,- #[allow(unused)] bias: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] num_cols: u32,- #[allow(unused)] length: u32,- #[allow(unused)] scale: f32,+ input: *const T,+ bias: *const T,+ output: *mut T,+ num_cols: u32,+ length: u32,+ scale: f32,) {- todo!()+ let num_cols = num_cols as usize;+ for position in 0..length as usize {+ let col = position % num_cols;+ unsafe {+ let input_value = (*input.add(position)).to_f32().unwrap();+ let bias_value = (*bias.add(col)).to_f32().unwrap();+ *output.add(position) = T::from((input_value + bias_value) * scale).unwrap();+ }+ }}diff --git a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metalindex 4ddc4c8b7..e4adb869f 100644--- a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal+++ b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal@@ -9,8 +9,6 @@ PUBLIC KERNEL(Sigmoid) (const constant uint& total_elements,uint tid AXIS(total_elements, 256)) {- if (tid >= total_elements)- return;float x = float(input[tid]);output[tid] = T(1.0f / (1.0f + exp(-x)));}\ No newline at end of filediff --git a/crates/uzu/tests/common/assert.rs b/crates/uzu/tests/common/assert.rsindex 5cf17e43f..ec9711815 100644--- a/crates/uzu/tests/common/assert.rs+++ b/crates/uzu/tests/common/assert.rs@@ -18,6 +18,10 @@ pub fn assert_eq_float<T: ArrayElement + Float + Display>();for i in 0..expected.len() {+ if expected[i] == actual[i] {+ continue;+ }+let diff = (expected[i] - actual[i]).to_f32().unwrap().abs();assert!(diff < eps,diff --git a/crates/uzu/tests/unit/kernel/activation_test.rs b/crates/uzu/tests/unit/kernel/activation_test.rsnew file mode 100644index 000000000..a06798046--- /dev/null+++ b/crates/uzu/tests/unit/kernel/activation_test.rs@@ -0,0 +1,260 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{Backend, Context, Encoder, Kernels, gpu_types::ActivationType, kernel::ActivationKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ data: Box<[T]>,+ act_type: ActivationType,+ in_place: bool,+}++fn make_data<T: ArrayElement + Float>(n: usize) -> Vec<T> {+ let mut data = Vec::with_capacity(n);+ for i in 0..n {+ // Use a range of values including negative to exercise both branches of activations+ data.push(T::from((i as f32 * 0.3) - 2.0).unwrap());+ }+ data+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");+ let kernel = <<B as Backend>::Kernels as Kernels>::ActivationKernel::new(&context, T::data_type(), input.in_place)+ .expect("Failed to create ActivationKernel");++ let n = input.data.len();++ if input.in_place {+ let output_array = context.create_array_from(&[n], &input.data, "");+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");+ kernel.encode(+ Option::<&B::Buffer>::None,+ output_array.buffer().borrow_mut().deref_mut(),+ n as u32,+ input.act_type,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ output_array.as_slice().to_vec()+ } else {+ let input_array = context.create_array_from(&[n], &input.data, "");+ let output_array = context.create_array_uninitialized(&[n], T::data_type(), "");+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");+ kernel.encode(+ Some(input_array.buffer().borrow().deref()),+ output_array.buffer().borrow_mut().deref_mut(),+ n as u32,+ input.act_type,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ output_array.as_slice().to_vec()+ }+}++fn get_test_data<T: ArrayElement + Float>(+ act_type: ActivationType,+ in_place: bool,+) -> (Input<T>, Vec<T>) {+ let n = 128;+ let data = make_data::<T>(n);++ let input = Input {+ data: data.into_boxed_slice(),+ act_type,+ in_place,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++/// Large input to exercise more threads on GPU.+fn get_test_data_large<T: ArrayElement + Float>(+ act_type: ActivationType,+ in_place: bool,+) -> (Input<T>, Vec<T>) {+ let n = 4096;+ let data = make_data::<T>(n);++ let input = Input {+ data: data.into_boxed_slice(),+ act_type,+ in_place,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ input: &Input<T>,+ expected: &[T],+ test_name: &str,+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.02f32+ } else {+ 1e-5+ };++ for_each_non_cpu_backend!(|B| {+ let actual = get_output::<T, B>(input);+ let msg = format!("Activation {} failed with backend={}", test_name, std::any::type_name::<B>(),);+ assert_eq_float::<T>(expected, &actual, eps, &msg);+ });+}++fn test_activation<T: ArrayElement + Float + Debug + Display>(+ act_type: ActivationType,+ in_place: bool,+) {+ let label = format!(+ "{:?}_{}",+ act_type,+ if in_place {+ "in_place"+ } else {+ "out_of_place"+ }+ );+ let (input, expected) = get_test_data::<T>(act_type, in_place);+ test_internal::<T>(&input, &expected, &label);+}++fn test_activation_large<T: ArrayElement + Float + Debug + Display>(+ act_type: ActivationType,+ in_place: bool,+) {+ let label = format!(+ "{:?}_{}_large",+ act_type,+ if in_place {+ "in_place"+ } else {+ "out_of_place"+ }+ );+ let (input, expected) = get_test_data_large::<T>(act_type, in_place);+ test_internal::<T>(&input, &expected, &label);+}++// SILU out-of-place tests+#[test]+fn test_silu_f32() {+ test_activation::<f32>(ActivationType::SILU, false);+}++#[test]+fn test_silu_f16() {+ test_activation::<f16>(ActivationType::SILU, false);+}++#[test]+fn test_silu_bf16() {+ test_activation::<bf16>(ActivationType::SILU, false);+}++// SILU in-place tests+#[test]+fn test_silu_in_place_f32() {+ test_activation::<f32>(ActivationType::SILU, true);+}++#[test]+fn test_silu_in_place_f16() {+ test_activation::<f16>(ActivationType::SILU, true);+}++#[test]+fn test_silu_in_place_bf16() {+ test_activation::<bf16>(ActivationType::SILU, true);+}++// GELU out-of-place tests+#[test]+fn test_gelu_f32() {+ test_activation::<f32>(ActivationType::GELU, false);+}++#[test]+fn test_gelu_f16() {+ test_activation::<f16>(ActivationType::GELU, false);+}++#[test]+fn test_gelu_bf16() {+ test_activation::<bf16>(ActivationType::GELU, false);+}++// GELU in-place tests+#[test]+fn test_gelu_in_place_f32() {+ test_activation::<f32>(ActivationType::GELU, true);+}++#[test]+fn test_gelu_in_place_f16() {+ test_activation::<f16>(ActivationType::GELU, true);+}++#[test]+fn test_gelu_in_place_bf16() {+ test_activation::<bf16>(ActivationType::GELU, true);+}++// Large SILU tests+#[test]+fn test_silu_large_f32() {+ test_activation_large::<f32>(ActivationType::SILU, false);+}++#[test]+fn test_silu_large_f16() {+ test_activation_large::<f16>(ActivationType::SILU, false);+}++#[test]+fn test_silu_large_bf16() {+ test_activation_large::<bf16>(ActivationType::SILU, false);+}++// Large GELU tests+#[test]+fn test_gelu_large_f32() {+ test_activation_large::<f32>(ActivationType::GELU, false);+}++#[test]+fn test_gelu_large_f16() {+ test_activation_large::<f16>(ActivationType::GELU, false);+}++#[test]+fn test_gelu_large_bf16() {+ test_activation_large::<bf16>(ActivationType::GELU, false);+}++// Large in-place tests+#[test]+fn test_silu_in_place_large_f32() {+ test_activation_large::<f32>(ActivationType::SILU, true);+}++#[test]+fn test_gelu_in_place_large_f32() {+ test_activation_large::<f32>(ActivationType::GELU, true);+}diff --git a/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rsnew file mode 100644index 000000000..f9b2936fa--- /dev/null+++ b/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs@@ -0,0 +1,480 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{+ Backend, Context, Encoder, Kernels,+ kernel::{AttentionTwoPass1Kernel, AttentionTwoPass2Kernel},+ },+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++const TOTAL_BLOCKS_COUNT: u32 = 32;++// --- First pass types and helpers ---++struct FirstPassInput<T: ArrayElement + Float> {+ queries: Box<[T]>,+ keys: Box<[T]>,+ values: Box<[T]>,+ mask: Option<Box<[T]>>,+ num_heads: u32,+ gqa_factor: u32,+ sequence_length: u32,+ suffix_length: u32,+ head_dim: u32,+ scale: f32,+ do_causal: bool,+ has_mask: bool,+}++struct FirstPassOutput {+ partials: Vec<f32>,+ sums: Vec<f32>,+ maxs: Vec<f32>,+}++fn get_first_pass_input<T: ArrayElement + Float>(+ num_heads: u32,+ num_kv_heads: u32,+ sequence_length: u32,+ suffix_length: u32,+ head_dim: u32,+ do_causal: bool,+ with_mask: bool,+) -> FirstPassInput<T> {+ let gqa_factor = num_heads / num_kv_heads;++ let q_size = (num_heads * suffix_length * head_dim) as usize;+ let mut queries = vec![T::zero(); q_size];+ for i in 0..q_size {+ queries[i] = T::from((i as f32 * 0.13 + 0.5).sin() * 0.5).unwrap();+ }++ let k_size = (num_kv_heads * sequence_length * head_dim) as usize;+ let mut keys = vec![T::zero(); k_size];+ for i in 0..k_size {+ keys[i] = T::from((i as f32 * 0.07 + 1.0).cos() * 0.5).unwrap();+ }++ let v_size = (num_kv_heads * sequence_length * head_dim) as usize;+ let mut values = vec![T::zero(); v_size];+ for i in 0..v_size {+ values[i] = T::from((i as f32 * 0.11 + 2.0).sin() * 0.5).unwrap();+ }++ let mask = if with_mask {+ let m_size = (suffix_length * sequence_length) as usize;+ let mut m = vec![T::zero(); m_size];+ for q in 0..suffix_length {+ for k in 0..sequence_length {+ let idx = (q * sequence_length + k) as usize;+ let diff = (k as i32) - (sequence_length as i32 - suffix_length as i32 + q as i32);+ if diff.abs() > 3 {+ m[idx] = T::from(-1e9f32).unwrap();+ }+ }+ }+ Some(m.into_boxed_slice())+ } else {+ None+ };++ let scale = 1.0 / (head_dim as f32).sqrt();++ FirstPassInput {+ queries: queries.into_boxed_slice(),+ keys: keys.into_boxed_slice(),+ values: values.into_boxed_slice(),+ mask,+ num_heads,+ gqa_factor,+ sequence_length,+ suffix_length,+ head_dim,+ scale,+ do_causal,+ has_mask: with_mask,+ }+}++fn get_first_pass_output<T: ArrayElement + Float, B: Backend>(input: &FirstPassInput<T>) -> FirstPassOutput {+ let context = B::Context::new().expect("Failed to create Context");++ let float_mask = input.has_mask;+ let kernel = <<B as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(+ &context,+ T::data_type(),+ input.head_dim,+ float_mask,+ input.has_mask,+ false,+ input.do_causal,+ )+ .expect("Failed to create AttentionTwoPass1Kernel");++ let queries_array = context.create_array_from(&[input.queries.len()], &input.queries, "");+ let keys_array = context.create_array_from(&[input.keys.len()], &input.keys, "");+ let values_array = context.create_array_from(&[input.values.len()], &input.values, "");++ let total_offsets = (input.suffix_length * input.num_heads) as usize;+ let partials_size = total_offsets * TOTAL_BLOCKS_COUNT as usize * input.head_dim as usize;+ let sums_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;+ let maxs_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;++ let partials_array = context.create_array_uninitialized(&[partials_size], DataType::F32, "");+ let sums_array = context.create_array_uninitialized(&[sums_size], DataType::F32, "");+ let maxs_array = context.create_array_uninitialized(&[maxs_size], DataType::F32, "");++ let mask_array = input.mask.as_ref().map(|m| context.create_array_from(&[m.len()], m, ""));+ let mask_buf_rc = mask_array.as_ref().map(|a| a.buffer());+ let mask_buf_borrow = mask_buf_rc.as_ref().map(|rc| rc.borrow());+ let mask_buffer: Option<&B::Buffer> = mask_buf_borrow.as_ref().map(|b| b.deref());++ let mask_kv_seq_stride: Option<u32> = input.has_mask.then_some(1);+ let mask_q_seq_stride: Option<u32> = input.has_mask.then_some(input.sequence_length);+ let mask_head_stride: Option<u32> = input.has_mask.then_some(0);++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ queries_array.buffer().borrow().deref(),+ keys_array.buffer().borrow().deref(),+ values_array.buffer().borrow().deref(),+ partials_array.buffer().borrow_mut().deref_mut(),+ sums_array.buffer().borrow_mut().deref_mut(),+ maxs_array.buffer().borrow_mut().deref_mut(),+ input.gqa_factor,+ input.sequence_length,+ input.sequence_length * input.head_dim,+ input.head_dim,+ input.sequence_length * input.head_dim,+ input.head_dim,+ input.scale,+ input.num_heads,+ input.suffix_length,+ mask_buffer,+ mask_kv_seq_stride,+ mask_q_seq_stride,+ mask_head_stride,+ None::<&B::Buffer>,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ FirstPassOutput {+ partials: partials_array.as_slice().to_vec(),+ sums: sums_array.as_slice().to_vec(),+ maxs: maxs_array.as_slice().to_vec(),+ }+}++// --- Second pass types and helpers ---++struct SecondPassInput {+ partials: Box<[f32]>,+ sums: Box<[f32]>,+ maxs: Box<[f32]>,+ num_heads: u32,+ suffix_length: u32,+ head_dim: u32,+}++fn get_second_pass_input(+ num_heads: u32,+ suffix_length: u32,+ head_dim: u32,+) -> SecondPassInput {+ let total_offsets = (suffix_length * num_heads) as usize;+ let partials_size = total_offsets * TOTAL_BLOCKS_COUNT as usize * head_dim as usize;+ let sums_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;+ let maxs_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;++ let mut partials = vec![0.0f32; partials_size];+ for i in 0..partials_size {+ partials[i] = (i as f32 * 0.03 + 0.1).sin() * 0.5;+ }++ let mut sums = vec![0.0f32; sums_size];+ for i in 0..sums_size {+ // Ensure sums are positive (they represent sum of exp scores)+ sums[i] = (i as f32 * 0.07 + 1.0).cos().abs() + 0.1;+ }++ let mut maxs = vec![0.0f32; maxs_size];+ for i in 0..maxs_size {+ maxs[i] = (i as f32 * 0.05 + 0.5).sin() * 2.0;+ }++ SecondPassInput {+ partials: partials.into_boxed_slice(),+ sums: sums.into_boxed_slice(),+ maxs: maxs.into_boxed_slice(),+ num_heads,+ suffix_length,+ head_dim,+ }+}++fn get_second_pass_output<T: ArrayElement + Float, B: Backend>(input: &SecondPassInput) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel =+ <<B as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(&context, T::data_type(), input.head_dim)+ .expect("Failed to create AttentionTwoPass2Kernel");++ let partials_array = context.create_array_from(&[input.partials.len()], &input.partials, "");+ let sums_array = context.create_array_from(&[input.sums.len()], &input.sums, "");+ let maxs_array = context.create_array_from(&[input.maxs.len()], &input.maxs, "");++ let output_size = (input.suffix_length * input.num_heads * input.head_dim) as usize;+ let output_array = context.create_array_uninitialized(&[output_size], T::data_type(), "");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ partials_array.buffer().borrow().deref(),+ sums_array.buffer().borrow().deref(),+ maxs_array.buffer().borrow().deref(),+ output_array.buffer().borrow_mut().deref_mut(),+ input.num_heads,+ input.suffix_length,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++// --- First pass tests ---++fn test_first_pass_internal<T: ArrayElement + Float + Debug + Display>(+ input: &FirstPassInput<T>,+ expected: &FirstPassOutput,+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-2+ } else {+ 1e-5+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_first_pass_output::<T, B>(input);+ let msg = format!(+ "AttentionTwoPass1 failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={}, mask={})",+ std::any::type_name::<B>(),+ input.num_heads,+ input.sequence_length,+ input.suffix_length,+ input.head_dim,+ input.do_causal,+ input.has_mask,+ );+ assert_eq_float::<f32>(&expected.partials, &output.partials, eps, &format!("{msg} [partials]"));+ assert_eq_float::<f32>(&expected.sums, &output.sums, eps, &format!("{msg} [sums]"));+ assert_eq_float::<f32>(&expected.maxs, &output.maxs, eps, &format!("{msg} [maxs]"));+ });+}++fn test_first_pass_basic<T: ArrayElement + Float + Debug + Display>() {+ let input = get_first_pass_input::<T>(4, 4, 8, 1, 64, false, false);+ let expected = get_first_pass_output::<T, Cpu>(&input);+ test_first_pass_internal(&input, &expected);++ let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, false, false);+ let expected = get_first_pass_output::<T, Cpu>(&input);+ test_first_pass_internal(&input, &expected);+}++fn test_first_pass_causal<T: ArrayElement + Float + Debug + Display>() {+ let input = get_first_pass_input::<T>(4, 4, 16, 1, 64, true, false);+ let expected = get_first_pass_output::<T, Cpu>(&input);+ test_first_pass_internal(&input, &expected);++ let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, true, false);+ let expected = get_first_pass_output::<T, Cpu>(&input);+ test_first_pass_internal(&input, &expected);+}++fn test_first_pass_gqa<T: ArrayElement + Float + Debug + Display>() {+ let input = get_first_pass_input::<T>(8, 2, 8, 1, 64, false, false);+ let expected = get_first_pass_output::<T, Cpu>(&input);+ test_first_pass_internal(&input, &expected);++ let input = get_first_pass_input::<T>(8, 2, 8, 4, 64, true, false);+ let expected = get_first_pass_output::<T, Cpu>(&input);+ test_first_pass_internal(&input, &expected);+}++fn test_first_pass_mask<T: ArrayElement + Float + Debug + Display>() {+ let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, false, true);+ let expected = get_first_pass_output::<T, Cpu>(&input);+ test_first_pass_internal(&input, &expected);++ let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, true, true);+ let expected = get_first_pass_output::<T, Cpu>(&input);+ test_first_pass_internal(&input, &expected);+}++fn test_first_pass_head_dim_128<T: ArrayElement + Float + Debug + Display>() {+ let input = get_first_pass_input::<T>(4, 4, 8, 2, 128, true, false);+ let expected = get_first_pass_output::<T, Cpu>(&input);+ test_first_pass_internal(&input, &expected);+}++// --- Second pass tests ---++fn test_second_pass_internal<T: ArrayElement + Float + Debug + Display>(+ input: &SecondPassInput,+ expected: &[T],+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-2+ } else {+ 1e-5+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_second_pass_output::<T, B>(input);+ let msg = format!(+ "AttentionTwoPass2 failed (backend={}, heads={}, suffix={}, head_dim={})",+ std::any::type_name::<B>(),+ input.num_heads,+ input.suffix_length,+ input.head_dim,+ );+ assert_eq_float::<T>(expected, &output, eps, &msg);+ });+}++fn test_second_pass_basic<T: ArrayElement + Float + Debug + Display>() {+ let input = get_second_pass_input(4, 1, 64);+ let expected = get_second_pass_output::<T, Cpu>(&input);+ test_second_pass_internal::<T>(&input, &expected);++ let input = get_second_pass_input(4, 4, 64);+ let expected = get_second_pass_output::<T, Cpu>(&input);+ test_second_pass_internal::<T>(&input, &expected);+}++fn test_second_pass_head_dim_128<T: ArrayElement + Float + Debug + Display>() {+ let input = get_second_pass_input(4, 2, 128);+ let expected = get_second_pass_output::<T, Cpu>(&input);+ test_second_pass_internal::<T>(&input, &expected);+}++// --- First pass test entries ---++#[test]+fn test_first_pass_basic_f32() {+ test_first_pass_basic::<f32>();+}++#[test]+fn test_first_pass_basic_f16() {+ test_first_pass_basic::<f16>();+}++#[test]+fn test_first_pass_basic_bf16() {+ test_first_pass_basic::<bf16>();+}++#[test]+fn test_first_pass_causal_f32() {+ test_first_pass_causal::<f32>();+}++#[test]+fn test_first_pass_causal_f16() {+ test_first_pass_causal::<f16>();+}++#[test]+fn test_first_pass_causal_bf16() {+ test_first_pass_causal::<bf16>();+}++#[test]+fn test_first_pass_gqa_f32() {+ test_first_pass_gqa::<f32>();+}++#[test]+fn test_first_pass_gqa_f16() {+ test_first_pass_gqa::<f16>();+}++#[test]+fn test_first_pass_gqa_bf16() {+ test_first_pass_gqa::<bf16>();+}++#[test]+fn test_first_pass_mask_f32() {+ test_first_pass_mask::<f32>();+}++#[test]+fn test_first_pass_mask_f16() {+ test_first_pass_mask::<f16>();+}++#[test]+fn test_first_pass_mask_bf16() {+ test_first_pass_mask::<bf16>();+}++#[test]+fn test_first_pass_head_dim_128_f32() {+ test_first_pass_head_dim_128::<f32>();+}++#[test]+fn test_first_pass_head_dim_128_f16() {+ test_first_pass_head_dim_128::<f16>();+}++#[test]+fn test_first_pass_head_dim_128_bf16() {+ test_first_pass_head_dim_128::<bf16>();+}++// --- Second pass test entries ---++#[test]+fn test_second_pass_basic_f32() {+ test_second_pass_basic::<f32>();+}++#[test]+fn test_second_pass_basic_f16() {+ test_second_pass_basic::<f16>();+}++#[test]+fn test_second_pass_basic_bf16() {+ test_second_pass_basic::<bf16>();+}++#[test]+fn test_second_pass_head_dim_128_f32() {+ test_second_pass_head_dim_128::<f32>();+}++#[test]+fn test_second_pass_head_dim_128_f16() {+ test_second_pass_head_dim_128::<f16>();+}++#[test]+fn test_second_pass_head_dim_128_bf16() {+ test_second_pass_head_dim_128::<bf16>();+}diff --git a/crates/uzu/tests/unit/kernel/attention/mod.rs b/crates/uzu/tests/unit/kernel/attention/mod.rsindex 7f8328e4b..661eb7fb1 100644--- a/crates/uzu/tests/unit/kernel/attention/mod.rs+++ b/crates/uzu/tests/unit/kernel/attention/mod.rs@@ -1,4 +1,5 @@mod attention_gemm_test;mod attention_single_pass_test;+mod attention_two_pass_test;mod attention_update_kv_cache_test;mod sigmoid_gate_test;diff --git a/crates/uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs b/crates/uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rsnew file mode 100644index 000000000..d3cb57570--- /dev/null+++ b/crates/uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs@@ -0,0 +1,229 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::EmbeddingRowsSumKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ token_indices: Box<[u32]>,+ weights: Box<[T]>,+ num_rows: u32,+ total_rows: u32,+ model_dim: u32,+ codebook_stride: u32,+}++fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let num_rows = 4u32;+ let codebook_size = 8u32;+ let codebook_stride = codebook_size;+ let total_rows = num_rows * codebook_size;+ let model_dim = 513u32;++ let token_indices: Box<[u32]> = Box::new([3, 7, 1, 5]);+ let weights: Vec<T> = (0..total_rows as usize * model_dim as usize)+ .map(|i| T::from((i as f32 * 0.1).sin() * 10.0).unwrap())+ .collect();++ let input = Input {+ token_indices,+ weights: weights.into_boxed_slice(),+ num_rows,+ total_rows,+ model_dim,+ codebook_stride,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_test_data_edge<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ // Single row, small dim+ let num_rows = 1u32;+ let codebook_stride = 4u32;+ let total_rows = 4u32;+ let model_dim = 4u32;++ let token_indices: Box<[u32]> = Box::new([2]);+ let weights: Vec<T> =+ (0..total_rows as usize * model_dim as usize).map(|i| T::from(i as f32 + 1.0).unwrap()).collect();++ let input = Input {+ token_indices,+ weights: weights.into_boxed_slice(),+ num_rows,+ total_rows,+ model_dim,+ codebook_stride,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_test_data_oob<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ // Token index causes row to exceed total_rows, should be skipped+ let num_rows = 2u32;+ let codebook_stride = 4u32;+ let total_rows = 4u32;+ let model_dim = 4u32;++ // row_idx=0: row = 0*4+1 = 1 (valid)+ // row_idx=1: row = 1*4+3 = 7 (out of bounds, total_rows=4)+ let token_indices: Box<[u32]> = Box::new([1, 3]);+ let weights: Vec<T> =+ (0..total_rows as usize * model_dim as usize).map(|i| T::from(i as f32 + 1.0).unwrap()).collect();++ let input = Input {+ token_indices,+ weights: weights.into_boxed_slice(),+ num_rows,+ total_rows,+ model_dim,+ codebook_stride,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::EmbeddingRowsSumKernel::new(&context, T::data_type())+ .expect("Failed to create EmbeddingRowsSumKernel");++ let token_indices_array = context.create_array_from(&[input.num_rows as usize], &input.token_indices, "");+ let weights_array =+ context.create_array_from(&[input.total_rows as usize, input.model_dim as usize], &input.weights, "");+ let output_array = context.create_array_uninitialized(&[input.model_dim as usize], T::data_type(), "");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ token_indices_array.buffer().borrow().deref(),+ weights_array.buffer().borrow().deref(),+ output_array.buffer().borrow_mut().deref_mut(),+ input.num_rows,+ input.total_rows,+ input.model_dim,+ input.codebook_stride,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++fn test_basic<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.1f32+ } else {+ 1e-4+ };+ let (input, expected) = get_test_data_basic::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ assert_eq_float::<T>(+ &expected,+ &output,+ eps,+ &format!("EmbeddingRowsSum basic test failed for backend {}", std::any::type_name::<B>()),+ );+ });+}++fn test_edge<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.05f32+ } else {+ 1e-5+ };+ let (input, expected) = get_test_data_edge::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ assert_eq_float::<T>(+ &expected,+ &output,+ eps,+ &format!("EmbeddingRowsSum edge test failed for backend {}", std::any::type_name::<B>()),+ );+ });+}++fn test_oob<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.05f32+ } else {+ 1e-5+ };+ let (input, expected) = get_test_data_oob::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ assert_eq_float::<T>(+ &expected,+ &output,+ eps,+ &format!("EmbeddingRowsSum oob test failed for backend {}", std::any::type_name::<B>()),+ );+ });+}++// basic tests+#[test]+fn test_basic_f32() {+ test_basic::<f32>();+}++#[test]+fn test_basic_f16() {+ test_basic::<f16>();+}++#[test]+fn test_basic_bf16() {+ test_basic::<bf16>();+}++// edge tests+#[test]+fn test_edge_f32() {+ test_edge::<f32>();+}++#[test]+fn test_edge_f16() {+ test_edge::<f16>();+}++#[test]+fn test_edge_bf16() {+ test_edge::<bf16>();+}++// out-of-bounds tests+#[test]+fn test_oob_f32() {+ test_oob::<f32>();+}++#[test]+fn test_oob_f16() {+ test_oob::<f16>();+}++#[test]+fn test_oob_bf16() {+ test_oob::<bf16>();+}diff --git a/crates/uzu/tests/unit/kernel/full_precision_embedding_test.rs b/crates/uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rssimilarity index 100%rename from crates/uzu/tests/unit/kernel/full_precision_embedding_test.rsrename to crates/uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rsdiff --git a/crates/uzu/tests/unit/kernel/embedding/mod.rs b/crates/uzu/tests/unit/kernel/embedding/mod.rsnew file mode 100644index 000000000..1f2dc0e7d--- /dev/null+++ b/crates/uzu/tests/unit/kernel/embedding/mod.rs@@ -0,0 +1,3 @@+mod embedding_rows_sum_test;+mod full_precision_embedding_test;+mod quant_embedding_test;diff --git a/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs b/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rsnew file mode 100644index 000000000..40c667400--- /dev/null+++ b/crates/uzu/tests/unit/kernel/embedding/quant_embedding_test.rs@@ -0,0 +1,246 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{+ Backend, Context, Encoder, Kernels, gpu_types::QuantizationMode, kernel::QuantizedEmbeddingLookupKernel,+ },+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ token_ids: Box<[u64]>,+ weights: Box<[u8]>,+ scales: Box<[T]>,+ biases: Box<[T]>,+ batch_size: u32,+ vocab_size: u32,+ model_dim: u32,+ input_scale: f32,+ group_size: u32,+ quant_mode: QuantizationMode,+}++fn get_test_data<T: ArrayElement + Float>(quant_mode: QuantizationMode) -> (Input<T>, Vec<T>) {+ let batch_size = 3u32;+ let vocab_size = 8u32;+ let model_dim = 64u32;+ let group_size = 32u32;+ let input_scale = 1.5f32;++ let token_ids: Box<[u64]> = Box::new([2, 5, 0]);++ let packing_divisor = quant_mode.packing_divisor() as u32;+ let weights_stride = model_dim / packing_divisor;+ let weights: Vec<u8> = (0..vocab_size as usize * weights_stride as usize)+ .map(|i| ((i % 16) as u8) | ((((i + 3) % 16) as u8) << 4))+ .collect();++ let num_groups = (model_dim + group_size - 1) / group_size;+ let scales: Vec<T> = (0..vocab_size as usize * num_groups as usize)+ .map(|i| T::from(0.5 + (i as f32 * 0.1).sin() * 0.3).unwrap())+ .collect();+ let biases: Vec<T> = (0..vocab_size as usize * num_groups as usize)+ .map(|i| T::from((i as f32 * 0.2).cos() * 0.1).unwrap())+ .collect();++ let input = Input {+ token_ids,+ weights: weights.into_boxed_slice(),+ scales: scales.into_boxed_slice(),+ biases: biases.into_boxed_slice(),+ batch_size,+ vocab_size,+ model_dim,+ input_scale,+ group_size,+ quant_mode,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_test_data_oob<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let batch_size = 2u32;+ let vocab_size = 4u32;+ let model_dim = 32u32;+ let group_size = 32u32;+ let input_scale = 1.0f32;++ // token_id 1 is valid, token_id 10 is out of bounds+ let token_ids: Box<[u64]> = Box::new([1, 10]);++ let weights: Vec<u8> = (0..vocab_size as usize * model_dim as usize).map(|i| (i % 200) as u8).collect();++ let num_groups = (model_dim + group_size - 1) / group_size;+ let scales: Vec<T> =+ (0..vocab_size as usize * num_groups as usize).map(|i| T::from(0.5 + i as f32 * 0.1).unwrap()).collect();+ let biases: Vec<T> =+ (0..vocab_size as usize * num_groups as usize).map(|i| T::from(i as f32 * 0.01).unwrap()).collect();++ let input = Input {+ token_ids,+ weights: weights.into_boxed_slice(),+ scales: scales.into_boxed_slice(),+ biases: biases.into_boxed_slice(),+ batch_size,+ vocab_size,+ model_dim,+ input_scale,+ group_size,+ quant_mode: QuantizationMode::UINT8,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(+ &context,+ T::data_type(),+ input.group_size,+ input.quant_mode.to_u32(),+ )+ .expect("Failed to create QuantizedEmbeddingLookupKernel");++ let packing_divisor = input.quant_mode.packing_divisor();+ let weights_stride = input.model_dim as usize / packing_divisor;+ let num_groups = (input.model_dim as usize + input.group_size as usize - 1) / input.group_size as usize;++ let token_ids_array = context.create_array_from(&[input.batch_size as usize], &input.token_ids, "");+ let weights_array = context.create_array_from(&[input.vocab_size as usize, weights_stride], &input.weights, "");+ let scales_array = context.create_array_from(&[input.vocab_size as usize, num_groups], &input.scales, "");+ let biases_array = context.create_array_from(&[input.vocab_size as usize, num_groups], &input.biases, "");+ let output_array =+ context.create_array_uninitialized(&[input.batch_size as usize, input.model_dim as usize], T::data_type(), "");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ token_ids_array.buffer().borrow().deref(),+ weights_array.buffer().borrow().deref(),+ scales_array.buffer().borrow().deref(),+ biases_array.buffer().borrow().deref(),+ output_array.buffer().borrow_mut().deref_mut(),+ input.batch_size,+ input.vocab_size,+ input.model_dim,+ input.input_scale,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++fn test_quant_mode<T: ArrayElement + Float + Debug + Display>(quant_mode: QuantizationMode) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.5f32+ } else {+ 1e-4+ };+ let (input, expected) = get_test_data::<T>(quant_mode);+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ assert_eq_float::<T>(+ &expected,+ &output,+ eps,+ &format!("QuantizedEmbeddingLookup {quant_mode:?} test failed for backend {}", std::any::type_name::<B>()),+ );+ });+}++fn test_oob<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.5f32+ } else {+ 1e-4+ };+ let (input, expected) = get_test_data_oob::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ assert_eq_float::<T>(+ &expected,+ &output,+ eps,+ &format!("QuantizedEmbeddingLookup OOB test failed for backend {}", std::any::type_name::<B>()),+ );+ });+}++// UINT4 tests+#[test]+fn test_uint4_f32() {+ test_quant_mode::<f32>(QuantizationMode::UINT4);+}++#[test]+fn test_uint4_f16() {+ test_quant_mode::<f16>(QuantizationMode::UINT4);+}++#[test]+fn test_uint4_bf16() {+ test_quant_mode::<bf16>(QuantizationMode::UINT4);+}++// INT8 tests+#[test]+fn test_int8_f32() {+ test_quant_mode::<f32>(QuantizationMode::INT8);+}++#[test]+fn test_int8_f16() {+ test_quant_mode::<f16>(QuantizationMode::INT8);+}++#[test]+fn test_int8_bf16() {+ test_quant_mode::<bf16>(QuantizationMode::INT8);+}++// UINT8 tests+#[test]+fn test_uint8_f32() {+ test_quant_mode::<f32>(QuantizationMode::UINT8);+}++#[test]+fn test_uint8_f16() {+ test_quant_mode::<f16>(QuantizationMode::UINT8);+}++#[test]+fn test_uint8_bf16() {+ test_quant_mode::<bf16>(QuantizationMode::UINT8);+}++// OOB tests+#[test]+fn test_oob_f32() {+ test_oob::<f32>();+}++#[test]+fn test_oob_f16() {+ test_oob::<f16>();+}++#[test]+fn test_oob_bf16() {+ test_oob::<bf16>();+}diff --git a/crates/uzu/tests/unit/kernel/kv_cache_update_test.rs b/crates/uzu/tests/unit/kernel/kv_cache_update_test.rsnew file mode 100644index 000000000..670344b39--- /dev/null+++ b/crates/uzu/tests/unit/kernel/kv_cache_update_test.rs@@ -0,0 +1,282 @@+use std::{+ fmt::{Debug, Display},+ ops::DerefMut,+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement,+ backends::{+ common::{Backend, Context, Encoder, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ keys: Box<[T]>,+ values: Box<[T]>,+ swaps: Vec<Swap>,+ num_heads: u32,+ max_sequence_length: u32,+ head_dim: u32,+}++fn make_data<T: ArrayElement + Float>(+ num_heads: usize,+ max_sequence_length: usize,+ head_dim: usize,+) -> (Vec<T>, Vec<T>) {+ let total = num_heads * max_sequence_length * head_dim;+ let mut keys = Vec::with_capacity(total);+ let mut values = Vec::with_capacity(total);+ for i in 0..total {+ keys.push(T::from(1.0 + i as f32 * 0.1).unwrap());+ values.push(T::from(100.0 + i as f32 * 0.1).unwrap());+ }+ (keys, values)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {+ let context = B::Context::new().expect("Failed to create Context");+ let kernel = <<B as Backend>::Kernels as Kernels>::KVCacheUpdateKernel::new(&context, T::data_type())+ .expect("Failed to create KVCacheUpdateKernel");++ let total = input.num_heads as usize * input.max_sequence_length as usize * input.head_dim as usize;+ let keys_array = context.create_array_from(&[total], &input.keys, "");+ let values_array = context.create_array_from(&[total], &input.values, "");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");+ kernel.encode(+ keys_array.buffer().borrow_mut().deref_mut(),+ values_array.buffer().borrow_mut().deref_mut(),+ &input.swaps,+ input.swaps.len() as u32,+ input.num_heads,+ input.max_sequence_length,+ input.head_dim,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ (keys_array.as_slice().to_vec(), values_array.as_slice().to_vec())+}++/// Single swap between two different positions.+fn get_test_data_single_swap<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {+ let num_heads = 2usize;+ let max_sequence_length = 4usize;+ let head_dim = 3usize;++ let (keys, values) = make_data::<T>(num_heads, max_sequence_length, head_dim);++ let swaps = vec![Swap {+ source: 0,+ destination: 2,+ }];++ let input = Input {+ keys: keys.into_boxed_slice(),+ values: values.into_boxed_slice(),+ swaps: swaps.clone(),+ num_heads: num_heads as u32,+ max_sequence_length: max_sequence_length as u32,+ head_dim: head_dim as u32,+ };++ let (expected_keys, expected_values) = get_output::<T, Cpu>(&input);+ (input, expected_keys, expected_values)+}++/// Multiple swaps.+fn get_test_data_multi_swap<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {+ let num_heads = 2usize;+ let max_sequence_length = 8usize;+ let head_dim = 4usize;++ let (keys, values) = make_data::<T>(num_heads, max_sequence_length, head_dim);++ let swaps = vec![+ Swap {+ source: 1,+ destination: 3,+ },+ Swap {+ source: 0,+ destination: 5,+ },+ ];++ let input = Input {+ keys: keys.into_boxed_slice(),+ values: values.into_boxed_slice(),+ swaps: swaps.clone(),+ num_heads: num_heads as u32,+ max_sequence_length: max_sequence_length as u32,+ head_dim: head_dim as u32,+ };++ let (expected_keys, expected_values) = get_output::<T, Cpu>(&input);+ (input, expected_keys, expected_values)+}++/// No swaps — data should remain unchanged.+fn get_test_data_no_swap<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {+ let num_heads = 2usize;+ let max_sequence_length = 4usize;+ let head_dim = 3usize;++ let (keys, values) = make_data::<T>(num_heads, max_sequence_length, head_dim);++ let expected_keys = keys.clone();+ let expected_values = values.clone();++ let input = Input {+ keys: keys.into_boxed_slice(),+ values: values.into_boxed_slice(),+ swaps: vec![],+ num_heads: num_heads as u32,+ max_sequence_length: max_sequence_length as u32,+ head_dim: head_dim as u32,+ };++ (input, expected_keys, expected_values)+}++/// Larger dimensions to exercise more threads.+fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec<T>) {+ let num_heads = 8usize;+ let max_sequence_length = 64usize;+ let head_dim = 128usize;++ let (keys, values) = make_data::<T>(num_heads, max_sequence_length, head_dim);++ let swaps = vec![+ Swap {+ source: 0,+ destination: 63,+ },+ Swap {+ source: 10,+ destination: 30,+ },+ Swap {+ source: 5,+ destination: 20,+ },+ ];++ let input = Input {+ keys: keys.into_boxed_slice(),+ values: values.into_boxed_slice(),+ swaps: swaps.clone(),+ num_heads: num_heads as u32,+ max_sequence_length: max_sequence_length as u32,+ head_dim: head_dim as u32,+ };++ let (expected_keys, expected_values) = get_output::<T, Cpu>(&input);+ (input, expected_keys, expected_values)+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ input: &Input<T>,+ expected_keys: &[T],+ expected_values: &[T],+ test_name: &str,+) {+ let eps = 1e-5;+ for_each_non_cpu_backend!(|B| {+ let (actual_keys, actual_values) = get_output::<T, B>(input);+ let msg = format!("KVCacheUpdate {} keys failed with backend={}", test_name, std::any::type_name::<B>(),);+ assert_eq_float::<T>(expected_keys, &actual_keys, eps, &msg);+ let msg = format!("KVCacheUpdate {} values failed with backend={}", test_name, std::any::type_name::<B>(),);+ assert_eq_float::<T>(expected_values, &actual_values, eps, &msg);+ });+}++fn test_single_swap_internal<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected_keys, expected_values) = get_test_data_single_swap::<T>();+ test_internal::<T>(&input, &expected_keys, &expected_values, "single_swap");+}++fn test_multi_swap_internal<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected_keys, expected_values) = get_test_data_multi_swap::<T>();+ test_internal::<T>(&input, &expected_keys, &expected_values, "multi_swap");+}++fn test_no_swap_internal<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected_keys, expected_values) = get_test_data_no_swap::<T>();+ test_internal::<T>(&input, &expected_keys, &expected_values, "no_swap");+}++fn test_large_internal<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected_keys, expected_values) = get_test_data_large::<T>();+ test_internal::<T>(&input, &expected_keys, &expected_values, "large");+}++// Single swap tests+#[test]+fn test_single_swap_f32() {+ test_single_swap_internal::<f32>();+}++#[test]+fn test_single_swap_f16() {+ test_single_swap_internal::<f16>();+}++#[test]+fn test_single_swap_bf16() {+ test_single_swap_internal::<bf16>();+}++// Multi swap tests+#[test]+fn test_multi_swap_f32() {+ test_multi_swap_internal::<f32>();+}++#[test]+fn test_multi_swap_f16() {+ test_multi_swap_internal::<f16>();+}++#[test]+fn test_multi_swap_bf16() {+ test_multi_swap_internal::<bf16>();+}++// No swap tests+#[test]+fn test_no_swap_f32() {+ test_no_swap_internal::<f32>();+}++#[test]+fn test_no_swap_f16() {+ test_no_swap_internal::<f16>();+}++#[test]+fn test_no_swap_bf16() {+ test_no_swap_internal::<bf16>();+}++// Large dimension tests+#[test]+fn test_large_f32() {+ test_large_internal::<f32>();+}++#[test]+fn test_large_f16() {+ test_large_internal::<f16>();+}++#[test]+fn test_large_bf16() {+ test_large_internal::<bf16>();+}diff --git a/crates/uzu/tests/unit/kernel/layer_norm_test.rs b/crates/uzu/tests/unit/kernel/layer_norm_test.rsnew file mode 100644index 000000000..f0af5edfb--- /dev/null+++ b/crates/uzu/tests/unit/kernel/layer_norm_test.rs@@ -0,0 +1,314 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::LayerNormKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++static BOOL_ALL: &[bool] = &[true, false];+static BOOL_FALSE: &[bool] = &[false];++struct Input<IN: ArrayElement + Float, SC: ArrayElement + Float, OUT: ArrayElement + Float> {+ input: Box<[IN]>,+ scales: Box<[SC]>,+ output: Box<[OUT]>,+ batch_size: u32,+ model_dim: u32,+ epsilon: f32,+ scale_offset: f32,+ full_layer: u32,+ in_place: bool,+}++fn get_test_data_basic<+ IN: ArrayElement + Float,+ SC: ArrayElement + Float,+ OUT: ArrayElement + Float,+ ACC: ArrayElement + Float,+>(+ full_layer: u32,+ in_place: bool,+) -> (Input<IN, SC, OUT>, Vec<OUT>) {+ let batch_size = 2usize;+ let model_dim = 4096usize;+ let epsilon = 1e-5f32;+ let scale_offset = 0.123f32;++ let total = batch_size * model_dim;+ let mut input_data: Vec<IN> = vec![IN::zero(); total];+ let mut output_data: Vec<OUT> = vec![OUT::zero(); total];+ let scales_data: Vec<SC> = (0..model_dim).map(|i| SC::from(1.0 + 0.01 * (i as f32).sin()).unwrap()).collect();++ for i in 0..total {+ let val = 2.0 * f32::sin(i as f32 * 0.01) + 0.1 * (i as f32 / total as f32);+ input_data[i] = IN::from(val).unwrap();+ output_data[i] = OUT::from(val).unwrap();+ }++ let input = Input::<IN, SC, OUT> {+ input: input_data.into_boxed_slice(),+ scales: scales_data.into_boxed_slice(),+ output: output_data.into_boxed_slice(),+ batch_size: batch_size as u32,+ model_dim: model_dim as u32,+ epsilon,+ scale_offset,+ full_layer,+ in_place,+ };++ let output = get_output::<Cpu, IN, SC, OUT, ACC>(&input);+ (input, output)+}++fn get_test_data_edge<+ IN: ArrayElement + Float,+ SC: ArrayElement + Float,+ OUT: ArrayElement + Float,+ ACC: ArrayElement + Float,+>(+ full_layer: u32,+ in_place: bool,+) -> (Input<IN, SC, OUT>, Vec<OUT>) {+ let batch_size = 1usize;+ let model_dim = 4usize;+ let epsilon = 1e-6f32;+ let scale_offset = 0.0f32;++ let input_data_f32: Vec<f32> = vec![1.0, -2.0, 3.0, -4.0];+ let scales_data_f32: Vec<f32> = vec![2.0, 0.5, 1.5, 0.8];+ let input_data: Vec<IN> = input_data_f32.iter().map(|&x| IN::from(x).unwrap()).collect();+ let output_data: Vec<OUT> = input_data_f32.iter().map(|&x| OUT::from(x).unwrap()).collect();+ let scales_data: Vec<SC> = scales_data_f32.iter().map(|&x| SC::from(x).unwrap()).collect();++ let input = Input::<IN, SC, OUT> {+ input: input_data.into_boxed_slice(),+ scales: scales_data.into_boxed_slice(),+ output: output_data.into_boxed_slice(),+ batch_size: batch_size as u32,+ model_dim: model_dim as u32,+ epsilon,+ scale_offset,+ full_layer,+ in_place,+ };++ let output = get_output::<Cpu, IN, SC, OUT, ACC>(&input);+ (input, output)+}++fn get_output<+ B: Backend,+ IN: ArrayElement + Float,+ SC: ArrayElement + Float,+ OUT: ArrayElement + Float,+ ACC: ArrayElement + Float,+>(+ input: &Input<IN, SC, OUT>+) -> Vec<OUT> {+ let context = B::Context::new().expect("Failed to create Context");+ let kernel = <<B as Backend>::Kernels as Kernels>::LayerNormKernel::new(+ &context,+ IN::data_type(),+ SC::data_type(),+ OUT::data_type(),+ ACC::data_type(),+ input.in_place,+ )+ .expect("Failed to create LayerNormKernel");++ let input_size = input.input.len();+ let input_array = context.create_array_from(&[input_size], &input.input, "");+ let input_array_buffer_rc = input_array.buffer();+ let input_array_borrow = input_array_buffer_rc.borrow();+ let input_array_deref = input_array_borrow.deref();+ let input_buffer = (!input.in_place).then(|| input_array_deref);++ let scales_array = context.create_array_from(&[input.scales.len()], &input.scales, "");+ let output_array = match input.in_place {+ true => context.create_array_from(&[input_size], &input.output, ""),+ false => context.create_array_uninitialized(&[input_size], OUT::data_type(), ""),+ };++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ input_buffer,+ scales_array.buffer().borrow().deref(),+ output_array.buffer().borrow_mut().deref_mut(),+ input.batch_size,+ input.model_dim,+ input.epsilon,+ input.scale_offset,+ input.full_layer,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++fn test_internal<+ IN: ArrayElement + Float,+ SC: ArrayElement + Float,+ OUT: ArrayElement + Float + Debug + Display,+ ACC: ArrayElement + Float,+>(+ input: &Input<IN, SC, OUT>,+ expected: &[OUT],+) {+ let eps = if matches!(IN::data_type(), DataType::F16 | DataType::BF16)+ || matches!(SC::data_type(), DataType::F16 | DataType::BF16)+ || matches!(OUT::data_type(), DataType::F16 | DataType::BF16)+ {+ 0.012f32+ } else {+ 1e-5+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_output::<B, IN, SC, OUT, ACC>(input);+ let msg = format!(+ "LayerNorm kernel test failed with backend={}, full_layer={}, in_place={}",+ std::any::type_name::<B>(),+ input.full_layer,+ input.in_place,+ );+ assert_eq_float::<OUT>(expected, &output, eps, &msg);+ });+}++fn test_basic<+ IN: ArrayElement + Float,+ SC: ArrayElement + Float,+ OUT: ArrayElement + Float + Debug + Display,+ ACC: ArrayElement + Float,+>() {+ let in_place_values: &[bool] = if IN::data_type() == OUT::data_type() {+ &BOOL_ALL+ } else {+ &BOOL_FALSE+ };++ for in_place in in_place_values {+ for full_layer in [1u32, 0u32] {+ let (input, expected) = get_test_data_basic::<IN, SC, OUT, ACC>(full_layer, *in_place);+ test_internal::<IN, SC, OUT, ACC>(&input, expected.as_slice())+ }+ }+}++fn test_edge<+ IN: ArrayElement + Float,+ SC: ArrayElement + Float,+ OUT: ArrayElement + Float + Debug + Display,+ ACC: ArrayElement + Float,+>() {+ let in_place_values: &[bool] = if IN::data_type() == OUT::data_type() {+ &BOOL_ALL+ } else {+ &BOOL_FALSE+ };++ for in_place in in_place_values {+ let (input, expected) = get_test_data_edge::<IN, SC, OUT, ACC>(0, *in_place);+ test_internal::<IN, SC, OUT, ACC>(&input, expected.as_slice())+ }+}++// basic tests - AccumT f32+#[test]+fn test_f32_f32_f32_f32() {+ test_basic::<f32, f32, f32, f32>();+}++#[test]+fn test_f16_f32_f32_f32() {+ test_basic::<f16, f32, f32, f32>();+}++#[test]+fn test_f32_f16_f32_f32() {+ test_basic::<f32, f16, f32, f32>();+}++#[test]+fn test_f32_f32_f16_f32() {+ test_basic::<f32, f32, f16, f32>();+}++#[test]+fn test_f16_f16_f32_f32() {+ test_basic::<f16, f16, f32, f32>();+}++#[test]+fn test_f16_f32_f16_f32() {+ test_basic::<f16, f32, f16, f32>();+}++#[test]+fn test_f32_f16_f16_f32() {+ test_basic::<f32, f16, f16, f32>();+}++#[test]+fn test_f16_f16_f16_f32() {+ test_basic::<f16, f16, f16, f32>();+}++// basic tests - AccumT f32 + bf16+#[test]+fn test_bf16_f32_f32_f32() {+ test_basic::<bf16, f32, f32, f32>();+}++#[test]+fn test_f32_bf16_f32_f32() {+ test_basic::<f32, bf16, f32, f32>();+}++#[test]+fn test_f32_f32_bf16_f32() {+ test_basic::<f32, f32, bf16, f32>();+}++#[test]+fn test_bf16_bf16_f32_f32() {+ test_basic::<bf16, bf16, f32, f32>();+}++#[test]+fn test_bf16_f32_bf16_f32() {+ test_basic::<bf16, f32, bf16, f32>();+}++#[test]+fn test_bf16_bf16_bf16_f32() {+ test_basic::<bf16, bf16, bf16, f32>();+}++// edge tests+#[test]+fn test_edge_f32_f32_f32_f32() {+ test_edge::<f32, f32, f32, f32>();+}++#[test]+fn test_edge_f32_f16_f16_f32() {+ test_edge::<f32, f16, f16, f32>();+}++#[test]+fn test_edge_f16_f16_f16_f32() {+ test_edge::<f16, f16, f16, f32>();+}diff --git a/crates/uzu/tests/unit/kernel/mask_update_test.rs b/crates/uzu/tests/unit/kernel/mask_update_test.rsnew file mode 100644index 000000000..f4113d5ab--- /dev/null+++ b/crates/uzu/tests/unit/kernel/mask_update_test.rs@@ -0,0 +1,240 @@+use std::{+ fmt::{Debug, Display},+ ops::DerefMut,+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement,+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::MaskUpdateKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ mask: Box<[T]>,+ unmask_col: i32,+ mask_col: i32,+}++fn make_mask<T: ArrayElement + Float>(size: usize) -> Vec<T> {+ let mut mask = Vec::with_capacity(size);+ for i in 0..size {+ mask.push(T::from(i as f32 * 0.5).unwrap());+ }+ mask+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");+ let kernel = <<B as Backend>::Kernels as Kernels>::MaskUpdateKernel::new(&context, T::data_type())+ .expect("Failed to create MaskUpdateKernel");++ let len = input.mask.len();+ let mask_array = context.create_array_from(&[len], &input.mask, "");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");+ kernel.encode(mask_array.buffer().borrow_mut().deref_mut(), input.unmask_col, input.mask_col, &mut encoder);+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ mask_array.as_slice().to_vec()+}++/// Unmask one column and mask another.+fn get_test_data_unmask_and_mask<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let size = 8;+ let mask = make_mask::<T>(size);++ let input = Input {+ mask: mask.into_boxed_slice(),+ unmask_col: 2,+ mask_col: 5,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++/// Only unmask (mask_col negative).+fn get_test_data_unmask_only<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let size = 8;+ let mask = make_mask::<T>(size);++ let input = Input {+ mask: mask.into_boxed_slice(),+ unmask_col: 3,+ mask_col: -1,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++/// Only mask (unmask_col negative).+fn get_test_data_mask_only<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let size = 8;+ let mask = make_mask::<T>(size);++ let input = Input {+ mask: mask.into_boxed_slice(),+ unmask_col: -1,+ mask_col: 4,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++/// Both columns negative — data should remain unchanged.+fn get_test_data_no_op<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let size = 8;+ let mask = make_mask::<T>(size);++ let expected = mask.clone();++ let input = Input {+ mask: mask.into_boxed_slice(),+ unmask_col: -1,+ mask_col: -1,+ };++ (input, expected)+}++/// Larger mask to exercise more threads.+fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let size = 1024;+ let mask = make_mask::<T>(size);++ let input = Input {+ mask: mask.into_boxed_slice(),+ unmask_col: 100,+ mask_col: 900,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ input: &Input<T>,+ expected: &[T],+ test_name: &str,+) {+ let eps = 1e-5;+ let expected_norm = expected.to_vec();+ for_each_non_cpu_backend!(|B| {+ let actual = get_output::<T, B>(input);+ let msg = format!("MaskUpdate {} failed with backend={}", test_name, std::any::type_name::<B>());+ assert_eq_float::<T>(&expected_norm, &actual, eps, &msg);+ });+}++fn test_unmask_and_mask_internal<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data_unmask_and_mask::<T>();+ test_internal::<T>(&input, &expected, "unmask_and_mask");+}++fn test_unmask_only_internal<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data_unmask_only::<T>();+ test_internal::<T>(&input, &expected, "unmask_only");+}++fn test_mask_only_internal<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data_mask_only::<T>();+ test_internal::<T>(&input, &expected, "mask_only");+}++fn test_no_op_internal<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data_no_op::<T>();+ test_internal::<T>(&input, &expected, "no_op");+}++fn test_large_internal<T: ArrayElement + Float + Debug + Display>() {+ let (input, expected) = get_test_data_large::<T>();+ test_internal::<T>(&input, &expected, "large");+}++// Unmask and mask tests+#[test]+fn test_unmask_and_mask_f32() {+ test_unmask_and_mask_internal::<f32>();+}++#[test]+fn test_unmask_and_mask_f16() {+ test_unmask_and_mask_internal::<f16>();+}++#[test]+fn test_unmask_and_mask_bf16() {+ test_unmask_and_mask_internal::<bf16>();+}++// Unmask only tests+#[test]+fn test_unmask_only_f32() {+ test_unmask_only_internal::<f32>();+}++#[test]+fn test_unmask_only_f16() {+ test_unmask_only_internal::<f16>();+}++#[test]+fn test_unmask_only_bf16() {+ test_unmask_only_internal::<bf16>();+}++// Mask only tests+#[test]+fn test_mask_only_f32() {+ test_mask_only_internal::<f32>();+}++#[test]+fn test_mask_only_f16() {+ test_mask_only_internal::<f16>();+}++#[test]+fn test_mask_only_bf16() {+ test_mask_only_internal::<bf16>();+}++// No-op tests+#[test]+fn test_no_op_f32() {+ test_no_op_internal::<f32>();+}++#[test]+fn test_no_op_f16() {+ test_no_op_internal::<f16>();+}++#[test]+fn test_no_op_bf16() {+ test_no_op_internal::<bf16>();+}++// Large dimension tests+#[test]+fn test_large_f32() {+ test_large_internal::<f32>();+}++#[test]+fn test_large_f16() {+ test_large_internal::<f16>();+}++#[test]+fn test_large_bf16() {+ test_large_internal::<bf16>();+}diff --git a/crates/uzu/tests/unit/kernel/mod.rs b/crates/uzu/tests/unit/kernel/mod.rsindex ad863e541..34e21cfb0 100644--- a/crates/uzu/tests/unit/kernel/mod.rs+++ b/crates/uzu/tests/unit/kernel/mod.rs@@ -3,16 +3,23 @@mod common;mod activation_mul_test;+mod activation_test;mod attention;mod audio;-mod full_precision_embedding_test;+mod embedding;+mod kv_cache_update_test;+mod layer_norm_test;+mod mask_update_test;mod matmul;mod moe;+mod pooling;mod qk_norm_test;mod rms_norm_test;mod rope_test;mod sampling;+mod sigmoid_test;mod tensor_add_bias_test;+mod tensor_add_scale_test;mod tensor_add_swap_test;mod tensor_copy_test;mod token_copy_test;diff --git a/crates/uzu/tests/unit/kernel/pooling/mod.rs b/crates/uzu/tests/unit/kernel/pooling/mod.rsnew file mode 100644index 000000000..22da4b449--- /dev/null+++ b/crates/uzu/tests/unit/kernel/pooling/mod.rs@@ -0,0 +1,2 @@+mod pooling_cls_test;+mod pooling_mean_test;diff --git a/crates/uzu/tests/unit/kernel/pooling/pooling_cls_test.rs b/crates/uzu/tests/unit/kernel/pooling/pooling_cls_test.rsnew file mode 100644index 000000000..248d9c87f--- /dev/null+++ b/crates/uzu/tests/unit/kernel/pooling/pooling_cls_test.rs@@ -0,0 +1,157 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::PoolingClsKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ input: Box<[T]>,+ seq_len: u32,+ hidden_dim: u32,+ batch_size: u32,+}++fn get_test_data<T: ArrayElement + Float>(+ batch_size: u32,+ seq_len: u32,+ hidden_dim: u32,+) -> (Input<T>, Vec<T>) {+ let len = (batch_size * seq_len * hidden_dim) as usize;+ let input: Vec<T> = (0..len).map(|i| T::from((i as f32 * 0.1).sin() * 2.0).unwrap()).collect();++ let input = Input {+ input: input.into_boxed_slice(),+ seq_len,+ hidden_dim,+ batch_size,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::PoolingClsKernel::new(&context, T::data_type())+ .expect("Failed to create PoolingClsKernel");++ let input_len = (input.batch_size * input.seq_len * input.hidden_dim) as usize;+ let output_len = (input.batch_size * input.hidden_dim) as usize;++ let input_array = context.create_array_from(&[input_len], &input.input, "");+ let output_array = context.create_array_uninitialized(&[output_len], T::data_type(), "");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ input_array.buffer().borrow().deref(),+ output_array.buffer().borrow_mut().deref_mut(),+ input.seq_len,+ input.hidden_dim,+ input.batch_size,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ batch_size: u32,+ seq_len: u32,+ hidden_dim: u32,+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-3+ } else {+ 1e-5+ };++ let (input, expected) = get_test_data::<T>(batch_size, seq_len, hidden_dim);+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ let msg = format!("PoolingCls test failed for backend {}", std::any::type_name::<B>());+ assert_eq_float::<T>(&expected, &output, eps, &msg);+ });+}++// Basic tests+#[test]+fn test_basic_f32() {+ test_internal::<f32>(2, 16, 64);+}++#[test]+fn test_basic_f16() {+ test_internal::<f16>(2, 16, 64);+}++#[test]+fn test_basic_bf16() {+ test_internal::<bf16>(2, 16, 64);+}++// Single batch+#[test]+fn test_single_batch_f32() {+ test_internal::<f32>(1, 32, 128);+}++#[test]+fn test_single_batch_f16() {+ test_internal::<f16>(1, 32, 128);+}++#[test]+fn test_single_batch_bf16() {+ test_internal::<bf16>(1, 32, 128);+}++// Single token sequence+#[test]+fn test_single_token_f32() {+ test_internal::<f32>(2, 1, 64);+}++#[test]+fn test_single_token_f16() {+ test_internal::<f16>(2, 1, 64);+}++#[test]+fn test_single_token_bf16() {+ test_internal::<bf16>(2, 1, 64);+}++// Large hidden dim+#[test]+fn test_large_hidden_dim_f32() {+ test_internal::<f32>(2, 8, 512);+}++// Non-aligned hidden dim+#[test]+fn test_non_aligned_f32() {+ test_internal::<f32>(3, 10, 100);+}++#[test]+fn test_non_aligned_f16() {+ test_internal::<f16>(3, 10, 100);+}++#[test]+fn test_non_aligned_bf16() {+ test_internal::<bf16>(3, 10, 100);+}diff --git a/crates/uzu/tests/unit/kernel/pooling/pooling_mean_test.rs b/crates/uzu/tests/unit/kernel/pooling/pooling_mean_test.rsnew file mode 100644index 000000000..526045fca--- /dev/null+++ b/crates/uzu/tests/unit/kernel/pooling/pooling_mean_test.rs@@ -0,0 +1,163 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::PoolingMeanKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ input: Box<[T]>,+ seq_len: u32,+ hidden_dim: u32,+ batch_size: u32,+}++fn get_test_data<T: ArrayElement + Float>(+ batch_size: u32,+ seq_len: u32,+ hidden_dim: u32,+) -> (Input<T>, Vec<T>) {+ let len = (batch_size * seq_len * hidden_dim) as usize;+ let input: Vec<T> = (0..len).map(|i| T::from((i as f32 * 0.1).sin() * 2.0).unwrap()).collect();++ let input = Input {+ input: input.into_boxed_slice(),+ seq_len,+ hidden_dim,+ batch_size,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::PoolingMeanKernel::new(&context, T::data_type())+ .expect("Failed to create PoolingMeanKernel");++ let input_len = (input.batch_size * input.seq_len * input.hidden_dim) as usize;+ let output_len = (input.batch_size * input.hidden_dim) as usize;++ let input_array = context.create_array_from(&[input_len], &input.input, "");+ let output_array = context.create_array_uninitialized(&[output_len], T::data_type(), "");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ input_array.buffer().borrow().deref(),+ output_array.buffer().borrow_mut().deref_mut(),+ input.seq_len,+ input.hidden_dim,+ input.batch_size,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ batch_size: u32,+ seq_len: u32,+ hidden_dim: u32,+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-2+ } else {+ 1e-5+ };++ let (input, expected) = get_test_data::<T>(batch_size, seq_len, hidden_dim);+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ let msg = format!("PoolingMean test failed for backend {}", std::any::type_name::<B>());+ assert_eq_float::<T>(&expected, &output, eps, &msg);+ });+}++// Basic tests+#[test]+fn test_basic_f32() {+ test_internal::<f32>(2, 16, 64);+}++#[test]+fn test_basic_f16() {+ test_internal::<f16>(2, 16, 64);+}++#[test]+fn test_basic_bf16() {+ test_internal::<bf16>(2, 16, 64);+}++// Single batch+#[test]+fn test_single_batch_f32() {+ test_internal::<f32>(1, 32, 128);+}++#[test]+fn test_single_batch_f16() {+ test_internal::<f16>(1, 32, 128);+}++#[test]+fn test_single_batch_bf16() {+ test_internal::<bf16>(1, 32, 128);+}++// Single token sequence — mean of one element equals that element+#[test]+fn test_single_token_f32() {+ test_internal::<f32>(2, 1, 64);+}++#[test]+fn test_single_token_f16() {+ test_internal::<f16>(2, 1, 64);+}++#[test]+fn test_single_token_bf16() {+ test_internal::<bf16>(2, 1, 64);+}++// Large hidden dim+#[test]+fn test_large_hidden_dim_f32() {+ test_internal::<f32>(2, 8, 512);+}++// Long sequence — more accumulation, tests precision+#[test]+fn test_long_seq_f32() {+ test_internal::<f32>(1, 256, 64);+}++// Non-aligned hidden dim+#[test]+fn test_non_aligned_f32() {+ test_internal::<f32>(3, 10, 100);+}++#[test]+fn test_non_aligned_f16() {+ test_internal::<f16>(3, 10, 100);+}++#[test]+fn test_non_aligned_bf16() {+ test_internal::<bf16>(3, 10, 100);+}diff --git a/crates/uzu/tests/unit/kernel/sampling/bitmask_test.rs b/crates/uzu/tests/unit/kernel/sampling/bitmask_test.rsnew file mode 100644index 000000000..71de2c260--- /dev/null+++ b/crates/uzu/tests/unit/kernel/sampling/bitmask_test.rs@@ -0,0 +1,299 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::BitmaskKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ logits: Box<[T]>,+ bitmask: Box<[u32]>,+ batch_size: u32,+ vocab_size: u32,+ in_place: bool,+}++fn get_test_data_basic<T: ArrayElement + Float>(in_place: bool) -> (Input<T>, Vec<T>) {+ let batch_size = 2u32;+ let vocab_size = 128u32;++ let len = (batch_size * vocab_size) as usize;+ let logits: Vec<T> = (0..len).map(|i| T::from((i as f32 * 0.1).sin() * 2.0).unwrap()).collect();++ let bitmask_size = ((vocab_size + 31) / 32) as usize;+ let mut bitmask = vec![0u32; batch_size as usize * bitmask_size];+ // Allow every other token+ for b in 0..batch_size as usize {+ for word in 0..bitmask_size {+ bitmask[b * bitmask_size + word] = 0x55555555; // bits 0,2,4,...+ }+ }++ let input = Input {+ logits: logits.into_boxed_slice(),+ bitmask: bitmask.into_boxed_slice(),+ batch_size,+ vocab_size,+ in_place,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_test_data_all_allowed<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let batch_size = 1u32;+ let vocab_size = 64u32;++ let len = (batch_size * vocab_size) as usize;+ let logits: Vec<T> = (0..len).map(|i| T::from(i as f32 * 0.5).unwrap()).collect();++ let bitmask_size = ((vocab_size + 31) / 32) as usize;+ let bitmask = vec![0xFFFFFFFFu32; batch_size as usize * bitmask_size];++ let input = Input {+ logits: logits.into_boxed_slice(),+ bitmask: bitmask.into_boxed_slice(),+ batch_size,+ vocab_size,+ in_place: false,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_test_data_none_allowed<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let batch_size = 1u32;+ let vocab_size = 64u32;++ let len = (batch_size * vocab_size) as usize;+ let logits: Vec<T> = (0..len).map(|i| T::from(i as f32 * 0.5).unwrap()).collect();++ let bitmask_size = ((vocab_size + 31) / 32) as usize;+ let bitmask = vec![0u32; batch_size as usize * bitmask_size];++ let input = Input {+ logits: logits.into_boxed_slice(),+ bitmask: bitmask.into_boxed_slice(),+ batch_size,+ vocab_size,+ in_place: false,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_test_data_non_aligned<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ // vocab_size not a multiple of 32+ let batch_size = 2u32;+ let vocab_size = 100u32;++ let len = (batch_size * vocab_size) as usize;+ let logits: Vec<T> = (0..len).map(|i| T::from((i as f32 * 0.3).cos() * 3.0).unwrap()).collect();++ let bitmask_size = ((vocab_size + 31) / 32) as usize;+ let mut bitmask = vec![0u32; batch_size as usize * bitmask_size];+ // Set specific bits+ for b in 0..batch_size as usize {+ for word in 0..bitmask_size {+ bitmask[b * bitmask_size + word] = 0xAAAAAAAA; // bits 1,3,5,...+ }+ }++ let input = Input {+ logits: logits.into_boxed_slice(),+ bitmask: bitmask.into_boxed_slice(),+ batch_size,+ vocab_size,+ in_place: false,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::BitmaskKernel::new(&context, T::data_type(), input.in_place)+ .expect("Failed to create BitmaskKernel");++ let len = (input.batch_size * input.vocab_size) as usize;+ let bitmask_len = input.bitmask.len();++ let logits_array = context.create_array_from(&[len], &input.logits, "");+ let bitmask_array = context.create_array_from(&[bitmask_len], &input.bitmask, "");+ let logits_array_buffer_rc = logits_array.buffer();+ let logits_array_borrow = logits_array_buffer_rc.borrow();+ let logits_array_deref = logits_array_borrow.deref();+ let logits_buffer = (!input.in_place).then(|| logits_array_deref);+ let output_array = match input.in_place {+ true => context.create_array_from(&[len], &input.logits, ""),+ false => context.create_array_uninitialized(&[len], T::data_type(), ""),+ };++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ logits_buffer,+ bitmask_array.buffer().borrow().deref(),+ output_array.buffer().borrow_mut().deref_mut(),+ input.batch_size,+ input.vocab_size,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++fn test_basic<T: ArrayElement + Float + Debug + Display>(in_place: bool) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-3+ } else {+ 1e-5+ };++ let (input, expected) = get_test_data_basic::<T>(in_place);+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ let msg = format!("Bitmask basic test failed for backend {}", std::any::type_name::<B>());+ assert_eq_float::<T>(&expected, &output, eps, &msg);+ });+}++fn test_all_allowed<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-3+ } else {+ 1e-5+ };++ let (input, expected) = get_test_data_all_allowed::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ let msg = format!("Bitmask all-allowed test failed for backend {}", std::any::type_name::<B>());+ assert_eq_float::<T>(&expected, &output, eps, &msg);+ });+}++fn test_none_allowed<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-3+ } else {+ 1e-5+ };++ let (input, expected) = get_test_data_none_allowed::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ let msg = format!("Bitmask none-allowed test failed for backend {}", std::any::type_name::<B>());+ assert_eq_float::<T>(&expected, &output, eps, &msg);+ });+}++fn test_non_aligned<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-3+ } else {+ 1e-5+ };++ let (input, expected) = get_test_data_non_aligned::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ let msg = format!("Bitmask non-aligned test failed for backend {}", std::any::type_name::<B>());+ assert_eq_float::<T>(&expected, &output, eps, &msg);+ });+}++// Out-of-place basic tests+#[test]+fn test_basic_f32() {+ test_basic::<f32>(false);+}++#[test]+fn test_basic_f16() {+ test_basic::<f16>(false);+}++#[test]+fn test_basic_bf16() {+ test_basic::<bf16>(false);+}++// In-place basic tests+#[test]+fn test_in_place_f32() {+ test_basic::<f32>(true);+}++#[test]+fn test_in_place_f16() {+ test_basic::<f16>(true);+}++#[test]+fn test_in_place_bf16() {+ test_basic::<bf16>(true);+}++// All-allowed tests+#[test]+fn test_all_allowed_f32() {+ test_all_allowed::<f32>();+}++#[test]+fn test_all_allowed_f16() {+ test_all_allowed::<f16>();+}++#[test]+fn test_all_allowed_bf16() {+ test_all_allowed::<bf16>();+}++// None-allowed tests+#[test]+fn test_none_allowed_f32() {+ test_none_allowed::<f32>();+}++#[test]+fn test_none_allowed_f16() {+ test_none_allowed::<f16>();+}++#[test]+fn test_none_allowed_bf16() {+ test_none_allowed::<bf16>();+}++// Non-aligned vocab_size tests+#[test]+fn test_non_aligned_f32() {+ test_non_aligned::<f32>();+}++#[test]+fn test_non_aligned_f16() {+ test_non_aligned::<f16>();+}++#[test]+fn test_non_aligned_bf16() {+ test_non_aligned::<bf16>();+}diff --git a/crates/uzu/tests/unit/kernel/sampling/min_p_test.rs b/crates/uzu/tests/unit/kernel/sampling/min_p_test.rsnew file mode 100644index 000000000..1ad9a74aa--- /dev/null+++ b/crates/uzu/tests/unit/kernel/sampling/min_p_test.rs@@ -0,0 +1,172 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::MinPKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ logits: Box<[T]>,+ batch_size: u32,+ vocab_size: u32,+ min_p: f32,+ in_place: bool,+}++fn get_test_data<T: ArrayElement + Float>(+ batch_size: u32,+ vocab_size: u32,+ min_p: f32,+ in_place: bool,+) -> (Input<T>, Vec<T>) {+ let len = (batch_size * vocab_size) as usize;+ let logits: Vec<T> = (0..len).map(|i| T::from((i as f32 * 0.1).sin() * 2.0).unwrap()).collect();++ let input = Input {+ logits: logits.into_boxed_slice(),+ batch_size,+ vocab_size,+ min_p,+ in_place,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::MinPKernel::new(&context, T::data_type(), input.in_place)+ .expect("Failed to create MinPKernel");++ let len = (input.batch_size * input.vocab_size) as usize;+ let logits_array = context.create_array_from(&[len], &input.logits, "");+ let logits_array_buffer_rc = logits_array.buffer();+ let logits_array_borrow = logits_array_buffer_rc.borrow();+ let logits_array_deref = logits_array_borrow.deref();+ let logits_buffer = (!input.in_place).then(|| logits_array_deref);+ let output_array = match input.in_place {+ true => context.create_array_from(&[len], &input.logits, ""),+ false => context.create_array_uninitialized(&[len], T::data_type(), ""),+ };++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ logits_buffer,+ output_array.buffer().borrow_mut().deref_mut(),+ input.batch_size,+ input.vocab_size,+ input.min_p,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ batch_size: u32,+ vocab_size: u32,+ min_p: f32,+ in_place: bool,+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-3+ } else {+ 1e-5+ };++ let (input, expected) = get_test_data::<T>(batch_size, vocab_size, min_p, in_place);+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ let msg = format!("MinP test failed for backend {}", std::any::type_name::<B>());+ assert_eq_float::<T>(&expected, &output, eps, &msg);+ });+}++// Out-of-place tests+#[test]+fn test_f32() {+ test_internal::<f32>(4, 128, 0.1, false);+}++#[test]+fn test_f16() {+ test_internal::<f16>(4, 128, 0.1, false);+}++#[test]+fn test_bf16() {+ test_internal::<bf16>(4, 128, 0.1, false);+}++// In-place tests+#[test]+fn test_in_place_f32() {+ test_internal::<f32>(4, 128, 0.1, true);+}++#[test]+fn test_in_place_f16() {+ test_internal::<f16>(4, 128, 0.1, true);+}++#[test]+fn test_in_place_bf16() {+ test_internal::<bf16>(4, 128, 0.1, true);+}++// High min_p — filters most tokens+#[test]+fn test_high_min_p_f32() {+ test_internal::<f32>(2, 128, 0.9, false);+}++#[test]+fn test_high_min_p_f16() {+ test_internal::<f16>(2, 128, 0.9, false);+}++#[test]+fn test_high_min_p_bf16() {+ test_internal::<bf16>(2, 128, 0.9, false);+}++// Low min_p — keeps most tokens+#[test]+fn test_low_min_p_f32() {+ test_internal::<f32>(2, 128, 0.01, false);+}++#[test]+fn test_low_min_p_f16() {+ test_internal::<f16>(2, 128, 0.01, false);+}++#[test]+fn test_low_min_p_bf16() {+ test_internal::<bf16>(2, 128, 0.01, false);+}++// Single batch+#[test]+fn test_single_batch_f32() {+ test_internal::<f32>(1, 256, 0.1, false);+}++// Large vocab+#[test]+fn test_large_vocab_f32() {+ test_internal::<f32>(2, 2048, 0.1, false);+}diff --git a/crates/uzu/tests/unit/kernel/sampling/mod.rs b/crates/uzu/tests/unit/kernel/sampling/mod.rsindex e87b9abd0..1331ba3df 100644--- a/crates/uzu/tests/unit/kernel/sampling/mod.rs+++ b/crates/uzu/tests/unit/kernel/sampling/mod.rs@@ -1,5 +1,7 @@mod argmax_test_old;+mod bitmask_test;mod gumbel_test;+mod min_p_test;mod temperature_test;mod test_argmax;mod top_k_test;diff --git a/crates/uzu/tests/unit/kernel/sigmoid_test.rs b/crates/uzu/tests/unit/kernel/sigmoid_test.rsnew file mode 100644index 000000000..ca4f467a9--- /dev/null+++ b/crates/uzu/tests/unit/kernel/sigmoid_test.rs@@ -0,0 +1,176 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::SigmoidKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ data: Box<[T]>,+}++fn get_input<T: ArrayElement + Float>(n: usize) -> Vec<T> {+ (0..n).map(|i| T::from((i as f32 * 0.3) - 5.0).unwrap()).collect()+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::SigmoidKernel::new(&context, T::data_type())+ .expect("Failed to create SigmoidKernel");++ let n = input.data.len();+ let input_array = context.create_array_from(&[n], &input.data, "");+ let output_array = context.create_array_uninitialized(&[n], T::data_type(), "");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ input_array.buffer().borrow().deref(),+ output_array.buffer().borrow_mut().deref_mut(),+ n as u32,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let data = get_input::<T>(128);+ let input = Input {+ data: data.into_boxed_slice(),+ };+ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let data = get_input::<T>(4096);+ let input = Input {+ data: data.into_boxed_slice(),+ };+ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_test_data_edge<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ // Values at sigmoid boundaries: very negative, zero, very positive+ let values: Vec<T> = [-100.0f32, -10.0, -1.0, 0.0, 1.0, 10.0, 100.0].iter().map(|&v| T::from(v).unwrap()).collect();+ let input = Input {+ data: values.into_boxed_slice(),+ };+ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn test_basic<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.02f32+ } else {+ 1e-5+ };+ let (input, expected) = get_test_data_basic::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ assert_eq_float::<T>(+ &expected,+ &output,+ eps,+ &format!("Sigmoid basic test failed for backend {}", std::any::type_name::<B>()),+ );+ });+}++fn test_large<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.02f32+ } else {+ 1e-5+ };+ let (input, expected) = get_test_data_large::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ assert_eq_float::<T>(+ &expected,+ &output,+ eps,+ &format!("Sigmoid large test failed for backend {}", std::any::type_name::<B>()),+ );+ });+}++fn test_edge<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.02f32+ } else {+ 1e-5+ };+ let (input, expected) = get_test_data_edge::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ assert_eq_float::<T>(+ &expected,+ &output,+ eps,+ &format!("Sigmoid edge test failed for backend {}", std::any::type_name::<B>()),+ );+ });+}++// basic tests+#[test]+fn test_basic_f32() {+ test_basic::<f32>();+}++#[test]+fn test_basic_f16() {+ test_basic::<f16>();+}++#[test]+fn test_basic_bf16() {+ test_basic::<bf16>();+}++// large tests+#[test]+fn test_large_f32() {+ test_large::<f32>();+}++#[test]+fn test_large_f16() {+ test_large::<f16>();+}++#[test]+fn test_large_bf16() {+ test_large::<bf16>();+}++// edge tests+#[test]+fn test_edge_f32() {+ test_edge::<f32>();+}++#[test]+fn test_edge_f16() {+ test_edge::<f16>();+}++#[test]+fn test_edge_bf16() {+ test_edge::<bf16>();+}diff --git a/crates/uzu/tests/unit/kernel/tensor_add_scale_test.rs b/crates/uzu/tests/unit/kernel/tensor_add_scale_test.rsnew file mode 100644index 000000000..9ebcbe6eb--- /dev/null+++ b/crates/uzu/tests/unit/kernel/tensor_add_scale_test.rs@@ -0,0 +1,216 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement, DataType,+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::TensorAddScaleKernel},+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ input: Box<[T]>,+ bias: Box<[T]>,+ num_cols: u32,+ length: u32,+ scale: f32,+}++fn get_input<T: ArrayElement + Float>(+ num_rows: usize,+ num_cols: usize,+) -> (Vec<T>, Vec<T>) {+ let length = num_rows * num_cols;+ let input: Vec<T> = (0..length).map(|i| T::from((i as f32 * 0.3) - 5.0).unwrap()).collect();+ let bias: Vec<T> = (0..num_cols).map(|i| T::from((i as f32 * 0.7) - 2.0).unwrap()).collect();+ (input, bias)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::TensorAddScaleKernel::new(&context, T::data_type())+ .expect("Failed to create TensorAddScaleKernel");++ let length = input.length as usize;+ let num_cols = input.num_cols as usize;+ let input_array = context.create_array_from(&[length], &input.input, "");+ let bias_array = context.create_array_from(&[num_cols], &input.bias, "");+ let output_array = context.create_array_uninitialized(&[length], T::data_type(), "");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ input_array.buffer().borrow().deref(),+ bias_array.buffer().borrow().deref(),+ output_array.buffer().borrow_mut().deref_mut(),+ input.num_cols,+ input.length,+ input.scale,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let num_rows = 4usize;+ let num_cols = 32usize;+ let (data, bias) = get_input::<T>(num_rows, num_cols);++ let input = Input {+ input: data.into_boxed_slice(),+ bias: bias.into_boxed_slice(),+ num_cols: num_cols as u32,+ length: (num_rows * num_cols) as u32,+ scale: 0.5,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let num_rows = 64usize;+ let num_cols = 128usize;+ let (data, bias) = get_input::<T>(num_rows, num_cols);++ let input = Input {+ input: data.into_boxed_slice(),+ bias: bias.into_boxed_slice(),+ num_cols: num_cols as u32,+ length: (num_rows * num_cols) as u32,+ scale: 2.0,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn get_test_data_edge<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ // Single row, small dim, scale = 1.0+ let num_rows = 1usize;+ let num_cols = 4usize;+ let data: Vec<T> = [1.0f32, -1.0, 0.0, 3.5].iter().map(|&v| T::from(v).unwrap()).collect();+ let bias: Vec<T> = [0.5f32, -0.5, 1.0, 0.0].iter().map(|&v| T::from(v).unwrap()).collect();++ let input = Input {+ input: data.into_boxed_slice(),+ bias: bias.into_boxed_slice(),+ num_cols: num_cols as u32,+ length: (num_rows * num_cols) as u32,+ scale: 1.0,+ };++ let expected = get_output::<T, Cpu>(&input);+ (input, expected)+}++fn test_basic<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.05f32+ } else {+ 1e-5+ };+ let (input, expected) = get_test_data_basic::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ assert_eq_float::<T>(+ &expected,+ &output,+ eps,+ &format!("TensorAddScale basic test failed for backend {}", std::any::type_name::<B>()),+ );+ });+}++fn test_large<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.05f32+ } else {+ 1e-5+ };+ let (input, expected) = get_test_data_large::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ assert_eq_float::<T>(+ &expected,+ &output,+ eps,+ &format!("TensorAddScale large test failed for backend {}", std::any::type_name::<B>()),+ );+ });+}++fn test_edge<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.05f32+ } else {+ 1e-5+ };+ let (input, expected) = get_test_data_edge::<T>();+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ assert_eq_float::<T>(+ &expected,+ &output,+ eps,+ &format!("TensorAddScale edge test failed for backend {}", std::any::type_name::<B>()),+ );+ });+}++// basic tests+#[test]+fn test_basic_f32() {+ test_basic::<f32>();+}++#[test]+fn test_basic_f16() {+ test_basic::<f16>();+}++#[test]+fn test_basic_bf16() {+ test_basic::<bf16>();+}++// large tests+#[test]+fn test_large_f32() {+ test_large::<f32>();+}++#[test]+fn test_large_f16() {+ test_large::<f16>();+}++#[test]+fn test_large_bf16() {+ test_large::<bf16>();+}++// edge tests+#[test]+fn test_edge_f32() {+ test_edge::<f32>();+}++#[test]+fn test_edge_f16() {+ test_edge::<f16>();+}++#[test]+fn test_edge_bf16() {+ test_edge::<bf16>();+}