A high-performance inference engine for AI models
More cpu kernels (#246)
Aleksandr Golokozfb6f86208046
diff --git a/crates/uzu/src/backends/cpu/kernel/activation/mod.rs b/crates/uzu/src/backends/cpu/kernel/activation/mod.rsindex 3e9730314..e70a83196 100644--- a/crates/uzu/src/backends/cpu/kernel/activation/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/activation/mod.rs@@ -1 +1,49 @@+use num_traits::Float;++use crate::ArrayElement;+pub mod activation;++const GELU_K0: f32 = 0.044715f32;+const GELU_K1: f32 = 0.7978845608; // sqrt(2/pi)++#[repr(u32)]+pub enum ActivationType {+ SiLU,+ GELU,+}++impl TryFrom<u32> for ActivationType {+ type Error = ();++ fn try_from(value: u32) -> Result<Self, Self::Error> {+ match value {+ 0 => Ok(Self::SiLU),+ 1 => Ok(Self::GELU),+ _ => Err(()),+ }+ }+}++fn silu<T: Float>(x: T) -> T {+ let x_float = x.to_f32().unwrap();+ let y_float = x_float / (1.0f32 + (-x_float).exp());+ T::from(y_float).unwrap()+}++fn gelu<T: Float>(x: T) -> T {+ let x_float = x.to_f32().unwrap();+ let tan_arg = GELU_K1 * (x_float + GELU_K0 * x_float * x_float * x_float);+ let y_float = 0.5f32 * x_float * (1.0f32 + tan_arg.tanh());+ T::from(y_float).unwrap()+}++pub fn activate<T: Float>(+ x: T,+ activation_type: &ActivationType,+) -> T {+ match activation_type {+ ActivationType::SiLU => silu(x),+ ActivationType::GELU => gelu(x),+ }+}diff --git a/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rsindex b831f6dbc..f76ff38d6 100644--- a/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs+++ b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs@@ -2,7 +2,10 @@ use dsl::kernel;use half::{bf16, f16};use num_traits::Float;-use crate::ArrayElement;+use crate::{+ Activation, ArrayElement,+ backends::cpu::kernel::activation::{ActivationType, activate},+};#[kernel(MlpGateActMul)]#[variants(T, f32, f16, bf16)]@@ -13,5 +16,21 @@ pub fn mlp_gate_act_mul<T: ArrayElement + Float>(#[allow(unused)] m: i32,#[allow(unused)] act_type: u32,) {- todo!()+ let activation_type = match ActivationType::try_from(act_type) {+ Ok(a) => a,+ Err(_) => return,+ };++ for j in 0..h {+ for row in 0..m {+ let base = row * 2 * h;+ unsafe {+ let up: T = *fused_up.add((base + j) as usize);+ let gate: T = *fused_up.add((base + h + j) as usize);+ let g: T = activate(gate, &activation_type);+ let out_float = up.to_f32().unwrap() * g.to_f32().unwrap();+ *hidden.add((row * h + j) as usize) = T::from(out_float).unwrap();+ }+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/gumbel.rs b/crates/uzu/src/backends/cpu/kernel/sampling/gumbel.rsindex 5c2eb686c..569b113a6 100644--- a/crates/uzu/src/backends/cpu/kernel/sampling/gumbel.rs+++ b/crates/uzu/src/backends/cpu/kernel/sampling/gumbel.rs@@ -2,21 +2,36 @@ use dsl::kernel;use half::{bf16, f16};use num_traits::Float;-use crate::ArrayElement;+use crate::{+ ArrayElement,+ language_model::gumbel::{gumbel_float, revidx},+};#[kernel(Gumbel)]#[variants(T, f32, f16, bf16)]pub fn gumbel<T: ArrayElement + Float>(- #[allow(unused)]- #[optional(!in_place)]- logits: Option<*const T>,- #[allow(unused)] batch_seeds: *const u64,- #[allow(unused)] processed_logits: *mut T,- #[allow(unused)] batch_size: u32,- #[allow(unused)] vocab_size: u32,- #[allow(unused)]- #[specialize]- in_place: bool,+ #[optional(!in_place)] logits: Option<*const T>,+ batch_seeds: *const u64,+ processed_logits: *mut T,+ batch_size: u32,+ vocab_size: u32,+ #[specialize] in_place: bool,) {- todo!()+ let logits: *const T = match in_place {+ true => processed_logits,+ false => logits.unwrap(),+ };++ for batch_idx in 0..batch_size as usize {+ let batch_start = batch_idx * vocab_size as usize;+ let rng_seed = unsafe { *batch_seeds.add(batch_idx) };+ for vocab_idx in 0..vocab_size as usize {+ let global_idx = batch_start + vocab_idx;+ unsafe {+ let logit = (*logits.add(global_idx)).to_f32().unwrap();+ let gumbel_noise = gumbel_float(rng_seed, revidx(vocab_idx as u32));+ *processed_logits.add(global_idx) = T::from(logit + gumbel_noise).unwrap();+ }+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/temperature.rs b/crates/uzu/src/backends/cpu/kernel/sampling/temperature.rsindex 8eaf0aa69..f27f3c96b 100644--- a/crates/uzu/src/backends/cpu/kernel/sampling/temperature.rs+++ b/crates/uzu/src/backends/cpu/kernel/sampling/temperature.rs@@ -7,16 +7,25 @@ use crate::ArrayElement;#[kernel(Temperature)]#[variants(T, f32, f16, bf16)]pub fn temperature<T: ArrayElement + Float>(- #[allow(unused)]- #[optional(!in_place)]- logits: Option<*const T>,- #[allow(unused)] processed_logits: *mut T,- #[allow(unused)] batch_size: u32,- #[allow(unused)] vocab_size: u32,- #[allow(unused)] temperature: f32,- #[allow(unused)]- #[specialize]- in_place: bool,+ #[optional(!in_place)] logits: Option<*const T>,+ processed_logits: *mut T,+ batch_size: u32,+ vocab_size: u32,+ temperature: f32,+ #[specialize] in_place: bool,) {- todo!()+ let logits: *const T = match in_place {+ true => processed_logits,+ false => logits.unwrap(),+ };++ for batch_idx in 0..batch_size as usize {+ for vocab_idx in 0..vocab_size as usize {+ let global_idx = batch_idx * vocab_size as usize + vocab_idx;+ unsafe {+ let value: f32 = (*logits.add(global_idx)).to_f32().unwrap() / temperature;+ *processed_logits.add(global_idx) = T::from(value).unwrap();+ }+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/top_k.rs b/crates/uzu/src/backends/cpu/kernel/sampling/top_k.rsindex 872ccb890..56b0e42d3 100644--- a/crates/uzu/src/backends/cpu/kernel/sampling/top_k.rs+++ b/crates/uzu/src/backends/cpu/kernel/sampling/top_k.rs@@ -7,16 +7,41 @@ use crate::ArrayElement;#[kernel(TopK)]#[variants(T, f32, f16, bf16)]pub fn top_k<T: ArrayElement + Float>(- #[allow(unused)]- #[optional(!in_place)]- logits: Option<*const T>,- #[allow(unused)] processed_logits: *mut T,- #[allow(unused)] batch_size: u32,- #[allow(unused)] vocab_size: u32,- #[allow(unused)] top_k: u32,- #[allow(unused)]- #[specialize]- in_place: bool,+ #[optional(!in_place)] logits: Option<*const T>,+ processed_logits: *mut T,+ batch_size: u32,+ vocab_size: u32,+ top_k: u32,+ #[specialize] in_place: bool,) {- todo!()+ if top_k == 0 || top_k > vocab_size {+ return;+ }++ let logits: *const T = match in_place {+ true => processed_logits,+ false => logits.unwrap(),+ };++ let vocab_size = vocab_size as usize;+ let top_k = top_k as usize;++ for batch_idx in 0..batch_size as usize {+ let batch_start = batch_idx * vocab_size;+ let mut sorted_logits = unsafe { std::slice::from_raw_parts(logits.add(batch_start), vocab_size) }.to_vec();+ sorted_logits.sort_unstable_by(|a, b| b.partial_cmp(a).unwrap());+ let t_threshold = sorted_logits[top_k - 1];++ // Mask everything below the threshold+ for i in 0..vocab_size {+ let logit_value = unsafe { *logits.add(batch_start + i) };+ unsafe {+ *processed_logits.add(batch_start + i) = if logit_value >= t_threshold {+ logit_value+ } else {+ T::from(f32::NEG_INFINITY).unwrap()+ };+ }+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/top_p.rs b/crates/uzu/src/backends/cpu/kernel/sampling/top_p.rsindex 9746c3e0e..578333047 100644--- a/crates/uzu/src/backends/cpu/kernel/sampling/top_p.rs+++ b/crates/uzu/src/backends/cpu/kernel/sampling/top_p.rs@@ -7,16 +7,62 @@ use crate::ArrayElement;#[kernel(TopP)]#[variants(T, f32, f16, bf16)]pub fn top_p<T: ArrayElement + Float>(- #[allow(unused)]- #[optional(!in_place)]- logits: Option<*const T>,- #[allow(unused)] processed_logits: *mut T,- #[allow(unused)] batch_size: u32,- #[allow(unused)] vocab_size: u32,- #[allow(unused)] top_p: f32,- #[allow(unused)]- #[specialize]- in_place: bool,+ #[optional(!in_place)] logits: Option<*const T>,+ processed_logits: *mut T,+ batch_size: u32,+ vocab_size: u32,+ top_p: f32,+ #[specialize] in_place: bool,) {- todo!()+ let logits: *const T = match in_place {+ true => processed_logits,+ false => logits.unwrap(),+ };++ let vocab_size = vocab_size as usize;++ for batch_idx in 0..batch_size as usize {+ let batch_start = batch_idx * vocab_size;++ // Sort logits in descending order+ let mut sorted_logits = unsafe { std::slice::from_raw_parts(logits.add(batch_start), vocab_size) }.to_vec();+ sorted_logits.sort_unstable_by(|a, b| b.partial_cmp(a).unwrap());+ let max_logit: f32 = sorted_logits[0].to_f32().unwrap();++ // Compute softmax probabilities and find threshold via cumulative sum+ let total_sum: f32 = sorted_logits+ .iter()+ .map(|&v| v.to_f32().unwrap())+ .filter(|&v| v > f32::NEG_INFINITY)+ .map(|v| (v - max_logit).exp())+ .sum();++ let mut cumulative: f32 = 0.0;+ let mut threshold: f32 = sorted_logits[0].to_f32().unwrap();+ for &logit in &sorted_logits {+ let logit_f32 = logit.to_f32().unwrap();+ if logit_f32 == f32::NEG_INFINITY {+ break;+ }+ threshold = logit_f32;+ cumulative += (logit_f32 - max_logit).exp() / total_sum;+ if cumulative >= top_p {+ break;+ }+ }++ let t_threshold = T::from(threshold).unwrap();++ // Mask everything below the threshold+ for i in 0..vocab_size {+ let logit_value = unsafe { *logits.add(batch_start + i) };+ unsafe {+ *processed_logits.add(batch_start + i) = if logit_value >= t_threshold {+ logit_value+ } else {+ T::from(f32::NEG_INFINITY).unwrap()+ };+ }+ }+ }}diff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.h b/crates/uzu/src/backends/metal/kernel/activation/activation.hindex 431183667..5212972b2 100644--- a/crates/uzu/src/backends/metal/kernel/activation/activation.h+++ b/crates/uzu/src/backends/metal/kernel/activation/activation.h@@ -8,7 +8,7 @@ enum ActivationType : uint {template <typename T>inline T silu(T x) {float xf = float(x);- float yf = xf / (1.0f + exp(-xf));+ float yf = xf / (1.0f + metal::exp(-xf));return T(yf);}diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metalindex 521cf3ca0..8a622144b 100644--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal@@ -1,7 +1,6 @@#include <metal_stdlib>-#include "../definitions.metal"-#include "../activation/activation.h"+#include "../definitions.metal"template <typename T>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rsindex 9ea9927cc..849220505 100644--- a/crates/uzu/tests/common/mod.rs+++ b/crates/uzu/tests/common/mod.rs@@ -95,3 +95,14 @@ macro_rules! for_each_backend {}}};}++#[macro_export]+macro_rules! for_each_non_cpu_backend {+ (|$B:ident| $body:expr) => {{+ #[cfg(feature = "metal")]+ {+ type $B = uzu::backends::metal::Metal;+ $body+ }+ }};+}diff --git a/crates/uzu/tests/kernel/activation_mul_test.rs b/crates/uzu/tests/kernel/activation_mul_test.rsnew file mode 100644index 000000000..4bbbce9b9--- /dev/null+++ b/crates/uzu/tests/kernel/activation_mul_test.rs@@ -0,0 +1,119 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayElement, DataType,+ array::ArrayContextExt,+ backends::{+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels, kernel::MlpGateActMulKernel,+ },+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ fused_up: Box<[T]>,+ h: i32,+ m: i32,+ act_type: u32,+}++fn get_test_data<T: ArrayElement + Float>(act_type: u32) -> (Input<T>, Vec<T>) {+ let h = 64i32;+ let m = 4i32;+ let fused_len = (m * 2 * h) as usize;++ let mut fused_up: Vec<T> = vec![T::zero(); fused_len];+ for i in 0..fused_len {+ fused_up[i] = T::from((i as f32 * 0.1).sin() * 2.0f32).unwrap();+ }++ let input = Input {+ fused_up: fused_up.into_boxed_slice(),+ h,+ m,+ act_type,+ };++ let expected = get_output::<T, Cpu>(&input);++ (input, expected)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::MlpGateActMulKernel::new(&context, T::data_type())+ .expect("Failed to create MlpGateActMulKernel");++ let fused_len = (input.m * 2 * input.h) as usize;+ let out_len = (input.m * input.h) as usize;+ let fused_up_array = context.create_array_from(&[fused_len], &input.fused_up, "");+ let hidden_array = context.create_array_uninitialized(&[out_len], T::data_type(), "");++ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(+ fused_up_array.buffer().borrow().deref(),+ hidden_array.buffer().borrow_mut().deref_mut(),+ input.h,+ input.m,+ input.act_type,+ &mut command_buffer,+ );+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();++ hidden_array.as_slice().to_vec()+}++fn test<T: ArrayElement + Float + Debug + Display>(act_type: u32) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.02f32+ } else {+ 1e-5+ };++ let (input, expected) = get_test_data::<T>(act_type);+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ let msg = format!("Results are not equal for backend {}", std::any::type_name::<B>());+ assert_eq_float::<T>(&expected, &output, eps, &msg);+ });+}++#[test]+fn test_silu_f32() {+ test::<f32>(0);+}++#[test]+fn test_silu_f16() {+ test::<f16>(0);+}++#[test]+fn test_silu_bf16() {+ test::<bf16>(0);+}++#[test]+fn test_gelu_f32() {+ test::<f32>(1);+}++#[test]+fn test_gelu_f16() {+ test::<f16>(1);+}++#[test]+fn test_gelu_bf16() {+ test::<bf16>(1);+}diff --git a/crates/uzu/tests/kernel/attention_single_pass_test.rs b/crates/uzu/tests/kernel/attention_single_pass_test.rsindex 2db0eb818..37e3a2070 100644--- a/crates/uzu/tests/kernel/attention_single_pass_test.rs+++ b/crates/uzu/tests/kernel/attention_single_pass_test.rs@@ -171,7 +171,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(1e-5};- for_each_backend!(|B| {+ for_each_non_cpu_backend!(|B| {let output = get_output::<T, B>(input);let msg = format!("AttentionSinglePass failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={}, mask={})",diff --git a/crates/uzu/tests/kernel/attention_update_kv_cache_test.rs b/crates/uzu/tests/kernel/attention_update_kv_cache_test.rsindex 98fb1c8f2..06666b65d 100644--- a/crates/uzu/tests/kernel/attention_update_kv_cache_test.rs+++ b/crates/uzu/tests/kernel/attention_update_kv_cache_test.rs@@ -222,7 +222,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(1e-6};- for_each_backend!(|B| {+ for_each_non_cpu_backend!(|B| {let (key_cache_out, value_cache_out) = get_output::<T, B>(input);let backend_name = std::any::type_name::<B>();assert_eq_float::<T>(diff --git a/crates/uzu/tests/kernel/mod.rs b/crates/uzu/tests/kernel/mod.rsindex 87a002059..1b6734c99 100644--- a/crates/uzu/tests/kernel/mod.rs+++ b/crates/uzu/tests/kernel/mod.rs@@ -2,12 +2,14 @@#[path = "../common/mod.rs"]mod common;+mod activation_mul_test;mod attention_single_pass_test;mod attention_update_kv_cache_test;mod full_precision_embedding_test;mod qk_norm_test;mod rms_norm_test;mod rope_test;+mod sampling;mod tensor_add_bias_test;mod tensor_add_swap_test;mod tensor_copy_test;diff --git a/crates/uzu/tests/kernel/qk_norm_test.rs b/crates/uzu/tests/kernel/qk_norm_test.rsindex b587efecc..2374b1b29 100644--- a/crates/uzu/tests/kernel/qk_norm_test.rs+++ b/crates/uzu/tests/kernel/qk_norm_test.rs@@ -174,7 +174,7 @@ fn test_internal<1e-5};- for_each_backend!(|B| {+ for_each_non_cpu_backend!(|B| {let output = get_output::<B, InputT, ScaleT, OutputT, AccumT>(input);let msg = format!("QKNorm kernel test failed with backend={}, head_offset={}, head_count={}, full_layer={}",@@ -227,7 +227,7 @@ fn test_addressing<let k_start = num_q_heads * head_dim;let qkv_len = input.qkv.len();- for_each_backend!(|B| {+ for_each_non_cpu_backend!(|B| {let output = get_output::<B, InputT, ScaleT, OutputT, AccumT>(&input);// Q heads should be modified (different from input)diff --git a/crates/uzu/tests/kernel/rms_norm_test.rs b/crates/uzu/tests/kernel/rms_norm_test.rsindex 3f0b13ad2..be4345970 100644--- a/crates/uzu/tests/kernel/rms_norm_test.rs+++ b/crates/uzu/tests/kernel/rms_norm_test.rs@@ -189,7 +189,7 @@ fn test_internal<1e-5};- for_each_backend!(|B| {+ for_each_non_cpu_backend!(|B| {let (output, _, _) = get_output::<B, InputT, ScaleT, OutputT, AccumT>(input);let msg = format!("RMSNorm kernel test failed with backend={}, full_layer={}, in_place={}",diff --git a/crates/uzu/tests/kernel/rope_test.rs b/crates/uzu/tests/kernel/rope_test.rsindex be119b7bb..453681955 100644--- a/crates/uzu/tests/kernel/rope_test.rs+++ b/crates/uzu/tests/kernel/rope_test.rs@@ -120,7 +120,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(input: &Input<T>) {let (expected_queries, expected_keys) = get_output::<T, Cpu>(input);- for_each_backend!(|B| {+ for_each_non_cpu_backend!(|B| {let (queries, keys) = get_output::<T, B>(input);let msg = format!("Rope queries test failed with backend={}", std::any::type_name::<B>(),);assert_eq_float::<T>(&expected_queries, &queries, eps, &msg);diff --git a/crates/uzu/tests/kernel/sampling/gumbel_test.rs b/crates/uzu/tests/kernel/sampling/gumbel_test.rsnew file mode 100644index 000000000..3bb528631--- /dev/null+++ b/crates/uzu/tests/kernel/sampling/gumbel_test.rs@@ -0,0 +1,164 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayElement,+ array::ArrayContextExt,+ backends::{+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels, kernel::GumbelKernel,+ },+ cpu::Cpu,+ },+ language_model::gumbel::{gumbel_float, revidx},+};++struct Input<T: ArrayElement + Float> {+ logits: Box<[T]>,+ seeds: Box<[u64]>,+ batch_size: u32,+ vocab_size: u32,+ in_place: bool,+}++fn get_test_data<T: ArrayElement + Float>(+ batch_size: u32,+ vocab_size: u32,+ in_place: bool,+) -> (Input<T>, Vec<T>) {+ let len = (batch_size * vocab_size) as usize;+ let mut logits: Vec<T> = vec![T::zero(); len];+ for i in 0..len {+ logits[i] = T::from((i as f32 * 0.1).sin() * 2.0f32).unwrap();+ }++ let seeds: Vec<u64> = (0..batch_size as u64).collect();++ let input = Input {+ logits: logits.into_boxed_slice(),+ seeds: seeds.into_boxed_slice(),+ batch_size,+ vocab_size,+ in_place,+ };++ let expected = get_output::<T, Cpu>(&input);++ (input, expected)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::GumbelKernel::new(&context, T::data_type(), input.in_place)+ .expect("Failed to create GumbelKernel");++ let len = (input.batch_size * input.vocab_size) as usize;+ let logits_array = context.create_array_from(&[len], &input.logits, "");+ let logits_array_buffer_rc = logits_array.buffer();+ let logits_array_borrow = logits_array_buffer_rc.borrow();+ let logits_array_deref = logits_array_borrow.deref();+ let logits_buffer = (!input.in_place).then(|| logits_array_deref);++ let seeds_array = context.create_array_from(&[input.batch_size as usize], &input.seeds, "");++ let output_array = match input.in_place {+ true => context.create_array_from(&[len], &input.logits, ""),+ false => context.create_array_uninitialized(&[len], T::data_type(), ""),+ };++ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(+ logits_buffer,+ seeds_array.buffer().borrow().deref(),+ output_array.buffer().borrow_mut().deref_mut(),+ input.batch_size,+ input.vocab_size,+ &mut command_buffer,+ );++ command_buffer.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++// Verify determinism: same seeds produce same output.+fn test_determinism<T: ArrayElement + Float + Debug + Display>(+ batch_size: u32,+ vocab_size: u32,+) {+ let (input, output1) = get_test_data::<T>(batch_size, vocab_size, false);+ let output2 = get_output::<T, Cpu>(&input);+ assert_eq!(output1, output2, "Determinism check failed");+}++fn test_internal<T: ArrayElement + Float + Display>(+ batch_size: usize,+ vocab_size: usize,+) {+ const RTOL: f32 = 0.01;+ const ATOL: f32 = 1e-6;++ let logits = vec![T::zero(); batch_size * vocab_size];+ let seeds: Vec<u64> = (0_u64..batch_size as u64).collect();++ let input = Input {+ logits: logits.into_boxed_slice(),+ seeds: seeds.clone().into_boxed_slice(),+ batch_size: batch_size as u32,+ vocab_size: vocab_size as u32,+ in_place: false,+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);++ for (batch_idx, batch_seed) in seeds.iter().copied().enumerate() {+ let results = &output[batch_idx * vocab_size..(batch_idx + 1) * vocab_size];+ for (logit_idx, gpu_logit_value) in results.iter().copied().enumerate() {+ let cpu_logit_value = T::from(gumbel_float(batch_seed, revidx(logit_idx as u32))).unwrap();+ let abs_diff = (cpu_logit_value - gpu_logit_value).abs().to_f32().unwrap();+ let tolerance = ATOL + RTOL * cpu_logit_value.abs().to_f32().unwrap();+ assert!(+ abs_diff <= tolerance,+ "Mismatch at batch {batch_idx} element {logit_idx}: CPU={cpu_logit_value} GPU={gpu_logit_value} (abs_diff={abs_diff}, tolerance={tolerance})"+ );+ }+ }+ });+}++#[test]+fn test_determinism_f32() {+ test_determinism::<f32>(4, 128);+}++#[test]+fn test_determinism_f16() {+ test_determinism::<f16>(4, 128);+}++#[test]+fn test_determinism_bf16() {+ test_determinism::<bf16>(4, 128);+}++#[test]+fn test_f32() {+ test_internal::<f32>(7, 16 * 1024 * 64);+}++#[test]+fn test_f16() {+ test_internal::<f16>(7, 16 * 1024 * 64);+}++#[test]+fn test_bf16() {+ test_internal::<bf16>(7, 16 * 1024 * 64);+}diff --git a/crates/uzu/tests/kernel/sampling/mod.rs b/crates/uzu/tests/kernel/sampling/mod.rsnew file mode 100644index 000000000..d1b51e5ff--- /dev/null+++ b/crates/uzu/tests/kernel/sampling/mod.rs@@ -0,0 +1,4 @@+mod gumbel_test;+mod temperature_test;+mod top_k_test;+mod top_p_test;diff --git a/crates/uzu/tests/kernel/sampling/temperature_test.rs b/crates/uzu/tests/kernel/sampling/temperature_test.rsnew file mode 100644index 000000000..53bbff087--- /dev/null+++ b/crates/uzu/tests/kernel/sampling/temperature_test.rs@@ -0,0 +1,158 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayElement, DataType,+ array::ArrayContextExt,+ backends::{+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels, kernel::TemperatureKernel,+ },+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ logits: Box<[T]>,+ batch_size: u32,+ vocab_size: u32,+ temperature: f32,+ in_place: bool,+}++fn get_test_data<T: ArrayElement + Float>(+ batch_size: u32,+ vocab_size: u32,+ temperature: f32,+ in_place: bool,+) -> (Input<T>, Vec<T>) {+ let len = (batch_size * vocab_size) as usize;+ let mut logits: Vec<T> = vec![T::zero(); len];+ for i in 0..len {+ logits[i] = T::from((i as f32 * 0.1).sin() * 2.0f32).unwrap();+ }++ let input = Input {+ logits: logits.into_boxed_slice(),+ batch_size,+ vocab_size,+ temperature,+ in_place,+ };++ let expected = get_output::<T, Cpu>(&input);++ (input, expected)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::TemperatureKernel::new(&context, T::data_type(), input.in_place)+ .expect("Failed to create TemperatureKernel");++ let len = (input.batch_size * input.vocab_size) as usize;+ let logits_array = context.create_array_from(&[len], &input.logits, "");+ let logits_array_buffer_rc = logits_array.buffer();+ let logits_array_borrow = logits_array_buffer_rc.borrow();+ let logits_array_deref = logits_array_borrow.deref();+ let logits_buffer = (!input.in_place).then(|| logits_array_deref);+ let output_array = match input.in_place {+ true => context.create_array_from(&[len], &input.logits, ""),+ false => context.create_array_uninitialized(&[len], T::data_type(), ""),+ };++ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(+ logits_buffer,+ output_array.buffer().borrow_mut().deref_mut(),+ input.batch_size,+ input.vocab_size,+ input.temperature,+ &mut command_buffer,+ );++ command_buffer.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ batch_size: u32,+ vocab_size: u32,+ temperature: f32,+ in_place: bool,+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.02f32+ } else {+ 1e-5+ };++ let (input, expected) = get_test_data::<T>(batch_size, vocab_size, temperature, in_place);+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ let msg = format!("Results are not equal for backend {}", std::any::type_name::<B>());+ assert_eq_float::<T>(&expected, &output, eps, &msg);+ });+}++// Out-of-place tests+#[test]+fn test_f32() {+ test_internal::<f32>(4, 128, 0.7, false);+}++#[test]+fn test_f16() {+ test_internal::<f16>(4, 128, 0.7, false);+}++#[test]+fn test_bf16() {+ test_internal::<bf16>(4, 128, 0.7, false);+}++// In-place tests+#[test]+fn test_in_place_f32() {+ test_internal::<f32>(4, 128, 0.7, true);+}++#[test]+fn test_in_place_f16() {+ test_internal::<f16>(4, 128, 0.7, true);+}++#[test]+fn test_in_place_bf16() {+ test_internal::<bf16>(4, 128, 0.7, true);+}++// Edge cases+#[test]+fn test_temperature_1_f32() {+ test_internal::<f32>(2, 64, 1.0, false);+}++#[test]+fn test_high_temperature_f32() {+ test_internal::<f32>(2, 64, 100.0, false);+}++#[test]+fn test_low_temperature_f32() {+ test_internal::<f32>(2, 64, 0.1, false);+}++#[test]+fn test_single_batch_f32() {+ test_internal::<f32>(1, 256, 0.7, false);+}diff --git a/crates/uzu/tests/kernel/sampling/top_k_test.rs b/crates/uzu/tests/kernel/sampling/top_k_test.rsnew file mode 100644index 000000000..7332e26fb--- /dev/null+++ b/crates/uzu/tests/kernel/sampling/top_k_test.rs@@ -0,0 +1,192 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use rand::{RngExt, SeedableRng, rngs::StdRng};+use uzu::{+ ArrayElement, DataType,+ array::ArrayContextExt,+ backends::{+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels, kernel::TopKKernel,+ },+ cpu::Cpu,+ },+};++struct Input<T: ArrayElement + Float> {+ logits: Box<[T]>,+ batch_size: u32,+ vocab_size: u32,+ top_k: u32,+ in_place: bool,+}++fn get_test_data<T: ArrayElement + Float>(+ batch_size: u32,+ vocab_size: u32,+ top_k: u32,+ in_place: bool,+) -> (Input<T>, Vec<T>) {+ let len = (batch_size * vocab_size) as usize;+ let mut rng = StdRng::seed_from_u64(42);+ let mut logits: Vec<T> = vec![T::zero(); len];+ for x in logits.iter_mut() {+ *x = T::from(rng.random_range(-1024.0f32..1024.0f32)).unwrap();+ }++ let input = Input {+ logits: logits.into_boxed_slice(),+ batch_size,+ vocab_size,+ top_k,+ in_place,+ };++ let expected = get_output::<T, Cpu>(&input);++ (input, expected)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::TopKKernel::new(&context, T::data_type(), input.in_place)+ .expect("Failed to create TopKKernel");++ let len = (input.batch_size * input.vocab_size) as usize;+ let logits_array = context.create_array_from(&[len], &input.logits, "");+ let logits_array_buffer_rc = logits_array.buffer();+ let logits_array_borrow = logits_array_buffer_rc.borrow();+ let logits_array_deref = logits_array_borrow.deref();+ let logits_buffer = (!input.in_place).then(|| logits_array_deref);+ let output_array = match input.in_place {+ true => context.create_array_from(&[len], &input.logits, ""),+ false => context.create_array_uninitialized(&[len], T::data_type(), ""),+ };++ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(+ logits_buffer,+ output_array.buffer().borrow_mut().deref_mut(),+ input.batch_size,+ input.vocab_size,+ input.top_k,+ &mut command_buffer,+ );++ command_buffer.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++fn assert_top_k_equal<T: ArrayElement + Float + Display>(+ expected: &[T],+ actual: &[T],+ vocab_size: u32,+ top_k: u32,+ eps: f32,+ msg: &str,+) {+ assert_eq!(expected.len(), actual.len(), "Slices size mismatch");+ let vocab_size = vocab_size as usize;+ let num_batches = expected.len() / vocab_size;++ for batch_idx in 0..num_batches {+ let start = batch_idx * vocab_size;+ let end = start + vocab_size;++ // Both should keep approximately top_k values and mask the rest+ let expected_kept: usize = expected[start..end].iter().filter(|v| v.is_finite()).count();+ let actual_kept: usize = actual[start..end].iter().filter(|v| v.is_finite()).count();++ // Allow small differences due to binary search convergence+ let diff = (expected_kept as i64 - actual_kept as i64).unsigned_abs();+ assert!(+ diff <= 3,+ "{msg}. Batch {batch_idx}: expected {expected_kept} kept values, got {actual_kept} (top_k={top_k})"+ );++ // For values that are kept by both, they should match the original values+ for i in start..end {+ let e = expected[i].to_f32().unwrap();+ let a = actual[i].to_f32().unwrap();+ if e.is_finite() && a.is_finite() {+ let d = (e - a).abs();+ assert!(d < eps, "{msg}. Mismatch at index {i}: expected {e}, got {a}, diff {d}");+ }+ }+ }+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ batch_size: u32,+ vocab_size: u32,+ top_k: u32,+ in_place: bool,+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-2+ } else {+ 1e-5+ };++ let (input, expected) = get_test_data::<T>(batch_size, vocab_size, top_k, in_place);+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ let msg = format!("Results are not equal for backend {}", std::any::type_name::<B>());+ assert_top_k_equal::<T>(&expected, &output, vocab_size, top_k, eps, &msg);+ });+}++// Out-of-place tests (same data as sampling_test::test_topk_gpu_cpu_match)+#[test]+fn test_f32() {+ test_internal::<f32>(4, 1024, 16, false);+}++#[test]+fn test_f16() {+ test_internal::<f16>(4, 1024, 16, false);+}++#[test]+fn test_bf16() {+ test_internal::<bf16>(4, 1024, 16, false);+}++// In-place tests+#[test]+fn test_in_place_f32() {+ test_internal::<f32>(4, 1024, 16, true);+}++#[test]+fn test_in_place_f16() {+ test_internal::<f16>(4, 1024, 16, true);+}++#[test]+fn test_in_place_bf16() {+ test_internal::<bf16>(4, 1024, 16, true);+}++// Edge cases+#[test]+fn test_single_batch_f32() {+ test_internal::<f32>(1, 1024, 16, false);+}++#[test]+fn test_top_k_1_f32() {+ test_internal::<f32>(4, 1024, 1, false);+}++#[test]+fn test_top_k_equals_vocab_f32() {+ test_internal::<f32>(4, 1024, 1024, false);+}diff --git a/crates/uzu/tests/kernel/sampling/top_p_test.rs b/crates/uzu/tests/kernel/sampling/top_p_test.rsnew file mode 100644index 000000000..4c773e420--- /dev/null+++ b/crates/uzu/tests/kernel/sampling/top_p_test.rs@@ -0,0 +1,519 @@+use std::{+ collections::HashSet,+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use rand::{RngExt, SeedableRng, rngs::StdRng, seq::SliceRandom};+use uzu::{+ ArrayElement, DataType,+ array::ArrayContextExt,+ backends::{+ common::{+ Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,+ CommandBufferPending, Context, Kernels,+ kernel::{TopPKernel, sampling::SamplingKernel},+ },+ cpu::Cpu,+ },+ session::parameter::SamplingMethod,+};++const TEST_SAMPLING_SEED: u64 = 42;++struct Input<T: ArrayElement + Float> {+ logits: Box<[T]>,+ batch_size: u32,+ vocab_size: u32,+ top_p: f32,+ in_place: bool,+}++fn get_test_data<T: ArrayElement + Float>(+ batch_size: u32,+ vocab_size: u32,+ top_p: f32,+ in_place: bool,+) -> (Input<T>, Vec<T>) {+ let len = (batch_size * vocab_size) as usize;+ let mut rng = StdRng::seed_from_u64(42);+ let mut logits: Vec<T> = vec![T::zero(); len];+ for x in logits.iter_mut() {+ *x = T::from(rng.random_range(-16.0f32..16.0f32)).unwrap();+ }++ let input = Input {+ logits: logits.into_boxed_slice(),+ batch_size,+ vocab_size,+ top_p,+ in_place,+ };++ let expected = get_output::<T, Cpu>(&input);++ (input, expected)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = <B as Backend>::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::TopPKernel::new(&context, T::data_type(), input.in_place)+ .expect("Failed to create TopPKernel");++ let len = (input.batch_size * input.vocab_size) as usize;+ let logits_array = context.create_array_from(&[len], &input.logits, "");+ let logits_array_buffer_rc = logits_array.buffer();+ let logits_array_borrow = logits_array_buffer_rc.borrow();+ let logits_array_deref = logits_array_borrow.deref();+ let logits_buffer = (!input.in_place).then(|| logits_array_deref);+ let output_array = match input.in_place {+ true => context.create_array_from(&[len], &input.logits, ""),+ false => context.create_array_uninitialized(&[len], T::data_type(), ""),+ };++ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(+ logits_buffer,+ output_array.buffer().borrow_mut().deref_mut(),+ input.batch_size,+ input.vocab_size,+ input.top_p,+ &mut command_buffer,+ );++ command_buffer.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++fn assert_top_p_equal<T: ArrayElement + Float + Display>(+ expected: &[T],+ actual: &[T],+ vocab_size: u32,+ eps: f32,+ msg: &str,+) {+ assert_eq!(expected.len(), actual.len(), "Slices size mismatch");+ let vocab_size = vocab_size as usize;+ let num_batches = expected.len() / vocab_size;++ for batch_idx in 0..num_batches {+ let start = batch_idx * vocab_size;+ let end = start + vocab_size;++ let expected_kept: usize = expected[start..end].iter().filter(|v| v.is_finite()).count();+ let actual_kept: usize = actual[start..end].iter().filter(|v| v.is_finite()).count();++ // Allow small differences due to binary search convergence+ let diff = (expected_kept as i64 - actual_kept as i64).unsigned_abs();+ assert!(diff <= 3, "{msg}. Batch {batch_idx}: expected {expected_kept} kept values, got {actual_kept}");++ // For values that are kept by both, they should match the original values+ for i in start..end {+ let e = expected[i].to_f32().unwrap();+ let a = actual[i].to_f32().unwrap();+ if e.is_finite() && a.is_finite() {+ let d = (e - a).abs();+ assert!(d < eps, "{msg}. Mismatch at index {i}: expected {e}, got {a}, diff {d}");+ }+ }+ }+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ batch_size: u32,+ vocab_size: u32,+ top_p: f32,+ in_place: bool,+) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-2+ } else {+ 1e-5+ };++ let (input, expected) = get_test_data::<T>(batch_size, vocab_size, top_p, in_place);+ for_each_non_cpu_backend!(|B| {+ let output = get_output::<T, B>(&input);+ let msg = format!("Results are not equal for backend {}", std::any::type_name::<B>());+ assert_top_p_equal::<T>(&expected, &output, vocab_size, eps, &msg);+ });+}++// Out-of-place tests (same data as sampling_test::test_topp_gpu_cpu_match)+#[test]+fn test_f32() {+ test_internal::<f32>(4, 1024, 0.9, false);+}++#[test]+fn test_f16() {+ test_internal::<f16>(4, 1024, 0.9, false);+}++#[test]+fn test_bf16() {+ test_internal::<bf16>(4, 1024, 0.9, false);+}++// In-place tests+#[test]+fn test_in_place_f32() {+ test_internal::<f32>(4, 1024, 0.9, true);+}++#[test]+fn test_in_place_f16() {+ test_internal::<f16>(4, 1024, 0.9, true);+}++#[test]+fn test_in_place_bf16() {+ test_internal::<bf16>(4, 1024, 0.9, true);+}++// Edge cases+#[test]+fn test_single_batch_f32() {+ test_internal::<f32>(1, 1024, 0.9, false);+}++#[test]+fn test_top_p_very_small() {+ test_internal::<f32>(4, 1024, 0.01, false);+}++#[test]+fn test_top_p_near_1() {+ test_internal::<f32>(4, 1024, 0.99, false);+}++// --- Sampling-level top-p tests (test the full SamplingKernel with top_p) ---++fn cpu_reference_top_p(+ row_logits: &[f32],+ top_p: f32,+) -> Vec<f32> {+ let max_logit = row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);+ let exp_vals: Vec<f32> = row_logits.iter().map(|&x| (x - max_logit).exp()).collect();+ let sum_exp: f32 = exp_vals.iter().sum();+ let probs: Vec<f32> = exp_vals.iter().map(|&x| x / sum_exp).collect();++ let mut indexed: Vec<(usize, f32)> = probs.iter().copied().enumerate().collect();+ indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());++ let mut cum = 0.0f32;+ let mut keep = vec![false; row_logits.len()];+ for &(idx, p) in &indexed {+ if cum >= top_p {+ break;+ }+ keep[idx] = true;+ cum += p;+ }++ // Renormalise kept probabilities+ let kept_sum: f32 = probs.iter().zip(keep.iter()).filter(|&(_, &k)| k).map(|(&p, _)| p).sum();+ probs+ .iter()+ .zip(keep.iter())+ .map(|(&p, &k)| {+ if k {+ p / kept_sum+ } else {+ 0.0+ }+ })+ .collect()+}++fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(+ batch_size: usize,+ k: usize,+ vocab_size: usize,+) {+ let context = <B as Backend>::Context::new().expect("Failed to create Context");++ let p = (k as f32) * 0.1;++ let kernel = SamplingKernel::<B>::new(&context, DataType::F32, batch_size, vocab_size)+ .expect("Failed to create sampling kernel");++ // Build probability table+ let low_prob = (1.0 - p) / (vocab_size - k) as f32;+ let mut probs = vec![low_prob; batch_size * vocab_size];+ let mut high_prob_token_sets: Vec<HashSet<usize>> = Vec::with_capacity(batch_size);++ let mut rng = StdRng::seed_from_u64(42);+ let mut all_token_ids: Vec<usize> = (0..vocab_size).collect();++ for b in 0..batch_size {+ all_token_ids.shuffle(&mut rng);+ let high_prob_tokens: HashSet<usize> = all_token_ids[..k].iter().cloned().collect();++ for &token_id in &high_prob_tokens {+ probs[b * vocab_size + token_id] = 0.1;+ }+ high_prob_token_sets.push(high_prob_tokens);+ }++ let logits: Vec<f32> = probs+ .iter()+ .map(|&prob| {+ if prob <= 0.0 {+ -50.0+ } else {+ prob.ln()+ }+ })+ .collect();++ let output_array = context.create_array_uninitialized(&[batch_size], DataType::U32, "");++ let num_samples = 1000;+ let mut counter = vec![0i32; batch_size * vocab_size];++ for draw in 0..num_samples {+ let logits_array = context.create_array_from(&[batch_size * vocab_size], &logits, "");+ let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; batch_size];+ let seeds_array = context.create_array_from(&[batch_size], &seeds, "");++ let mut command_buffer =+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel+ .encode(+ logits_array.buffer().borrow_mut().deref_mut(),+ Some(seeds_array.buffer().borrow().deref()),+ 0,+ None,+ 0,+ output_array.buffer().borrow_mut().deref_mut(),+ SamplingMethod::Stochastic {+ temperature: None,+ top_k: None,+ top_p: Some(p),+ min_p: None,+ },+ batch_size,+ vocab_size,+ &mut command_buffer,+ )+ .expect("encode");+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();++ let sampled_ids: &[u32] = output_array.as_slice();++ for (i, &sampled_id) in sampled_ids.iter().enumerate() {+ assert!((sampled_id as usize) < vocab_size, "Sampled token out of range");+ counter[i * vocab_size + sampled_id as usize] += 1;+ }+ }++ for i in 0..batch_size {+ for j in 0..vocab_size {+ if counter[i * vocab_size + j] > 0 {+ assert!(+ high_prob_token_sets[i].contains(&j),+ "high_prob_token_sets[{}] does not contain {} (appeared {} times)",+ i,+ j,+ counter[i * vocab_size + j]+ );+ }+ }+ }++ println!(+ "batch_size: {}, p: {:.1}, vocab_size: {}, accuracy test passed (backend: {}).",+ batch_size,+ p,+ vocab_size,+ std::any::type_name::<B>()+ );+}++#[test]+fn test_topp_sampling_match_small() {+ for_each_non_cpu_backend!(|B| {+ test_topp_sampling_from_prob_exact_match_internal::<B>(8, 10, 1024);+ });+}++#[test]+fn test_topp_sampling_match_large() {+ for_each_non_cpu_backend!(|B| {+ test_topp_sampling_from_prob_exact_match_internal::<B>(32, 50, 4096);+ });+}++#[test]+fn test_topp_sampling_statistical_large() {+ for_each_non_cpu_backend!(|B| {+ const BATCH: usize = 32;+ const VOCAB: usize = 4096;+ const NUM_DRAWS: usize = 10_000;+ const TOP_P: f32 = 0.9;+ const TOLERANCE_KL: f32 = 0.05;++ let context = <B as Backend>::Context::new().expect("Failed to create Context");++ let kernel =+ SamplingKernel::<B>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");++ let mut rng = StdRng::seed_from_u64(42);+ let mut logits = vec![0.0f32; BATCH * VOCAB];+ for x in logits.iter_mut() {+ *x = rng.random_range(-6.0f32..6.0f32);+ }++ // Build Top-p renormalised target distribution per row+ let mut probs = vec![0.0f32; BATCH * VOCAB];+ for b in 0..BATCH {+ let row_logits = &logits[b * VOCAB..(b + 1) * VOCAB];+ let dist = cpu_reference_top_p(row_logits, TOP_P);+ probs[b * VOCAB..(b + 1) * VOCAB].copy_from_slice(&dist);+ }++ let output_array = context.create_array_uninitialized(&[BATCH], DataType::U32, "");+ let mut counters = vec![0u32; BATCH * VOCAB];++ for draw in 0..NUM_DRAWS {+ let logits_array = context.create_array_from(&[BATCH * VOCAB], &logits, "");+ let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; BATCH];+ let seeds_array = context.create_array_from(&[BATCH], &seeds, "");++ let mut command_buffer =+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();++ kernel+ .encode(+ logits_array.buffer().borrow_mut().deref_mut(),+ Some(seeds_array.buffer().borrow().deref()),+ 0,+ None,+ 0,+ output_array.buffer().borrow_mut().deref_mut(),+ SamplingMethod::Stochastic {+ temperature: None,+ top_k: None,+ top_p: Some(TOP_P),+ min_p: None,+ },+ BATCH,+ VOCAB,+ &mut command_buffer,+ )+ .expect("encode");+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();++ let sample_ids: &[u32] = output_array.as_slice();+ for (b, &tok) in sample_ids.iter().enumerate() {+ counters[b * VOCAB + tok as usize] += 1;+ }+ }++ // Compute KL divergence per row+ for b in 0..BATCH {+ let mut kl = 0.0_f32;+ for j in 0..VOCAB {+ let expected = probs[b * VOCAB + j];+ if expected < 1e-12 {+ continue;+ }+ let observed = (counters[b * VOCAB + j] as f32) / (NUM_DRAWS as f32);+ if observed > 0.0 {+ kl += observed * (observed.ln() - expected.ln());+ }+ }+ assert!(kl < TOLERANCE_KL, "Row {} KL {:.4} exceeded tolerance {:.3}", b, kl, TOLERANCE_KL);+ }++ println!(+ "Large statistical Top-p test passed (KL < {:.3}, backend: {})",+ TOLERANCE_KL,+ std::any::type_name::<B>()+ );+ });+}++#[test]+fn perf_topp_128k_vocab() {+ for_each_non_cpu_backend!(|B| {+ const BATCH: usize = 8;+ const VOCAB: usize = 128000;+ const TOP_P: f32 = 0.9;++ let context = <B as Backend>::Context::new().expect("Failed to create Context");++ let kernel =+ SamplingKernel::<B>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");++ let mut rng = StdRng::seed_from_u64(123);+ let mut logits = vec![0.0f32; BATCH * VOCAB];+ for x in logits.iter_mut() {+ *x = rng.random_range(-6.0f32..6.0f32);+ }++ let logits_array = context.create_array_from(&[BATCH * VOCAB], &logits, "");+ let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];+ let seeds_array = context.create_array_from(&[BATCH], &seeds, "");+ let output_array = context.create_array_uninitialized(&[BATCH], DataType::U32, "");++ let mut command_buffer =+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();++ kernel+ .encode(+ logits_array.buffer().borrow_mut().deref_mut(),+ Some(seeds_array.buffer().borrow().deref()),+ 0,+ None,+ 0,+ output_array.buffer().borrow_mut().deref_mut(),+ SamplingMethod::Stochastic {+ temperature: None,+ top_k: None,+ top_p: Some(TOP_P),+ min_p: None,+ },+ BATCH,+ VOCAB,+ &mut command_buffer,+ )+ .expect("encode");++ let host_timer = std::time::Instant::now();+ let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;++ match completed.gpu_execution_time() {+ Some(gpu_time) => {+ println!(+ "Top-p sampling perf (batch={}, vocab={}, backend={}): GPU={:.2} ms, Host-side={:.2} ms",+ BATCH,+ VOCAB,+ std::any::type_name::<B>(),+ gpu_time.as_secs_f64() * 1e3,+ host_elapsed_ms+ );+ },+ None => {+ println!(+ "Top-p sampling perf (batch={}, vocab={}, backend={}): Host-side={:.2} ms (GPU timing unavailable)",+ BATCH,+ VOCAB,+ std::any::type_name::<B>(),+ host_elapsed_ms+ );+ },+ }++ let sample_ids: &[u32] = output_array.as_slice();+ for &tok in sample_ids {+ assert!((tok as usize) < VOCAB, "Sampled id out of range");+ }+ });+}diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rsindex 6eed789db..9dfcd9b59 100644--- a/crates/uzu/tests/sampling_test.rs+++ b/crates/uzu/tests/sampling_test.rs@@ -1,7 +1,7 @@mod common;use bytemuck;use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};-use rand::{RngExt, seq::SliceRandom};+use rand::seq::SliceRandom;// for Vec::shuffleuse uzu::{DataType,@@ -10,52 +10,18 @@ use uzu::{Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,CommandBufferPending, Context, Kernels,kernel::{- GumbelKernel, MinPKernel, TemperatureKernel, TopKKernel, TopPKernel,+ MinPKernel, TemperatureKernel,sampling::{ArgmaxStrategy, SamplingKernel},},},metal::Metal,},- language_model::gumbel::{gumbel_float, revidx},session::parameter::SamplingMethod,};// Constant seed for reproducible test resultsconst TEST_SAMPLING_SEED: u64 = 42;-fn cpu_reference_top_p(- row_logits: &[f32],- top_p: f32,-) -> Vec<f32> {- // 1. soft‑max (unnormalised)- let max_logit = row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);- let mut softmax: Vec<(usize, f32)> =- row_logits.iter().enumerate().map(|(idx, &z)| (idx, (z - max_logit).exp())).collect();- let total_mass: f32 = softmax.iter().map(|(_, p)| p).sum();-- // 2. sort descending- softmax.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());-- // 3. accumulate until cumulative ≥ top_p- let mut cum = 0.0_f32;- let mut cutoff = 0;- let target = top_p * total_mass;- while cutoff < softmax.len() && cum < target {- cum += softmax[cutoff].1;- cutoff += 1;- }-- // 4. renormalise the kept mass- let mut dist = vec![0.0f32; row_logits.len()];- if cum > 0.0 {- let renorm = 1.0 / cum;- for (idx, p) in &softmax[..cutoff] {- dist[*idx] = *p * renorm;- }- }- dist-}-fn cpu_reference_min_p(row_logits: &[f32],min_p: f32,@@ -151,358 +117,6 @@ fn test_argmax_sampling() {test_argmax_sampling_with_strategy(ArgmaxStrategy::SinglePass);}-fn test_topp_sampling_from_prob_exact_match(- batch_size: usize,- k: usize,- vocab_size: usize,-) {- use std::collections::HashSet;-- use rand::{SeedableRng, rngs::StdRng};-- let context = match <Metal as Backend>::Context::new() {- Ok(ctx) => ctx,- Err(e) => {- eprintln!("Skipping topP exact match test: {}", e);- return;- },- };-- // Calculate top_p threshold- let p = (k as f32) * 0.1;-- // Create kernel- let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32, batch_size, vocab_size)- .expect("Failed to create sampling kernel");-- // Build probability table- let low_prob = (1.0 - p) / (vocab_size - k) as f32;- let mut probs = vec![low_prob; batch_size * vocab_size];- let mut high_prob_token_sets: Vec<HashSet<usize>> = Vec::with_capacity(batch_size);-- // Use std::sample equivalent (Fisher-Yates shuffle + take first k)- let mut rng = StdRng::seed_from_u64(42); // Deterministic for reproducibility- let mut all_token_ids: Vec<usize> = (0..vocab_size).collect();-- for b in 0..batch_size {- // Equivalent to std::sample - shuffle and take first k- all_token_ids.shuffle(&mut rng);- let high_prob_tokens: HashSet<usize> = all_token_ids[..k].iter().cloned().collect();-- // Set selected tokens to high probability- for &token_id in &high_prob_tokens {- probs[b * vocab_size + token_id] = 0.1;- }- high_prob_token_sets.push(high_prob_tokens);- }-- // Convert to logits more carefully to avoid -∞- let logits: Vec<f32> = probs- .iter()- .map(|&prob| {- if prob <= 0.0 {- -50.0 // Very negative but finite value instead of -∞- } else {- prob.ln()- }- })- .collect();-- // GPU buffers- let mut output_buf = context- .device- .new_buffer(batch_size * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- // Counter for samples- let num_samples = 1000;- let mut counter = vec![0i32; batch_size * vocab_size];-- // Draw samples- for draw in 0..num_samples {- // Create fresh logits buffer since kernel mutates in-place- let mut logits_buf = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; batch_size];- let seeds_buf = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut command_buffer =- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();- kernel- .encode(- &mut logits_buf,- Some(&seeds_buf),- 0,- None,- 0,- &mut output_buf,- SamplingMethod::Stochastic {- temperature: None,- top_k: None,- top_p: Some(p),- min_p: None,- },- batch_size,- vocab_size,- &mut command_buffer,- )- .expect("encode");- command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let ptr = output_buf.contents().as_ptr() as *const u32;- let sampled_ids = unsafe { std::slice::from_raw_parts(ptr, batch_size) };-- for (i, &sampled_id) in sampled_ids.iter().enumerate() {- assert!((sampled_id as usize) < vocab_size, "Sampled token out of range");- counter[i * vocab_size + sampled_id as usize] += 1;- }- }-- for i in 0..batch_size {- for j in 0..vocab_size {- if counter[i * vocab_size + j] > 0 {- assert!(- high_prob_token_sets[i].contains(&j),- "high_prob_token_sets[{}] does not contain {} (appeared {} times)",- i,- j,- counter[i * vocab_size + j]- );- }- }- }-- println!("batch_size: {}, p: {:.1}, vocab_size: {}, accuracy test passed.", batch_size, p, vocab_size);-}--#[test]-fn test_topp_sampling_match_small() {- test_topp_sampling_from_prob_exact_match(8, 10, 1024);-}--#[test]-fn test_topp_sampling_match_large() {- test_topp_sampling_from_prob_exact_match(32, 50, 4096);-}--#[test]-fn test_topp_sampling_statistical_large() {- use rand::{RngExt, SeedableRng, rngs::StdRng};-- // ===== 1. Create Metal context =====- let context = match <Metal as Backend>::Context::new() {- Ok(ctx) => ctx,- Err(e) => {- println!("Skipping large statistical test: {}", e);- return;- },- };-- // ===== 2. Problem sizes =====- const BATCH: usize = 32;- const VOCAB: usize = 4096;- const NUM_DRAWS: usize = 10_000;- const TOP_P: f32 = 0.9;- const TOLERANCE_KL: f32 = 0.05;-- // ===== 3. Build kernel =====- let kernel =- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");-- // ===== 4. Generate reproducible random logits =====- let mut rng = StdRng::seed_from_u64(42);- let mut logits = vec![0.0f32; BATCH * VOCAB];- for x in logits.iter_mut() {- // Uniform[-6, 6] (broad range avoids numerical under/overflow)- *x = rng.random_range(-6.0f32..6.0f32);- }-- // ===== 5. Build Top‑p renormalised target distribution per row =====- let mut probs = vec![0.0f32; BATCH * VOCAB];- for b in 0..BATCH {- let row_logits = &logits[b * VOCAB..(b + 1) * VOCAB];- let dist = cpu_reference_top_p(row_logits, TOP_P);- probs[b * VOCAB..(b + 1) * VOCAB].copy_from_slice(&dist);- }-- // ===== 6. Allocate output & counters =====- let mut output_buf = context- .device- .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let mut counters = vec![0u32; BATCH * VOCAB];-- // ===== 8. Draw NUM_DRAWS samples =====- for draw in 0..NUM_DRAWS {- // Create fresh logits buffer since kernel mutates in-place- let mut logits_buf = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; BATCH];- let seeds_buf = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut command_buffer =- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();-- kernel- .encode(- &mut logits_buf,- Some(&seeds_buf),- 0,- None,- 0,- &mut output_buf,- SamplingMethod::Stochastic {- temperature: None,- top_k: None,- top_p: Some(TOP_P),- min_p: None,- },- BATCH,- VOCAB,- &mut command_buffer,- )- .expect("encode");- command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let ptr = output_buf.contents().as_ptr() as *const u32;- let sample_ids = unsafe { std::slice::from_raw_parts(ptr, BATCH) };- for (b, &tok) in sample_ids.iter().enumerate() {- counters[b * VOCAB + tok as usize] += 1;- }- }-- // ===== 9. Compute KL divergence per row =====- for b in 0..BATCH {- let mut kl = 0.0_f32;- for j in 0..VOCAB {- let expected = probs[b * VOCAB + j];- if expected < 1e-12 {- continue;- } // ignore tiny- let observed = (counters[b * VOCAB + j] as f32) / (NUM_DRAWS as f32);- if observed > 0.0 {- kl += observed * (observed.ln() - expected.ln());- }- }- assert!(kl < TOLERANCE_KL, "Row {} KL {:.4} exceeded tolerance {:.3}", b, kl, TOLERANCE_KL);- }-- println!("✓ Large statistical Top‑p test passed (KL < {:.3})", TOLERANCE_KL);-}--#[test]-fn perf_topp_128k_vocab() {- use std::time::Instant;-- use rand::{RngExt, SeedableRng, rngs::StdRng};-- // ---- Metal context ----- let context = match <Metal as Backend>::Context::new() {- Ok(ctx) => ctx,- Err(e) => {- println!("Skipping perf test: {}", e);- return;- },- };-- // ---- Problem sizes ----- const BATCH: usize = 8;- const VOCAB: usize = 128000; // 128K- const TOP_P: f32 = 0.9;-- // ---- Kernel ----- let kernel =- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");-- // ---- Build random logits ----- let mut rng = StdRng::seed_from_u64(123);- let mut logits = vec![0.0f32; BATCH * VOCAB];- for x in logits.iter_mut() {- *x = rng.random_range(-6.0f32..6.0f32);- }-- let mut logits_buf = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];- let seeds_buf = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut output_buf = context- .device- .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- // ---- Launch once and time ----- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();-- kernel- .encode(- &mut logits_buf,- Some(&seeds_buf),- 0,- None,- 0,- &mut output_buf,- SamplingMethod::Stochastic {- temperature: None,- top_k: None,- top_p: Some(TOP_P),- min_p: None,- },- BATCH,- VOCAB,- &mut command_buffer,- )- .expect("encode");-- // Time both host-side and GPU execution- let host_timer = Instant::now();- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();- let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;-- match completed.gpu_execution_time() {- Some(gpu_time) => {- println!(- "Top‑p sampling perf (batch={}, vocab={}): GPU={:.2} ms, Host-side={:.2} ms",- BATCH,- VOCAB,- gpu_time.as_secs_f64() * 1e3,- host_elapsed_ms- );- },- None => {- println!(- "Top‑p sampling perf (batch={}, vocab={}): Host-side={:.2} ms (GPU timing unavailable)",- BATCH, VOCAB, host_elapsed_ms- );- },- }-- // Ensure the kernel produced *some* output (sanity).- let ptr = output_buf.contents().as_ptr() as *const u32;- let sample_ids = unsafe { std::slice::from_raw_parts(ptr, BATCH) };- for &tok in sample_ids {- assert!((tok as usize) < VOCAB, "Sampled id out of range");- }-}-#[allow(dead_code)]fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {use std::time::Instant;@@ -1022,129 +636,6 @@ fn test_temperature_gpu_cpu_match() {println!("✓ Temperature processor gpu cpu match (temp={}, rtol={}, atol={})", TEMPERATURE, RTOL, ATOL);}-#[test]-fn test_topk_gpu_cpu_match() {- use rand::{SeedableRng, rngs::StdRng};-- let context = match <Metal as Backend>::Context::new() {- Ok(ctx) => ctx,- Err(e) => {- println!("Skipping topk gpu cpu match test: {}", e);- return;- },- };-- const BATCH: usize = 4;- const VOCAB: usize = 1024;- const TOPK: u32 = 16;-- let kernel = <<Metal as Backend>::Kernels as Kernels>::TopKKernel::new(&context, DataType::F32, false)- .expect("Failed to create topk kernel");-- let mut rng = StdRng::seed_from_u64(42);- let mut logits = vec![0.0f32; BATCH * VOCAB];- for x in logits.iter_mut() {- *x = rng.random_range(-1024.0f32..1024.0f32);- }-- let logits_buffer = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut processed_buffer = context- .device- .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();- kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TOPK, &mut command_buffer);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let results_ptr = processed_buffer.contents().as_ptr() as *const f32;- let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };-- for batch_idx in 0..BATCH {- let cpu_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];- let mut cpu_sorted_logits: Vec<(usize, f32)> = cpu_logits.iter().copied().enumerate().collect();- cpu_sorted_logits.sort_by(|(_, a), (_, b)| f32::total_cmp(b, a));- let mut cpu_processed_logits: Vec<f32> = vec![f32::NEG_INFINITY; VOCAB];- for (idx, val) in cpu_sorted_logits.into_iter().take(TOPK as usize) {- cpu_processed_logits[idx] = val;- }-- let gpu_processed_logits = &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];-- assert_eq!(cpu_processed_logits, gpu_processed_logits);- }-- println!("✓ Topk processor gpu cpu match (topk={})", TOPK);-}--#[test]-fn test_topp_gpu_cpu_match() {- use rand::{SeedableRng, rngs::StdRng};-- let context = match <Metal as Backend>::Context::new() {- Ok(ctx) => ctx,- Err(e) => {- println!("Skipping topp gpu cpu match test: {}", e);- return;- },- };-- const BATCH: usize = 4;- const VOCAB: usize = 1024;- const TOPP: f32 = 0.9;-- let kernel = <<Metal as Backend>::Kernels as Kernels>::TopPKernel::new(&context, DataType::F32, false)- .expect("Failed to create topp kernel");-- let mut rng = StdRng::seed_from_u64(42);- let mut logits = vec![0.0f32; BATCH * VOCAB];- for x in logits.iter_mut() {- *x = rng.random_range(-16.0f32..16.0f32);- }-- let logits_buffer = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut processed_buffer = context- .device- .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();- kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TOPP, &mut command_buffer);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let results_ptr = processed_buffer.contents().as_ptr() as *const f32;- let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };-- for batch_idx in 0..BATCH {- let cpu_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];- let cpu_processed_logits = cpu_reference_top_p(cpu_logits, TOPP)- .into_iter()- .zip(cpu_logits.iter().copied())- .map(|(cpu_ref_topp, logit)| {- if cpu_ref_topp > 0.0 {- logit- } else {- f32::NEG_INFINITY- }- })- .collect::<Vec<f32>>();-- let gpu_processed_logits = &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];-- assert_eq!(&cpu_processed_logits, gpu_processed_logits);- }-- println!("✓ Topp processor gpu cpu match (topp={})", TOPP);-}-#[test]fn test_minp_gpu_cpu_match() {use rand::{RngExt, SeedableRng, rngs::StdRng};@@ -1321,69 +812,3 @@ fn test_minp_sampling_match_small() {fn test_minp_sampling_match_large() {test_minp_sampling_exact_match(32, 0.05, 4096);}--#[test]-fn test_gumbel_gpu_cpu_match() {- let context = match <Metal as Backend>::Context::new() {- Ok(ctx) => ctx,- Err(e) => {- println!("Skipping gumbel gpu cpu match test: {}", e);- return;- },- };-- const BATCH: usize = 7;- const VOCAB: usize = 16 * 1024 * 64;- const RTOL: f32 = 0.01;- const ATOL: f32 = 1e-6;-- let kernel = <<Metal as Backend>::Kernels as Kernels>::GumbelKernel::new(&context, DataType::F32, false)- .expect("Failed to create gumbel kernel");-- let logits = vec![0.0f32; BATCH * VOCAB];- let seeds: Vec<u64> = (0_u64..BATCH as u64).collect();-- let logits_buffer = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let seeds_buffer = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&seeds), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut gumbel_logits_buffer = context- .device- .new_buffer(BATCH * VOCAB * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();- kernel.encode(- Some(&logits_buffer),- &seeds_buffer,- &mut gumbel_logits_buffer,- BATCH as u32,- VOCAB as u32,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let result_ptr = gumbel_logits_buffer.contents().as_ptr() as *const f32;- let all_results = unsafe { std::slice::from_raw_parts(result_ptr, BATCH * VOCAB) };-- for (batch_idx, batch_seed) in seeds.iter().copied().enumerate() {- let results = &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];- for (logit_idx, gpu_logit_value) in results.iter().copied().enumerate() {- let cpu_logit_value = gumbel_float(batch_seed, revidx(logit_idx as u32));- let abs_diff = (cpu_logit_value - gpu_logit_value).abs();- let tolerance = ATOL + RTOL * cpu_logit_value.abs();- assert!(- abs_diff <= tolerance,- "Mismatch at batch {batch_idx} element {logit_idx}: CPU={cpu_logit_value} GPU={gpu_logit_value} (abs_diff={abs_diff}, tolerance={tolerance})"- );- }- }-- println!("✓ Gumbel cpu gpu match test passed (rtol: {:.1}%, atol: {:.1e})", RTOL * 100.0, ATOL);-}