Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

More cpu kernels (#246)

Aleksandr Golokozfb6f86208046
diff --git a/crates/uzu/src/backends/cpu/kernel/activation/mod.rs b/crates/uzu/src/backends/cpu/kernel/activation/mod.rs
index 3e9730314..e70a83196 100644
--- a/crates/uzu/src/backends/cpu/kernel/activation/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/activation/mod.rs
@@ -1 +1,49 @@
+use num_traits::Float;
+
+use crate::ArrayElement;
+
pub mod activation;
+
+const GELU_K0: f32 = 0.044715f32;
+const GELU_K1: f32 = 0.7978845608; // sqrt(2/pi)
+
+#[repr(u32)]
+pub enum ActivationType {
+ SiLU,
+ GELU,
+}
+
+impl TryFrom<u32> for ActivationType {
+ type Error = ();
+
+ fn try_from(value: u32) -> Result<Self, Self::Error> {
+ match value {
+ 0 => Ok(Self::SiLU),
+ 1 => Ok(Self::GELU),
+ _ => Err(()),
+ }
+ }
+}
+
+fn silu<T: Float>(x: T) -> T {
+ let x_float = x.to_f32().unwrap();
+ let y_float = x_float / (1.0f32 + (-x_float).exp());
+ T::from(y_float).unwrap()
+}
+
+fn gelu<T: Float>(x: T) -> T {
+ let x_float = x.to_f32().unwrap();
+ let tan_arg = GELU_K1 * (x_float + GELU_K0 * x_float * x_float * x_float);
+ let y_float = 0.5f32 * x_float * (1.0f32 + tan_arg.tanh());
+ T::from(y_float).unwrap()
+}
+
+pub fn activate<T: Float>(
+ x: T,
+ activation_type: &ActivationType,
+) -> T {
+ match activation_type {
+ ActivationType::SiLU => silu(x),
+ ActivationType::GELU => gelu(x),
+ }
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs
index b831f6dbc..f76ff38d6 100644
--- a/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs
+++ b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs
@@ -2,7 +2,10 @@ use dsl::kernel;
use half::{bf16, f16};
use num_traits::Float;
-use crate::ArrayElement;
+use crate::{
+ Activation, ArrayElement,
+ backends::cpu::kernel::activation::{ActivationType, activate},
+};
#[kernel(MlpGateActMul)]
#[variants(T, f32, f16, bf16)]
@@ -13,5 +16,21 @@ pub fn mlp_gate_act_mul<T: ArrayElement + Float>(
#[allow(unused)] m: i32,
#[allow(unused)] act_type: u32,
) {
- todo!()
+ let activation_type = match ActivationType::try_from(act_type) {
+ Ok(a) => a,
+ Err(_) => return,
+ };
+
+ for j in 0..h {
+ for row in 0..m {
+ let base = row * 2 * h;
+ unsafe {
+ let up: T = *fused_up.add((base + j) as usize);
+ let gate: T = *fused_up.add((base + h + j) as usize);
+ let g: T = activate(gate, &activation_type);
+ let out_float = up.to_f32().unwrap() * g.to_f32().unwrap();
+ *hidden.add((row * h + j) as usize) = T::from(out_float).unwrap();
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/gumbel.rs b/crates/uzu/src/backends/cpu/kernel/sampling/gumbel.rs
index 5c2eb686c..569b113a6 100644
--- a/crates/uzu/src/backends/cpu/kernel/sampling/gumbel.rs
+++ b/crates/uzu/src/backends/cpu/kernel/sampling/gumbel.rs
@@ -2,21 +2,36 @@ use dsl::kernel;
use half::{bf16, f16};
use num_traits::Float;
-use crate::ArrayElement;
+use crate::{
+ ArrayElement,
+ language_model::gumbel::{gumbel_float, revidx},
+};
#[kernel(Gumbel)]
#[variants(T, f32, f16, bf16)]
pub fn gumbel<T: ArrayElement + Float>(
- #[allow(unused)]
- #[optional(!in_place)]
- logits: Option<*const T>,
- #[allow(unused)] batch_seeds: *const u64,
- #[allow(unused)] processed_logits: *mut T,
- #[allow(unused)] batch_size: u32,
- #[allow(unused)] vocab_size: u32,
- #[allow(unused)]
- #[specialize]
- in_place: bool,
+ #[optional(!in_place)] logits: Option<*const T>,
+ batch_seeds: *const u64,
+ processed_logits: *mut T,
+ batch_size: u32,
+ vocab_size: u32,
+ #[specialize] in_place: bool,
) {
- todo!()
+ let logits: *const T = match in_place {
+ true => processed_logits,
+ false => logits.unwrap(),
+ };
+
+ for batch_idx in 0..batch_size as usize {
+ let batch_start = batch_idx * vocab_size as usize;
+ let rng_seed = unsafe { *batch_seeds.add(batch_idx) };
+ for vocab_idx in 0..vocab_size as usize {
+ let global_idx = batch_start + vocab_idx;
+ unsafe {
+ let logit = (*logits.add(global_idx)).to_f32().unwrap();
+ let gumbel_noise = gumbel_float(rng_seed, revidx(vocab_idx as u32));
+ *processed_logits.add(global_idx) = T::from(logit + gumbel_noise).unwrap();
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/temperature.rs b/crates/uzu/src/backends/cpu/kernel/sampling/temperature.rs
index 8eaf0aa69..f27f3c96b 100644
--- a/crates/uzu/src/backends/cpu/kernel/sampling/temperature.rs
+++ b/crates/uzu/src/backends/cpu/kernel/sampling/temperature.rs
@@ -7,16 +7,25 @@ use crate::ArrayElement;
#[kernel(Temperature)]
#[variants(T, f32, f16, bf16)]
pub fn temperature<T: ArrayElement + Float>(
- #[allow(unused)]
- #[optional(!in_place)]
- logits: Option<*const T>,
- #[allow(unused)] processed_logits: *mut T,
- #[allow(unused)] batch_size: u32,
- #[allow(unused)] vocab_size: u32,
- #[allow(unused)] temperature: f32,
- #[allow(unused)]
- #[specialize]
- in_place: bool,
+ #[optional(!in_place)] logits: Option<*const T>,
+ processed_logits: *mut T,
+ batch_size: u32,
+ vocab_size: u32,
+ temperature: f32,
+ #[specialize] in_place: bool,
) {
- todo!()
+ let logits: *const T = match in_place {
+ true => processed_logits,
+ false => logits.unwrap(),
+ };
+
+ for batch_idx in 0..batch_size as usize {
+ for vocab_idx in 0..vocab_size as usize {
+ let global_idx = batch_idx * vocab_size as usize + vocab_idx;
+ unsafe {
+ let value: f32 = (*logits.add(global_idx)).to_f32().unwrap() / temperature;
+ *processed_logits.add(global_idx) = T::from(value).unwrap();
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/top_k.rs b/crates/uzu/src/backends/cpu/kernel/sampling/top_k.rs
index 872ccb890..56b0e42d3 100644
--- a/crates/uzu/src/backends/cpu/kernel/sampling/top_k.rs
+++ b/crates/uzu/src/backends/cpu/kernel/sampling/top_k.rs
@@ -7,16 +7,41 @@ use crate::ArrayElement;
#[kernel(TopK)]
#[variants(T, f32, f16, bf16)]
pub fn top_k<T: ArrayElement + Float>(
- #[allow(unused)]
- #[optional(!in_place)]
- logits: Option<*const T>,
- #[allow(unused)] processed_logits: *mut T,
- #[allow(unused)] batch_size: u32,
- #[allow(unused)] vocab_size: u32,
- #[allow(unused)] top_k: u32,
- #[allow(unused)]
- #[specialize]
- in_place: bool,
+ #[optional(!in_place)] logits: Option<*const T>,
+ processed_logits: *mut T,
+ batch_size: u32,
+ vocab_size: u32,
+ top_k: u32,
+ #[specialize] in_place: bool,
) {
- todo!()
+ if top_k == 0 || top_k > vocab_size {
+ return;
+ }
+
+ let logits: *const T = match in_place {
+ true => processed_logits,
+ false => logits.unwrap(),
+ };
+
+ let vocab_size = vocab_size as usize;
+ let top_k = top_k as usize;
+
+ for batch_idx in 0..batch_size as usize {
+ let batch_start = batch_idx * vocab_size;
+ let mut sorted_logits = unsafe { std::slice::from_raw_parts(logits.add(batch_start), vocab_size) }.to_vec();
+ sorted_logits.sort_unstable_by(|a, b| b.partial_cmp(a).unwrap());
+ let t_threshold = sorted_logits[top_k - 1];
+
+ // Mask everything below the threshold
+ for i in 0..vocab_size {
+ let logit_value = unsafe { *logits.add(batch_start + i) };
+ unsafe {
+ *processed_logits.add(batch_start + i) = if logit_value >= t_threshold {
+ logit_value
+ } else {
+ T::from(f32::NEG_INFINITY).unwrap()
+ };
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/sampling/top_p.rs b/crates/uzu/src/backends/cpu/kernel/sampling/top_p.rs
index 9746c3e0e..578333047 100644
--- a/crates/uzu/src/backends/cpu/kernel/sampling/top_p.rs
+++ b/crates/uzu/src/backends/cpu/kernel/sampling/top_p.rs
@@ -7,16 +7,62 @@ use crate::ArrayElement;
#[kernel(TopP)]
#[variants(T, f32, f16, bf16)]
pub fn top_p<T: ArrayElement + Float>(
- #[allow(unused)]
- #[optional(!in_place)]
- logits: Option<*const T>,
- #[allow(unused)] processed_logits: *mut T,
- #[allow(unused)] batch_size: u32,
- #[allow(unused)] vocab_size: u32,
- #[allow(unused)] top_p: f32,
- #[allow(unused)]
- #[specialize]
- in_place: bool,
+ #[optional(!in_place)] logits: Option<*const T>,
+ processed_logits: *mut T,
+ batch_size: u32,
+ vocab_size: u32,
+ top_p: f32,
+ #[specialize] in_place: bool,
) {
- todo!()
+ let logits: *const T = match in_place {
+ true => processed_logits,
+ false => logits.unwrap(),
+ };
+
+ let vocab_size = vocab_size as usize;
+
+ for batch_idx in 0..batch_size as usize {
+ let batch_start = batch_idx * vocab_size;
+
+ // Sort logits in descending order
+ let mut sorted_logits = unsafe { std::slice::from_raw_parts(logits.add(batch_start), vocab_size) }.to_vec();
+ sorted_logits.sort_unstable_by(|a, b| b.partial_cmp(a).unwrap());
+ let max_logit: f32 = sorted_logits[0].to_f32().unwrap();
+
+ // Compute softmax probabilities and find threshold via cumulative sum
+ let total_sum: f32 = sorted_logits
+ .iter()
+ .map(|&v| v.to_f32().unwrap())
+ .filter(|&v| v > f32::NEG_INFINITY)
+ .map(|v| (v - max_logit).exp())
+ .sum();
+
+ let mut cumulative: f32 = 0.0;
+ let mut threshold: f32 = sorted_logits[0].to_f32().unwrap();
+ for &logit in &sorted_logits {
+ let logit_f32 = logit.to_f32().unwrap();
+ if logit_f32 == f32::NEG_INFINITY {
+ break;
+ }
+ threshold = logit_f32;
+ cumulative += (logit_f32 - max_logit).exp() / total_sum;
+ if cumulative >= top_p {
+ break;
+ }
+ }
+
+ let t_threshold = T::from(threshold).unwrap();
+
+ // Mask everything below the threshold
+ for i in 0..vocab_size {
+ let logit_value = unsafe { *logits.add(batch_start + i) };
+ unsafe {
+ *processed_logits.add(batch_start + i) = if logit_value >= t_threshold {
+ logit_value
+ } else {
+ T::from(f32::NEG_INFINITY).unwrap()
+ };
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.h b/crates/uzu/src/backends/metal/kernel/activation/activation.h
index 431183667..5212972b2 100644
--- a/crates/uzu/src/backends/metal/kernel/activation/activation.h
+++ b/crates/uzu/src/backends/metal/kernel/activation/activation.h
@@ -8,7 +8,7 @@ enum ActivationType : uint {
template <typename T>
inline T silu(T x) {
float xf = float(x);
- float yf = xf / (1.0f + exp(-xf));
+ float yf = xf / (1.0f + metal::exp(-xf));
return T(yf);
}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
index 521cf3ca0..8a622144b 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
@@ -1,7 +1,6 @@
#include <metal_stdlib>
-#include "../definitions.metal"
-
#include "../activation/activation.h"
+#include "../definitions.metal"
template <typename T>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rs
index 9ea9927cc..849220505 100644
--- a/crates/uzu/tests/common/mod.rs
+++ b/crates/uzu/tests/common/mod.rs
@@ -95,3 +95,14 @@ macro_rules! for_each_backend {
}
}};
}
+
+#[macro_export]
+macro_rules! for_each_non_cpu_backend {
+ (|$B:ident| $body:expr) => {{
+ #[cfg(feature = "metal")]
+ {
+ type $B = uzu::backends::metal::Metal;
+ $body
+ }
+ }};
+}
diff --git a/crates/uzu/tests/kernel/activation_mul_test.rs b/crates/uzu/tests/kernel/activation_mul_test.rs
new file mode 100644
index 000000000..4bbbce9b9
--- /dev/null
+++ b/crates/uzu/tests/kernel/activation_mul_test.rs
@@ -0,0 +1,119 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayElement, DataType,
+ array::ArrayContextExt,
+ backends::{
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels, kernel::MlpGateActMulKernel,
+ },
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ fused_up: Box<[T]>,
+ h: i32,
+ m: i32,
+ act_type: u32,
+}
+
+fn get_test_data<T: ArrayElement + Float>(act_type: u32) -> (Input<T>, Vec<T>) {
+ let h = 64i32;
+ let m = 4i32;
+ let fused_len = (m * 2 * h) as usize;
+
+ let mut fused_up: Vec<T> = vec![T::zero(); fused_len];
+ for i in 0..fused_len {
+ fused_up[i] = T::from((i as f32 * 0.1).sin() * 2.0f32).unwrap();
+ }
+
+ let input = Input {
+ fused_up: fused_up.into_boxed_slice(),
+ h,
+ m,
+ act_type,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+
+ (input, expected)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::MlpGateActMulKernel::new(&context, T::data_type())
+ .expect("Failed to create MlpGateActMulKernel");
+
+ let fused_len = (input.m * 2 * input.h) as usize;
+ let out_len = (input.m * input.h) as usize;
+ let fused_up_array = context.create_array_from(&[fused_len], &input.fused_up, "");
+ let hidden_array = context.create_array_uninitialized(&[out_len], T::data_type(), "");
+
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(
+ fused_up_array.buffer().borrow().deref(),
+ hidden_array.buffer().borrow_mut().deref_mut(),
+ input.h,
+ input.m,
+ input.act_type,
+ &mut command_buffer,
+ );
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+
+ hidden_array.as_slice().to_vec()
+}
+
+fn test<T: ArrayElement + Float + Debug + Display>(act_type: u32) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.02f32
+ } else {
+ 1e-5
+ };
+
+ let (input, expected) = get_test_data::<T>(act_type);
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ let msg = format!("Results are not equal for backend {}", std::any::type_name::<B>());
+ assert_eq_float::<T>(&expected, &output, eps, &msg);
+ });
+}
+
+#[test]
+fn test_silu_f32() {
+ test::<f32>(0);
+}
+
+#[test]
+fn test_silu_f16() {
+ test::<f16>(0);
+}
+
+#[test]
+fn test_silu_bf16() {
+ test::<bf16>(0);
+}
+
+#[test]
+fn test_gelu_f32() {
+ test::<f32>(1);
+}
+
+#[test]
+fn test_gelu_f16() {
+ test::<f16>(1);
+}
+
+#[test]
+fn test_gelu_bf16() {
+ test::<bf16>(1);
+}
diff --git a/crates/uzu/tests/kernel/attention_single_pass_test.rs b/crates/uzu/tests/kernel/attention_single_pass_test.rs
index 2db0eb818..37e3a2070 100644
--- a/crates/uzu/tests/kernel/attention_single_pass_test.rs
+++ b/crates/uzu/tests/kernel/attention_single_pass_test.rs
@@ -171,7 +171,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
1e-5
};
- for_each_backend!(|B| {
+ for_each_non_cpu_backend!(|B| {
let output = get_output::<T, B>(input);
let msg = format!(
"AttentionSinglePass failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={}, mask={})",
diff --git a/crates/uzu/tests/kernel/attention_update_kv_cache_test.rs b/crates/uzu/tests/kernel/attention_update_kv_cache_test.rs
index 98fb1c8f2..06666b65d 100644
--- a/crates/uzu/tests/kernel/attention_update_kv_cache_test.rs
+++ b/crates/uzu/tests/kernel/attention_update_kv_cache_test.rs
@@ -222,7 +222,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
1e-6
};
- for_each_backend!(|B| {
+ for_each_non_cpu_backend!(|B| {
let (key_cache_out, value_cache_out) = get_output::<T, B>(input);
let backend_name = std::any::type_name::<B>();
assert_eq_float::<T>(
diff --git a/crates/uzu/tests/kernel/mod.rs b/crates/uzu/tests/kernel/mod.rs
index 87a002059..1b6734c99 100644
--- a/crates/uzu/tests/kernel/mod.rs
+++ b/crates/uzu/tests/kernel/mod.rs
@@ -2,12 +2,14 @@
#[path = "../common/mod.rs"]
mod common;
+mod activation_mul_test;
mod attention_single_pass_test;
mod attention_update_kv_cache_test;
mod full_precision_embedding_test;
mod qk_norm_test;
mod rms_norm_test;
mod rope_test;
+mod sampling;
mod tensor_add_bias_test;
mod tensor_add_swap_test;
mod tensor_copy_test;
diff --git a/crates/uzu/tests/kernel/qk_norm_test.rs b/crates/uzu/tests/kernel/qk_norm_test.rs
index b587efecc..2374b1b29 100644
--- a/crates/uzu/tests/kernel/qk_norm_test.rs
+++ b/crates/uzu/tests/kernel/qk_norm_test.rs
@@ -174,7 +174,7 @@ fn test_internal<
1e-5
};
- for_each_backend!(|B| {
+ for_each_non_cpu_backend!(|B| {
let output = get_output::<B, InputT, ScaleT, OutputT, AccumT>(input);
let msg = format!(
"QKNorm kernel test failed with backend={}, head_offset={}, head_count={}, full_layer={}",
@@ -227,7 +227,7 @@ fn test_addressing<
let k_start = num_q_heads * head_dim;
let qkv_len = input.qkv.len();
- for_each_backend!(|B| {
+ for_each_non_cpu_backend!(|B| {
let output = get_output::<B, InputT, ScaleT, OutputT, AccumT>(&input);
// Q heads should be modified (different from input)
diff --git a/crates/uzu/tests/kernel/rms_norm_test.rs b/crates/uzu/tests/kernel/rms_norm_test.rs
index 3f0b13ad2..be4345970 100644
--- a/crates/uzu/tests/kernel/rms_norm_test.rs
+++ b/crates/uzu/tests/kernel/rms_norm_test.rs
@@ -189,7 +189,7 @@ fn test_internal<
1e-5
};
- for_each_backend!(|B| {
+ for_each_non_cpu_backend!(|B| {
let (output, _, _) = get_output::<B, InputT, ScaleT, OutputT, AccumT>(input);
let msg = format!(
"RMSNorm kernel test failed with backend={}, full_layer={}, in_place={}",
diff --git a/crates/uzu/tests/kernel/rope_test.rs b/crates/uzu/tests/kernel/rope_test.rs
index be119b7bb..453681955 100644
--- a/crates/uzu/tests/kernel/rope_test.rs
+++ b/crates/uzu/tests/kernel/rope_test.rs
@@ -120,7 +120,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(input: &Input<T>) {
let (expected_queries, expected_keys) = get_output::<T, Cpu>(input);
- for_each_backend!(|B| {
+ for_each_non_cpu_backend!(|B| {
let (queries, keys) = get_output::<T, B>(input);
let msg = format!("Rope queries test failed with backend={}", std::any::type_name::<B>(),);
assert_eq_float::<T>(&expected_queries, &queries, eps, &msg);
diff --git a/crates/uzu/tests/kernel/sampling/gumbel_test.rs b/crates/uzu/tests/kernel/sampling/gumbel_test.rs
new file mode 100644
index 000000000..3bb528631
--- /dev/null
+++ b/crates/uzu/tests/kernel/sampling/gumbel_test.rs
@@ -0,0 +1,164 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayElement,
+ array::ArrayContextExt,
+ backends::{
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels, kernel::GumbelKernel,
+ },
+ cpu::Cpu,
+ },
+ language_model::gumbel::{gumbel_float, revidx},
+};
+
+struct Input<T: ArrayElement + Float> {
+ logits: Box<[T]>,
+ seeds: Box<[u64]>,
+ batch_size: u32,
+ vocab_size: u32,
+ in_place: bool,
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ batch_size: u32,
+ vocab_size: u32,
+ in_place: bool,
+) -> (Input<T>, Vec<T>) {
+ let len = (batch_size * vocab_size) as usize;
+ let mut logits: Vec<T> = vec![T::zero(); len];
+ for i in 0..len {
+ logits[i] = T::from((i as f32 * 0.1).sin() * 2.0f32).unwrap();
+ }
+
+ let seeds: Vec<u64> = (0..batch_size as u64).collect();
+
+ let input = Input {
+ logits: logits.into_boxed_slice(),
+ seeds: seeds.into_boxed_slice(),
+ batch_size,
+ vocab_size,
+ in_place,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+
+ (input, expected)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::GumbelKernel::new(&context, T::data_type(), input.in_place)
+ .expect("Failed to create GumbelKernel");
+
+ let len = (input.batch_size * input.vocab_size) as usize;
+ let logits_array = context.create_array_from(&[len], &input.logits, "");
+ let logits_array_buffer_rc = logits_array.buffer();
+ let logits_array_borrow = logits_array_buffer_rc.borrow();
+ let logits_array_deref = logits_array_borrow.deref();
+ let logits_buffer = (!input.in_place).then(|| logits_array_deref);
+
+ let seeds_array = context.create_array_from(&[input.batch_size as usize], &input.seeds, "");
+
+ let output_array = match input.in_place {
+ true => context.create_array_from(&[len], &input.logits, ""),
+ false => context.create_array_uninitialized(&[len], T::data_type(), ""),
+ };
+
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(
+ logits_buffer,
+ seeds_array.buffer().borrow().deref(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.batch_size,
+ input.vocab_size,
+ &mut command_buffer,
+ );
+
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+// Verify determinism: same seeds produce same output.
+fn test_determinism<T: ArrayElement + Float + Debug + Display>(
+ batch_size: u32,
+ vocab_size: u32,
+) {
+ let (input, output1) = get_test_data::<T>(batch_size, vocab_size, false);
+ let output2 = get_output::<T, Cpu>(&input);
+ assert_eq!(output1, output2, "Determinism check failed");
+}
+
+fn test_internal<T: ArrayElement + Float + Display>(
+ batch_size: usize,
+ vocab_size: usize,
+) {
+ const RTOL: f32 = 0.01;
+ const ATOL: f32 = 1e-6;
+
+ let logits = vec![T::zero(); batch_size * vocab_size];
+ let seeds: Vec<u64> = (0_u64..batch_size as u64).collect();
+
+ let input = Input {
+ logits: logits.into_boxed_slice(),
+ seeds: seeds.clone().into_boxed_slice(),
+ batch_size: batch_size as u32,
+ vocab_size: vocab_size as u32,
+ in_place: false,
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+
+ for (batch_idx, batch_seed) in seeds.iter().copied().enumerate() {
+ let results = &output[batch_idx * vocab_size..(batch_idx + 1) * vocab_size];
+ for (logit_idx, gpu_logit_value) in results.iter().copied().enumerate() {
+ let cpu_logit_value = T::from(gumbel_float(batch_seed, revidx(logit_idx as u32))).unwrap();
+ let abs_diff = (cpu_logit_value - gpu_logit_value).abs().to_f32().unwrap();
+ let tolerance = ATOL + RTOL * cpu_logit_value.abs().to_f32().unwrap();
+ assert!(
+ abs_diff <= tolerance,
+ "Mismatch at batch {batch_idx} element {logit_idx}: CPU={cpu_logit_value} GPU={gpu_logit_value} (abs_diff={abs_diff}, tolerance={tolerance})"
+ );
+ }
+ }
+ });
+}
+
+#[test]
+fn test_determinism_f32() {
+ test_determinism::<f32>(4, 128);
+}
+
+#[test]
+fn test_determinism_f16() {
+ test_determinism::<f16>(4, 128);
+}
+
+#[test]
+fn test_determinism_bf16() {
+ test_determinism::<bf16>(4, 128);
+}
+
+#[test]
+fn test_f32() {
+ test_internal::<f32>(7, 16 * 1024 * 64);
+}
+
+#[test]
+fn test_f16() {
+ test_internal::<f16>(7, 16 * 1024 * 64);
+}
+
+#[test]
+fn test_bf16() {
+ test_internal::<bf16>(7, 16 * 1024 * 64);
+}
diff --git a/crates/uzu/tests/kernel/sampling/mod.rs b/crates/uzu/tests/kernel/sampling/mod.rs
new file mode 100644
index 000000000..d1b51e5ff
--- /dev/null
+++ b/crates/uzu/tests/kernel/sampling/mod.rs
@@ -0,0 +1,4 @@
+mod gumbel_test;
+mod temperature_test;
+mod top_k_test;
+mod top_p_test;
diff --git a/crates/uzu/tests/kernel/sampling/temperature_test.rs b/crates/uzu/tests/kernel/sampling/temperature_test.rs
new file mode 100644
index 000000000..53bbff087
--- /dev/null
+++ b/crates/uzu/tests/kernel/sampling/temperature_test.rs
@@ -0,0 +1,158 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayElement, DataType,
+ array::ArrayContextExt,
+ backends::{
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels, kernel::TemperatureKernel,
+ },
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ logits: Box<[T]>,
+ batch_size: u32,
+ vocab_size: u32,
+ temperature: f32,
+ in_place: bool,
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ batch_size: u32,
+ vocab_size: u32,
+ temperature: f32,
+ in_place: bool,
+) -> (Input<T>, Vec<T>) {
+ let len = (batch_size * vocab_size) as usize;
+ let mut logits: Vec<T> = vec![T::zero(); len];
+ for i in 0..len {
+ logits[i] = T::from((i as f32 * 0.1).sin() * 2.0f32).unwrap();
+ }
+
+ let input = Input {
+ logits: logits.into_boxed_slice(),
+ batch_size,
+ vocab_size,
+ temperature,
+ in_place,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+
+ (input, expected)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::TemperatureKernel::new(&context, T::data_type(), input.in_place)
+ .expect("Failed to create TemperatureKernel");
+
+ let len = (input.batch_size * input.vocab_size) as usize;
+ let logits_array = context.create_array_from(&[len], &input.logits, "");
+ let logits_array_buffer_rc = logits_array.buffer();
+ let logits_array_borrow = logits_array_buffer_rc.borrow();
+ let logits_array_deref = logits_array_borrow.deref();
+ let logits_buffer = (!input.in_place).then(|| logits_array_deref);
+ let output_array = match input.in_place {
+ true => context.create_array_from(&[len], &input.logits, ""),
+ false => context.create_array_uninitialized(&[len], T::data_type(), ""),
+ };
+
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(
+ logits_buffer,
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.batch_size,
+ input.vocab_size,
+ input.temperature,
+ &mut command_buffer,
+ );
+
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ batch_size: u32,
+ vocab_size: u32,
+ temperature: f32,
+ in_place: bool,
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.02f32
+ } else {
+ 1e-5
+ };
+
+ let (input, expected) = get_test_data::<T>(batch_size, vocab_size, temperature, in_place);
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ let msg = format!("Results are not equal for backend {}", std::any::type_name::<B>());
+ assert_eq_float::<T>(&expected, &output, eps, &msg);
+ });
+}
+
+// Out-of-place tests
+#[test]
+fn test_f32() {
+ test_internal::<f32>(4, 128, 0.7, false);
+}
+
+#[test]
+fn test_f16() {
+ test_internal::<f16>(4, 128, 0.7, false);
+}
+
+#[test]
+fn test_bf16() {
+ test_internal::<bf16>(4, 128, 0.7, false);
+}
+
+// In-place tests
+#[test]
+fn test_in_place_f32() {
+ test_internal::<f32>(4, 128, 0.7, true);
+}
+
+#[test]
+fn test_in_place_f16() {
+ test_internal::<f16>(4, 128, 0.7, true);
+}
+
+#[test]
+fn test_in_place_bf16() {
+ test_internal::<bf16>(4, 128, 0.7, true);
+}
+
+// Edge cases
+#[test]
+fn test_temperature_1_f32() {
+ test_internal::<f32>(2, 64, 1.0, false);
+}
+
+#[test]
+fn test_high_temperature_f32() {
+ test_internal::<f32>(2, 64, 100.0, false);
+}
+
+#[test]
+fn test_low_temperature_f32() {
+ test_internal::<f32>(2, 64, 0.1, false);
+}
+
+#[test]
+fn test_single_batch_f32() {
+ test_internal::<f32>(1, 256, 0.7, false);
+}
diff --git a/crates/uzu/tests/kernel/sampling/top_k_test.rs b/crates/uzu/tests/kernel/sampling/top_k_test.rs
new file mode 100644
index 000000000..7332e26fb
--- /dev/null
+++ b/crates/uzu/tests/kernel/sampling/top_k_test.rs
@@ -0,0 +1,192 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use rand::{RngExt, SeedableRng, rngs::StdRng};
+use uzu::{
+ ArrayElement, DataType,
+ array::ArrayContextExt,
+ backends::{
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels, kernel::TopKKernel,
+ },
+ cpu::Cpu,
+ },
+};
+
+struct Input<T: ArrayElement + Float> {
+ logits: Box<[T]>,
+ batch_size: u32,
+ vocab_size: u32,
+ top_k: u32,
+ in_place: bool,
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ batch_size: u32,
+ vocab_size: u32,
+ top_k: u32,
+ in_place: bool,
+) -> (Input<T>, Vec<T>) {
+ let len = (batch_size * vocab_size) as usize;
+ let mut rng = StdRng::seed_from_u64(42);
+ let mut logits: Vec<T> = vec![T::zero(); len];
+ for x in logits.iter_mut() {
+ *x = T::from(rng.random_range(-1024.0f32..1024.0f32)).unwrap();
+ }
+
+ let input = Input {
+ logits: logits.into_boxed_slice(),
+ batch_size,
+ vocab_size,
+ top_k,
+ in_place,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+
+ (input, expected)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::TopKKernel::new(&context, T::data_type(), input.in_place)
+ .expect("Failed to create TopKKernel");
+
+ let len = (input.batch_size * input.vocab_size) as usize;
+ let logits_array = context.create_array_from(&[len], &input.logits, "");
+ let logits_array_buffer_rc = logits_array.buffer();
+ let logits_array_borrow = logits_array_buffer_rc.borrow();
+ let logits_array_deref = logits_array_borrow.deref();
+ let logits_buffer = (!input.in_place).then(|| logits_array_deref);
+ let output_array = match input.in_place {
+ true => context.create_array_from(&[len], &input.logits, ""),
+ false => context.create_array_uninitialized(&[len], T::data_type(), ""),
+ };
+
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(
+ logits_buffer,
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.batch_size,
+ input.vocab_size,
+ input.top_k,
+ &mut command_buffer,
+ );
+
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+fn assert_top_k_equal<T: ArrayElement + Float + Display>(
+ expected: &[T],
+ actual: &[T],
+ vocab_size: u32,
+ top_k: u32,
+ eps: f32,
+ msg: &str,
+) {
+ assert_eq!(expected.len(), actual.len(), "Slices size mismatch");
+ let vocab_size = vocab_size as usize;
+ let num_batches = expected.len() / vocab_size;
+
+ for batch_idx in 0..num_batches {
+ let start = batch_idx * vocab_size;
+ let end = start + vocab_size;
+
+ // Both should keep approximately top_k values and mask the rest
+ let expected_kept: usize = expected[start..end].iter().filter(|v| v.is_finite()).count();
+ let actual_kept: usize = actual[start..end].iter().filter(|v| v.is_finite()).count();
+
+ // Allow small differences due to binary search convergence
+ let diff = (expected_kept as i64 - actual_kept as i64).unsigned_abs();
+ assert!(
+ diff <= 3,
+ "{msg}. Batch {batch_idx}: expected {expected_kept} kept values, got {actual_kept} (top_k={top_k})"
+ );
+
+ // For values that are kept by both, they should match the original values
+ for i in start..end {
+ let e = expected[i].to_f32().unwrap();
+ let a = actual[i].to_f32().unwrap();
+ if e.is_finite() && a.is_finite() {
+ let d = (e - a).abs();
+ assert!(d < eps, "{msg}. Mismatch at index {i}: expected {e}, got {a}, diff {d}");
+ }
+ }
+ }
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ batch_size: u32,
+ vocab_size: u32,
+ top_k: u32,
+ in_place: bool,
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-2
+ } else {
+ 1e-5
+ };
+
+ let (input, expected) = get_test_data::<T>(batch_size, vocab_size, top_k, in_place);
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ let msg = format!("Results are not equal for backend {}", std::any::type_name::<B>());
+ assert_top_k_equal::<T>(&expected, &output, vocab_size, top_k, eps, &msg);
+ });
+}
+
+// Out-of-place tests (same data as sampling_test::test_topk_gpu_cpu_match)
+#[test]
+fn test_f32() {
+ test_internal::<f32>(4, 1024, 16, false);
+}
+
+#[test]
+fn test_f16() {
+ test_internal::<f16>(4, 1024, 16, false);
+}
+
+#[test]
+fn test_bf16() {
+ test_internal::<bf16>(4, 1024, 16, false);
+}
+
+// In-place tests
+#[test]
+fn test_in_place_f32() {
+ test_internal::<f32>(4, 1024, 16, true);
+}
+
+#[test]
+fn test_in_place_f16() {
+ test_internal::<f16>(4, 1024, 16, true);
+}
+
+#[test]
+fn test_in_place_bf16() {
+ test_internal::<bf16>(4, 1024, 16, true);
+}
+
+// Edge cases
+#[test]
+fn test_single_batch_f32() {
+ test_internal::<f32>(1, 1024, 16, false);
+}
+
+#[test]
+fn test_top_k_1_f32() {
+ test_internal::<f32>(4, 1024, 1, false);
+}
+
+#[test]
+fn test_top_k_equals_vocab_f32() {
+ test_internal::<f32>(4, 1024, 1024, false);
+}
diff --git a/crates/uzu/tests/kernel/sampling/top_p_test.rs b/crates/uzu/tests/kernel/sampling/top_p_test.rs
new file mode 100644
index 000000000..4c773e420
--- /dev/null
+++ b/crates/uzu/tests/kernel/sampling/top_p_test.rs
@@ -0,0 +1,519 @@
+use std::{
+ collections::HashSet,
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use rand::{RngExt, SeedableRng, rngs::StdRng, seq::SliceRandom};
+use uzu::{
+ ArrayElement, DataType,
+ array::ArrayContextExt,
+ backends::{
+ common::{
+ Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
+ CommandBufferPending, Context, Kernels,
+ kernel::{TopPKernel, sampling::SamplingKernel},
+ },
+ cpu::Cpu,
+ },
+ session::parameter::SamplingMethod,
+};
+
+const TEST_SAMPLING_SEED: u64 = 42;
+
+struct Input<T: ArrayElement + Float> {
+ logits: Box<[T]>,
+ batch_size: u32,
+ vocab_size: u32,
+ top_p: f32,
+ in_place: bool,
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ batch_size: u32,
+ vocab_size: u32,
+ top_p: f32,
+ in_place: bool,
+) -> (Input<T>, Vec<T>) {
+ let len = (batch_size * vocab_size) as usize;
+ let mut rng = StdRng::seed_from_u64(42);
+ let mut logits: Vec<T> = vec![T::zero(); len];
+ for x in logits.iter_mut() {
+ *x = T::from(rng.random_range(-16.0f32..16.0f32)).unwrap();
+ }
+
+ let input = Input {
+ logits: logits.into_boxed_slice(),
+ batch_size,
+ vocab_size,
+ top_p,
+ in_place,
+ };
+
+ let expected = get_output::<T, Cpu>(&input);
+
+ (input, expected)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = <B as Backend>::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::TopPKernel::new(&context, T::data_type(), input.in_place)
+ .expect("Failed to create TopPKernel");
+
+ let len = (input.batch_size * input.vocab_size) as usize;
+ let logits_array = context.create_array_from(&[len], &input.logits, "");
+ let logits_array_buffer_rc = logits_array.buffer();
+ let logits_array_borrow = logits_array_buffer_rc.borrow();
+ let logits_array_deref = logits_array_borrow.deref();
+ let logits_buffer = (!input.in_place).then(|| logits_array_deref);
+ let output_array = match input.in_place {
+ true => context.create_array_from(&[len], &input.logits, ""),
+ false => context.create_array_uninitialized(&[len], T::data_type(), ""),
+ };
+
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(
+ logits_buffer,
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.batch_size,
+ input.vocab_size,
+ input.top_p,
+ &mut command_buffer,
+ );
+
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+fn assert_top_p_equal<T: ArrayElement + Float + Display>(
+ expected: &[T],
+ actual: &[T],
+ vocab_size: u32,
+ eps: f32,
+ msg: &str,
+) {
+ assert_eq!(expected.len(), actual.len(), "Slices size mismatch");
+ let vocab_size = vocab_size as usize;
+ let num_batches = expected.len() / vocab_size;
+
+ for batch_idx in 0..num_batches {
+ let start = batch_idx * vocab_size;
+ let end = start + vocab_size;
+
+ let expected_kept: usize = expected[start..end].iter().filter(|v| v.is_finite()).count();
+ let actual_kept: usize = actual[start..end].iter().filter(|v| v.is_finite()).count();
+
+ // Allow small differences due to binary search convergence
+ let diff = (expected_kept as i64 - actual_kept as i64).unsigned_abs();
+ assert!(diff <= 3, "{msg}. Batch {batch_idx}: expected {expected_kept} kept values, got {actual_kept}");
+
+ // For values that are kept by both, they should match the original values
+ for i in start..end {
+ let e = expected[i].to_f32().unwrap();
+ let a = actual[i].to_f32().unwrap();
+ if e.is_finite() && a.is_finite() {
+ let d = (e - a).abs();
+ assert!(d < eps, "{msg}. Mismatch at index {i}: expected {e}, got {a}, diff {d}");
+ }
+ }
+ }
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ batch_size: u32,
+ vocab_size: u32,
+ top_p: f32,
+ in_place: bool,
+) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-2
+ } else {
+ 1e-5
+ };
+
+ let (input, expected) = get_test_data::<T>(batch_size, vocab_size, top_p, in_place);
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ let msg = format!("Results are not equal for backend {}", std::any::type_name::<B>());
+ assert_top_p_equal::<T>(&expected, &output, vocab_size, eps, &msg);
+ });
+}
+
+// Out-of-place tests (same data as sampling_test::test_topp_gpu_cpu_match)
+#[test]
+fn test_f32() {
+ test_internal::<f32>(4, 1024, 0.9, false);
+}
+
+#[test]
+fn test_f16() {
+ test_internal::<f16>(4, 1024, 0.9, false);
+}
+
+#[test]
+fn test_bf16() {
+ test_internal::<bf16>(4, 1024, 0.9, false);
+}
+
+// In-place tests
+#[test]
+fn test_in_place_f32() {
+ test_internal::<f32>(4, 1024, 0.9, true);
+}
+
+#[test]
+fn test_in_place_f16() {
+ test_internal::<f16>(4, 1024, 0.9, true);
+}
+
+#[test]
+fn test_in_place_bf16() {
+ test_internal::<bf16>(4, 1024, 0.9, true);
+}
+
+// Edge cases
+#[test]
+fn test_single_batch_f32() {
+ test_internal::<f32>(1, 1024, 0.9, false);
+}
+
+#[test]
+fn test_top_p_very_small() {
+ test_internal::<f32>(4, 1024, 0.01, false);
+}
+
+#[test]
+fn test_top_p_near_1() {
+ test_internal::<f32>(4, 1024, 0.99, false);
+}
+
+// --- Sampling-level top-p tests (test the full SamplingKernel with top_p) ---
+
+fn cpu_reference_top_p(
+ row_logits: &[f32],
+ top_p: f32,
+) -> Vec<f32> {
+ let max_logit = row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
+ let exp_vals: Vec<f32> = row_logits.iter().map(|&x| (x - max_logit).exp()).collect();
+ let sum_exp: f32 = exp_vals.iter().sum();
+ let probs: Vec<f32> = exp_vals.iter().map(|&x| x / sum_exp).collect();
+
+ let mut indexed: Vec<(usize, f32)> = probs.iter().copied().enumerate().collect();
+ indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
+
+ let mut cum = 0.0f32;
+ let mut keep = vec![false; row_logits.len()];
+ for &(idx, p) in &indexed {
+ if cum >= top_p {
+ break;
+ }
+ keep[idx] = true;
+ cum += p;
+ }
+
+ // Renormalise kept probabilities
+ let kept_sum: f32 = probs.iter().zip(keep.iter()).filter(|&(_, &k)| k).map(|(&p, _)| p).sum();
+ probs
+ .iter()
+ .zip(keep.iter())
+ .map(|(&p, &k)| {
+ if k {
+ p / kept_sum
+ } else {
+ 0.0
+ }
+ })
+ .collect()
+}
+
+fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(
+ batch_size: usize,
+ k: usize,
+ vocab_size: usize,
+) {
+ let context = <B as Backend>::Context::new().expect("Failed to create Context");
+
+ let p = (k as f32) * 0.1;
+
+ let kernel = SamplingKernel::<B>::new(&context, DataType::F32, batch_size, vocab_size)
+ .expect("Failed to create sampling kernel");
+
+ // Build probability table
+ let low_prob = (1.0 - p) / (vocab_size - k) as f32;
+ let mut probs = vec![low_prob; batch_size * vocab_size];
+ let mut high_prob_token_sets: Vec<HashSet<usize>> = Vec::with_capacity(batch_size);
+
+ let mut rng = StdRng::seed_from_u64(42);
+ let mut all_token_ids: Vec<usize> = (0..vocab_size).collect();
+
+ for b in 0..batch_size {
+ all_token_ids.shuffle(&mut rng);
+ let high_prob_tokens: HashSet<usize> = all_token_ids[..k].iter().cloned().collect();
+
+ for &token_id in &high_prob_tokens {
+ probs[b * vocab_size + token_id] = 0.1;
+ }
+ high_prob_token_sets.push(high_prob_tokens);
+ }
+
+ let logits: Vec<f32> = probs
+ .iter()
+ .map(|&prob| {
+ if prob <= 0.0 {
+ -50.0
+ } else {
+ prob.ln()
+ }
+ })
+ .collect();
+
+ let output_array = context.create_array_uninitialized(&[batch_size], DataType::U32, "");
+
+ let num_samples = 1000;
+ let mut counter = vec![0i32; batch_size * vocab_size];
+
+ for draw in 0..num_samples {
+ let logits_array = context.create_array_from(&[batch_size * vocab_size], &logits, "");
+ let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; batch_size];
+ let seeds_array = context.create_array_from(&[batch_size], &seeds, "");
+
+ let mut command_buffer =
+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel
+ .encode(
+ logits_array.buffer().borrow_mut().deref_mut(),
+ Some(seeds_array.buffer().borrow().deref()),
+ 0,
+ None,
+ 0,
+ output_array.buffer().borrow_mut().deref_mut(),
+ SamplingMethod::Stochastic {
+ temperature: None,
+ top_k: None,
+ top_p: Some(p),
+ min_p: None,
+ },
+ batch_size,
+ vocab_size,
+ &mut command_buffer,
+ )
+ .expect("encode");
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+
+ let sampled_ids: &[u32] = output_array.as_slice();
+
+ for (i, &sampled_id) in sampled_ids.iter().enumerate() {
+ assert!((sampled_id as usize) < vocab_size, "Sampled token out of range");
+ counter[i * vocab_size + sampled_id as usize] += 1;
+ }
+ }
+
+ for i in 0..batch_size {
+ for j in 0..vocab_size {
+ if counter[i * vocab_size + j] > 0 {
+ assert!(
+ high_prob_token_sets[i].contains(&j),
+ "high_prob_token_sets[{}] does not contain {} (appeared {} times)",
+ i,
+ j,
+ counter[i * vocab_size + j]
+ );
+ }
+ }
+ }
+
+ println!(
+ "batch_size: {}, p: {:.1}, vocab_size: {}, accuracy test passed (backend: {}).",
+ batch_size,
+ p,
+ vocab_size,
+ std::any::type_name::<B>()
+ );
+}
+
+#[test]
+fn test_topp_sampling_match_small() {
+ for_each_non_cpu_backend!(|B| {
+ test_topp_sampling_from_prob_exact_match_internal::<B>(8, 10, 1024);
+ });
+}
+
+#[test]
+fn test_topp_sampling_match_large() {
+ for_each_non_cpu_backend!(|B| {
+ test_topp_sampling_from_prob_exact_match_internal::<B>(32, 50, 4096);
+ });
+}
+
+#[test]
+fn test_topp_sampling_statistical_large() {
+ for_each_non_cpu_backend!(|B| {
+ const BATCH: usize = 32;
+ const VOCAB: usize = 4096;
+ const NUM_DRAWS: usize = 10_000;
+ const TOP_P: f32 = 0.9;
+ const TOLERANCE_KL: f32 = 0.05;
+
+ let context = <B as Backend>::Context::new().expect("Failed to create Context");
+
+ let kernel =
+ SamplingKernel::<B>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");
+
+ let mut rng = StdRng::seed_from_u64(42);
+ let mut logits = vec![0.0f32; BATCH * VOCAB];
+ for x in logits.iter_mut() {
+ *x = rng.random_range(-6.0f32..6.0f32);
+ }
+
+ // Build Top-p renormalised target distribution per row
+ let mut probs = vec![0.0f32; BATCH * VOCAB];
+ for b in 0..BATCH {
+ let row_logits = &logits[b * VOCAB..(b + 1) * VOCAB];
+ let dist = cpu_reference_top_p(row_logits, TOP_P);
+ probs[b * VOCAB..(b + 1) * VOCAB].copy_from_slice(&dist);
+ }
+
+ let output_array = context.create_array_uninitialized(&[BATCH], DataType::U32, "");
+ let mut counters = vec![0u32; BATCH * VOCAB];
+
+ for draw in 0..NUM_DRAWS {
+ let logits_array = context.create_array_from(&[BATCH * VOCAB], &logits, "");
+ let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; BATCH];
+ let seeds_array = context.create_array_from(&[BATCH], &seeds, "");
+
+ let mut command_buffer =
+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+
+ kernel
+ .encode(
+ logits_array.buffer().borrow_mut().deref_mut(),
+ Some(seeds_array.buffer().borrow().deref()),
+ 0,
+ None,
+ 0,
+ output_array.buffer().borrow_mut().deref_mut(),
+ SamplingMethod::Stochastic {
+ temperature: None,
+ top_k: None,
+ top_p: Some(TOP_P),
+ min_p: None,
+ },
+ BATCH,
+ VOCAB,
+ &mut command_buffer,
+ )
+ .expect("encode");
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+
+ let sample_ids: &[u32] = output_array.as_slice();
+ for (b, &tok) in sample_ids.iter().enumerate() {
+ counters[b * VOCAB + tok as usize] += 1;
+ }
+ }
+
+ // Compute KL divergence per row
+ for b in 0..BATCH {
+ let mut kl = 0.0_f32;
+ for j in 0..VOCAB {
+ let expected = probs[b * VOCAB + j];
+ if expected < 1e-12 {
+ continue;
+ }
+ let observed = (counters[b * VOCAB + j] as f32) / (NUM_DRAWS as f32);
+ if observed > 0.0 {
+ kl += observed * (observed.ln() - expected.ln());
+ }
+ }
+ assert!(kl < TOLERANCE_KL, "Row {} KL {:.4} exceeded tolerance {:.3}", b, kl, TOLERANCE_KL);
+ }
+
+ println!(
+ "Large statistical Top-p test passed (KL < {:.3}, backend: {})",
+ TOLERANCE_KL,
+ std::any::type_name::<B>()
+ );
+ });
+}
+
+#[test]
+fn perf_topp_128k_vocab() {
+ for_each_non_cpu_backend!(|B| {
+ const BATCH: usize = 8;
+ const VOCAB: usize = 128000;
+ const TOP_P: f32 = 0.9;
+
+ let context = <B as Backend>::Context::new().expect("Failed to create Context");
+
+ let kernel =
+ SamplingKernel::<B>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");
+
+ let mut rng = StdRng::seed_from_u64(123);
+ let mut logits = vec![0.0f32; BATCH * VOCAB];
+ for x in logits.iter_mut() {
+ *x = rng.random_range(-6.0f32..6.0f32);
+ }
+
+ let logits_array = context.create_array_from(&[BATCH * VOCAB], &logits, "");
+ let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];
+ let seeds_array = context.create_array_from(&[BATCH], &seeds, "");
+ let output_array = context.create_array_uninitialized(&[BATCH], DataType::U32, "");
+
+ let mut command_buffer =
+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+
+ kernel
+ .encode(
+ logits_array.buffer().borrow_mut().deref_mut(),
+ Some(seeds_array.buffer().borrow().deref()),
+ 0,
+ None,
+ 0,
+ output_array.buffer().borrow_mut().deref_mut(),
+ SamplingMethod::Stochastic {
+ temperature: None,
+ top_k: None,
+ top_p: Some(TOP_P),
+ min_p: None,
+ },
+ BATCH,
+ VOCAB,
+ &mut command_buffer,
+ )
+ .expect("encode");
+
+ let host_timer = std::time::Instant::now();
+ let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
+
+ match completed.gpu_execution_time() {
+ Some(gpu_time) => {
+ println!(
+ "Top-p sampling perf (batch={}, vocab={}, backend={}): GPU={:.2} ms, Host-side={:.2} ms",
+ BATCH,
+ VOCAB,
+ std::any::type_name::<B>(),
+ gpu_time.as_secs_f64() * 1e3,
+ host_elapsed_ms
+ );
+ },
+ None => {
+ println!(
+ "Top-p sampling perf (batch={}, vocab={}, backend={}): Host-side={:.2} ms (GPU timing unavailable)",
+ BATCH,
+ VOCAB,
+ std::any::type_name::<B>(),
+ host_elapsed_ms
+ );
+ },
+ }
+
+ let sample_ids: &[u32] = output_array.as_slice();
+ for &tok in sample_ids {
+ assert!((tok as usize) < VOCAB, "Sampled id out of range");
+ }
+ });
+}
diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rs
index 6eed789db..9dfcd9b59 100644
--- a/crates/uzu/tests/sampling_test.rs
+++ b/crates/uzu/tests/sampling_test.rs
@@ -1,7 +1,7 @@
mod common;
use bytemuck;
use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
-use rand::{RngExt, seq::SliceRandom};
+use rand::seq::SliceRandom;
// for Vec::shuffle
use uzu::{
DataType,
@@ -10,52 +10,18 @@ use uzu::{
Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
CommandBufferPending, Context, Kernels,
kernel::{
- GumbelKernel, MinPKernel, TemperatureKernel, TopKKernel, TopPKernel,
+ MinPKernel, TemperatureKernel,
sampling::{ArgmaxStrategy, SamplingKernel},
},
},
metal::Metal,
},
- language_model::gumbel::{gumbel_float, revidx},
session::parameter::SamplingMethod,
};
// Constant seed for reproducible test results
const TEST_SAMPLING_SEED: u64 = 42;
-fn cpu_reference_top_p(
- row_logits: &[f32],
- top_p: f32,
-) -> Vec<f32> {
- // 1. soft‑max (unnormalised)
- let max_logit = row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
- let mut softmax: Vec<(usize, f32)> =
- row_logits.iter().enumerate().map(|(idx, &z)| (idx, (z - max_logit).exp())).collect();
- let total_mass: f32 = softmax.iter().map(|(_, p)| p).sum();
-
- // 2. sort descending
- softmax.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
-
- // 3. accumulate until cumulative ≥ top_p
- let mut cum = 0.0_f32;
- let mut cutoff = 0;
- let target = top_p * total_mass;
- while cutoff < softmax.len() && cum < target {
- cum += softmax[cutoff].1;
- cutoff += 1;
- }
-
- // 4. renormalise the kept mass
- let mut dist = vec![0.0f32; row_logits.len()];
- if cum > 0.0 {
- let renorm = 1.0 / cum;
- for (idx, p) in &softmax[..cutoff] {
- dist[*idx] = *p * renorm;
- }
- }
- dist
-}
-
fn cpu_reference_min_p(
row_logits: &[f32],
min_p: f32,
@@ -151,358 +117,6 @@ fn test_argmax_sampling() {
test_argmax_sampling_with_strategy(ArgmaxStrategy::SinglePass);
}
-fn test_topp_sampling_from_prob_exact_match(
- batch_size: usize,
- k: usize,
- vocab_size: usize,
-) {
- use std::collections::HashSet;
-
- use rand::{SeedableRng, rngs::StdRng};
-
- let context = match <Metal as Backend>::Context::new() {
- Ok(ctx) => ctx,
- Err(e) => {
- eprintln!("Skipping topP exact match test: {}", e);
- return;
- },
- };
-
- // Calculate top_p threshold
- let p = (k as f32) * 0.1;
-
- // Create kernel
- let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32, batch_size, vocab_size)
- .expect("Failed to create sampling kernel");
-
- // Build probability table
- let low_prob = (1.0 - p) / (vocab_size - k) as f32;
- let mut probs = vec![low_prob; batch_size * vocab_size];
- let mut high_prob_token_sets: Vec<HashSet<usize>> = Vec::with_capacity(batch_size);
-
- // Use std::sample equivalent (Fisher-Yates shuffle + take first k)
- let mut rng = StdRng::seed_from_u64(42); // Deterministic for reproducibility
- let mut all_token_ids: Vec<usize> = (0..vocab_size).collect();
-
- for b in 0..batch_size {
- // Equivalent to std::sample - shuffle and take first k
- all_token_ids.shuffle(&mut rng);
- let high_prob_tokens: HashSet<usize> = all_token_ids[..k].iter().cloned().collect();
-
- // Set selected tokens to high probability
- for &token_id in &high_prob_tokens {
- probs[b * vocab_size + token_id] = 0.1;
- }
- high_prob_token_sets.push(high_prob_tokens);
- }
-
- // Convert to logits more carefully to avoid -∞
- let logits: Vec<f32> = probs
- .iter()
- .map(|&prob| {
- if prob <= 0.0 {
- -50.0 // Very negative but finite value instead of -∞
- } else {
- prob.ln()
- }
- })
- .collect();
-
- // GPU buffers
- let mut output_buf = context
- .device
- .new_buffer(batch_size * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- // Counter for samples
- let num_samples = 1000;
- let mut counter = vec![0i32; batch_size * vocab_size];
-
- // Draw samples
- for draw in 0..num_samples {
- // Create fresh logits buffer since kernel mutates in-place
- let mut logits_buf = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; batch_size];
- let seeds_buf = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut command_buffer =
- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
- kernel
- .encode(
- &mut logits_buf,
- Some(&seeds_buf),
- 0,
- None,
- 0,
- &mut output_buf,
- SamplingMethod::Stochastic {
- temperature: None,
- top_k: None,
- top_p: Some(p),
- min_p: None,
- },
- batch_size,
- vocab_size,
- &mut command_buffer,
- )
- .expect("encode");
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let ptr = output_buf.contents().as_ptr() as *const u32;
- let sampled_ids = unsafe { std::slice::from_raw_parts(ptr, batch_size) };
-
- for (i, &sampled_id) in sampled_ids.iter().enumerate() {
- assert!((sampled_id as usize) < vocab_size, "Sampled token out of range");
- counter[i * vocab_size + sampled_id as usize] += 1;
- }
- }
-
- for i in 0..batch_size {
- for j in 0..vocab_size {
- if counter[i * vocab_size + j] > 0 {
- assert!(
- high_prob_token_sets[i].contains(&j),
- "high_prob_token_sets[{}] does not contain {} (appeared {} times)",
- i,
- j,
- counter[i * vocab_size + j]
- );
- }
- }
- }
-
- println!("batch_size: {}, p: {:.1}, vocab_size: {}, accuracy test passed.", batch_size, p, vocab_size);
-}
-
-#[test]
-fn test_topp_sampling_match_small() {
- test_topp_sampling_from_prob_exact_match(8, 10, 1024);
-}
-
-#[test]
-fn test_topp_sampling_match_large() {
- test_topp_sampling_from_prob_exact_match(32, 50, 4096);
-}
-
-#[test]
-fn test_topp_sampling_statistical_large() {
- use rand::{RngExt, SeedableRng, rngs::StdRng};
-
- // ===== 1. Create Metal context =====
- let context = match <Metal as Backend>::Context::new() {
- Ok(ctx) => ctx,
- Err(e) => {
- println!("Skipping large statistical test: {}", e);
- return;
- },
- };
-
- // ===== 2. Problem sizes =====
- const BATCH: usize = 32;
- const VOCAB: usize = 4096;
- const NUM_DRAWS: usize = 10_000;
- const TOP_P: f32 = 0.9;
- const TOLERANCE_KL: f32 = 0.05;
-
- // ===== 3. Build kernel =====
- let kernel =
- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");
-
- // ===== 4. Generate reproducible random logits =====
- let mut rng = StdRng::seed_from_u64(42);
- let mut logits = vec![0.0f32; BATCH * VOCAB];
- for x in logits.iter_mut() {
- // Uniform[-6, 6] (broad range avoids numerical under/overflow)
- *x = rng.random_range(-6.0f32..6.0f32);
- }
-
- // ===== 5. Build Top‑p renormalised target distribution per row =====
- let mut probs = vec![0.0f32; BATCH * VOCAB];
- for b in 0..BATCH {
- let row_logits = &logits[b * VOCAB..(b + 1) * VOCAB];
- let dist = cpu_reference_top_p(row_logits, TOP_P);
- probs[b * VOCAB..(b + 1) * VOCAB].copy_from_slice(&dist);
- }
-
- // ===== 6. Allocate output & counters =====
- let mut output_buf = context
- .device
- .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let mut counters = vec![0u32; BATCH * VOCAB];
-
- // ===== 8. Draw NUM_DRAWS samples =====
- for draw in 0..NUM_DRAWS {
- // Create fresh logits buffer since kernel mutates in-place
- let mut logits_buf = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; BATCH];
- let seeds_buf = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut command_buffer =
- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
-
- kernel
- .encode(
- &mut logits_buf,
- Some(&seeds_buf),
- 0,
- None,
- 0,
- &mut output_buf,
- SamplingMethod::Stochastic {
- temperature: None,
- top_k: None,
- top_p: Some(TOP_P),
- min_p: None,
- },
- BATCH,
- VOCAB,
- &mut command_buffer,
- )
- .expect("encode");
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let ptr = output_buf.contents().as_ptr() as *const u32;
- let sample_ids = unsafe { std::slice::from_raw_parts(ptr, BATCH) };
- for (b, &tok) in sample_ids.iter().enumerate() {
- counters[b * VOCAB + tok as usize] += 1;
- }
- }
-
- // ===== 9. Compute KL divergence per row =====
- for b in 0..BATCH {
- let mut kl = 0.0_f32;
- for j in 0..VOCAB {
- let expected = probs[b * VOCAB + j];
- if expected < 1e-12 {
- continue;
- } // ignore tiny
- let observed = (counters[b * VOCAB + j] as f32) / (NUM_DRAWS as f32);
- if observed > 0.0 {
- kl += observed * (observed.ln() - expected.ln());
- }
- }
- assert!(kl < TOLERANCE_KL, "Row {} KL {:.4} exceeded tolerance {:.3}", b, kl, TOLERANCE_KL);
- }
-
- println!("✓ Large statistical Top‑p test passed (KL < {:.3})", TOLERANCE_KL);
-}
-
-#[test]
-fn perf_topp_128k_vocab() {
- use std::time::Instant;
-
- use rand::{RngExt, SeedableRng, rngs::StdRng};
-
- // ---- Metal context ----
- let context = match <Metal as Backend>::Context::new() {
- Ok(ctx) => ctx,
- Err(e) => {
- println!("Skipping perf test: {}", e);
- return;
- },
- };
-
- // ---- Problem sizes ----
- const BATCH: usize = 8;
- const VOCAB: usize = 128000; // 128K
- const TOP_P: f32 = 0.9;
-
- // ---- Kernel ----
- let kernel =
- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");
-
- // ---- Build random logits ----
- let mut rng = StdRng::seed_from_u64(123);
- let mut logits = vec![0.0f32; BATCH * VOCAB];
- for x in logits.iter_mut() {
- *x = rng.random_range(-6.0f32..6.0f32);
- }
-
- let mut logits_buf = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];
- let seeds_buf = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut output_buf = context
- .device
- .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- // ---- Launch once and time ----
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
-
- kernel
- .encode(
- &mut logits_buf,
- Some(&seeds_buf),
- 0,
- None,
- 0,
- &mut output_buf,
- SamplingMethod::Stochastic {
- temperature: None,
- top_k: None,
- top_p: Some(TOP_P),
- min_p: None,
- },
- BATCH,
- VOCAB,
- &mut command_buffer,
- )
- .expect("encode");
-
- // Time both host-side and GPU execution
- let host_timer = Instant::now();
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
-
- match completed.gpu_execution_time() {
- Some(gpu_time) => {
- println!(
- "Top‑p sampling perf (batch={}, vocab={}): GPU={:.2} ms, Host-side={:.2} ms",
- BATCH,
- VOCAB,
- gpu_time.as_secs_f64() * 1e3,
- host_elapsed_ms
- );
- },
- None => {
- println!(
- "Top‑p sampling perf (batch={}, vocab={}): Host-side={:.2} ms (GPU timing unavailable)",
- BATCH, VOCAB, host_elapsed_ms
- );
- },
- }
-
- // Ensure the kernel produced *some* output (sanity).
- let ptr = output_buf.contents().as_ptr() as *const u32;
- let sample_ids = unsafe { std::slice::from_raw_parts(ptr, BATCH) };
- for &tok in sample_ids {
- assert!((tok as usize) < VOCAB, "Sampled id out of range");
- }
-}
-
#[allow(dead_code)]
fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
use std::time::Instant;
@@ -1022,129 +636,6 @@ fn test_temperature_gpu_cpu_match() {
println!("✓ Temperature processor gpu cpu match (temp={}, rtol={}, atol={})", TEMPERATURE, RTOL, ATOL);
}
-#[test]
-fn test_topk_gpu_cpu_match() {
- use rand::{SeedableRng, rngs::StdRng};
-
- let context = match <Metal as Backend>::Context::new() {
- Ok(ctx) => ctx,
- Err(e) => {
- println!("Skipping topk gpu cpu match test: {}", e);
- return;
- },
- };
-
- const BATCH: usize = 4;
- const VOCAB: usize = 1024;
- const TOPK: u32 = 16;
-
- let kernel = <<Metal as Backend>::Kernels as Kernels>::TopKKernel::new(&context, DataType::F32, false)
- .expect("Failed to create topk kernel");
-
- let mut rng = StdRng::seed_from_u64(42);
- let mut logits = vec![0.0f32; BATCH * VOCAB];
- for x in logits.iter_mut() {
- *x = rng.random_range(-1024.0f32..1024.0f32);
- }
-
- let logits_buffer = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut processed_buffer = context
- .device
- .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
- kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TOPK, &mut command_buffer);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let results_ptr = processed_buffer.contents().as_ptr() as *const f32;
- let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };
-
- for batch_idx in 0..BATCH {
- let cpu_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
- let mut cpu_sorted_logits: Vec<(usize, f32)> = cpu_logits.iter().copied().enumerate().collect();
- cpu_sorted_logits.sort_by(|(_, a), (_, b)| f32::total_cmp(b, a));
- let mut cpu_processed_logits: Vec<f32> = vec![f32::NEG_INFINITY; VOCAB];
- for (idx, val) in cpu_sorted_logits.into_iter().take(TOPK as usize) {
- cpu_processed_logits[idx] = val;
- }
-
- let gpu_processed_logits = &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
-
- assert_eq!(cpu_processed_logits, gpu_processed_logits);
- }
-
- println!("✓ Topk processor gpu cpu match (topk={})", TOPK);
-}
-
-#[test]
-fn test_topp_gpu_cpu_match() {
- use rand::{SeedableRng, rngs::StdRng};
-
- let context = match <Metal as Backend>::Context::new() {
- Ok(ctx) => ctx,
- Err(e) => {
- println!("Skipping topp gpu cpu match test: {}", e);
- return;
- },
- };
-
- const BATCH: usize = 4;
- const VOCAB: usize = 1024;
- const TOPP: f32 = 0.9;
-
- let kernel = <<Metal as Backend>::Kernels as Kernels>::TopPKernel::new(&context, DataType::F32, false)
- .expect("Failed to create topp kernel");
-
- let mut rng = StdRng::seed_from_u64(42);
- let mut logits = vec![0.0f32; BATCH * VOCAB];
- for x in logits.iter_mut() {
- *x = rng.random_range(-16.0f32..16.0f32);
- }
-
- let logits_buffer = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut processed_buffer = context
- .device
- .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
- kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TOPP, &mut command_buffer);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let results_ptr = processed_buffer.contents().as_ptr() as *const f32;
- let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };
-
- for batch_idx in 0..BATCH {
- let cpu_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
- let cpu_processed_logits = cpu_reference_top_p(cpu_logits, TOPP)
- .into_iter()
- .zip(cpu_logits.iter().copied())
- .map(|(cpu_ref_topp, logit)| {
- if cpu_ref_topp > 0.0 {
- logit
- } else {
- f32::NEG_INFINITY
- }
- })
- .collect::<Vec<f32>>();
-
- let gpu_processed_logits = &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
-
- assert_eq!(&cpu_processed_logits, gpu_processed_logits);
- }
-
- println!("✓ Topp processor gpu cpu match (topp={})", TOPP);
-}
-
#[test]
fn test_minp_gpu_cpu_match() {
use rand::{RngExt, SeedableRng, rngs::StdRng};
@@ -1321,69 +812,3 @@ fn test_minp_sampling_match_small() {
fn test_minp_sampling_match_large() {
test_minp_sampling_exact_match(32, 0.05, 4096);
}
-
-#[test]
-fn test_gumbel_gpu_cpu_match() {
- let context = match <Metal as Backend>::Context::new() {
- Ok(ctx) => ctx,
- Err(e) => {
- println!("Skipping gumbel gpu cpu match test: {}", e);
- return;
- },
- };
-
- const BATCH: usize = 7;
- const VOCAB: usize = 16 * 1024 * 64;
- const RTOL: f32 = 0.01;
- const ATOL: f32 = 1e-6;
-
- let kernel = <<Metal as Backend>::Kernels as Kernels>::GumbelKernel::new(&context, DataType::F32, false)
- .expect("Failed to create gumbel kernel");
-
- let logits = vec![0.0f32; BATCH * VOCAB];
- let seeds: Vec<u64> = (0_u64..BATCH as u64).collect();
-
- let logits_buffer = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let seeds_buffer = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&seeds), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut gumbel_logits_buffer = context
- .device
- .new_buffer(BATCH * VOCAB * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
- kernel.encode(
- Some(&logits_buffer),
- &seeds_buffer,
- &mut gumbel_logits_buffer,
- BATCH as u32,
- VOCAB as u32,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let result_ptr = gumbel_logits_buffer.contents().as_ptr() as *const f32;
- let all_results = unsafe { std::slice::from_raw_parts(result_ptr, BATCH * VOCAB) };
-
- for (batch_idx, batch_seed) in seeds.iter().copied().enumerate() {
- let results = &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
- for (logit_idx, gpu_logit_value) in results.iter().copied().enumerate() {
- let cpu_logit_value = gumbel_float(batch_seed, revidx(logit_idx as u32));
- let abs_diff = (cpu_logit_value - gpu_logit_value).abs();
- let tolerance = ATOL + RTOL * cpu_logit_value.abs();
- assert!(
- abs_diff <= tolerance,
- "Mismatch at batch {batch_idx} element {logit_idx}: CPU={cpu_logit_value} GPU={gpu_logit_value} (abs_diff={abs_diff}, tolerance={tolerance})"
- );
- }
- }
-
- println!("✓ Gumbel cpu gpu match test passed (rtol: {:.1}%, atol: {:.1e})", RTOL * 100.0, ATOL);
-}