A high-performance inference engine for AI models
min_p (#69)
uuuvn15aa6f4955fe
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metalnew file mode 100644index 000000000..94e9e2d47--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal@@ -0,0 +1,87 @@+#include <metal_stdlib>+#include "../definitions.metal"++#define BLOCK_SIZE 1024+#define BLOCK_SIZE_IN_SIMDS (BLOCK_SIZE / 32)++template <typename T>+void batched_minp(+ device const T* logits_data,+ device T* processed_logits,+ threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],+ constant uint& vocab_size,+ constant float& min_p,+ uint batch_idx,+ uint thread_idx+) {+ uint batch_start = batch_idx * vocab_size;++ // Find maximum logit+ float local_max = -INFINITY;+#pragma unroll(4)+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {+ float logit_value = float(logits_data[batch_start + i]);+ local_max = fmax(local_max, logit_value);+ }+ float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(+ local_max,+ shared_reduce_buffer,+ thread_idx+ );++ // Then the threshold is just max_logit + log(min_p), mask everything strictly below it+ T t_threshold = T(max_logit + log(min_p));+#pragma unroll(4)+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {+ T logit_value = logits_data[batch_start + i];+ processed_logits[batch_start + i] =+ select(T(-INFINITY), logit_value, logit_value >= t_threshold);+ }+}++#define outerArguments(T) \+ (device const T* logits_data [[buffer(0)]], \+ device T* processed_logits [[buffer(1)]], \+ constant uint& vocab_size [[buffer(2)]], \+ constant float& min_p [[buffer(3)]], \+ uint3 threadgroup_idx [[threadgroup_position_in_grid]], \+ uint3 thread_idx [[thread_position_in_threadgroup]])++#define innerArguments \+ (logits_data, \+ processed_logits, \+ shared_reduce_buffer, \+ vocab_size, \+ min_p, \+ threadgroup_idx.x, \+ thread_idx.x)++#define generateMinpKernel(functionName, scalarType, outerArgs, innerArgs) \+ [[max_total_threads_per_threadgroup( \+ 1024 \+ )]] kernel void functionName##_##scalarType outerArgs { \+ threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \+ functionName innerArgs; \+ }++#define generateMinpKernels(functionName) \+ generateMinpKernel( \+ functionName, \+ float, \+ outerArguments(float), \+ innerArguments \+ ); \+ generateMinpKernel( \+ functionName, \+ bfloat, \+ outerArguments(bfloat), \+ innerArguments \+ ); \+ generateMinpKernel(functionName, half, outerArguments(half), innerArguments);++generateMinpKernels(batched_minp)++#undef outerArguments+#undef innerArguments+#undef generateMinpKernel+#undef generateMinpKernelsdiff --git a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs b/crates/uzu/src/backends/metal/kernel/sampling/mod.rsindex aedbd34bd..b5dcf636d 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs@@ -59,6 +59,8 @@ pub struct SamplingKernel {topk_applied_logits: MTLBuffer,topp_pipeline: MTLComputePipelineState,topp_applied_logits: MTLBuffer,+ minp_pipeline: MTLComputePipelineState,+ minp_applied_logits: MTLBuffer,gumbel_pipeline: MTLComputePipelineState,gumbel_applied_logits: MTLBuffer,argmax_implementation: ArgmaxImplementation,@@ -162,6 +164,20 @@ impl SamplingKernel {MTLResourceOptions::StorageModeShared,);+ let minp_pipeline = context+ .compute_pipeline_state_with_reflection(+ &format!("batched_minp_{}", data_suffix),+ None,+ )+ .map(|(pipeline, _)| pipeline)+ .map_err(SamplingError::MetalError)?;++ let minp_applied_logits = context.device.new_buffer(+ (max_elements * Into::<DataType>::into(data_type).size_in_bytes())+ as u64,+ MTLResourceOptions::StorageModeShared,+ );+let gumbel_pipeline = context.compute_pipeline_state_with_reflection(&format!("batched_gumbel_{}", data_suffix),@@ -236,6 +252,8 @@ impl SamplingKernel {topk_applied_logits,topp_pipeline,topp_applied_logits,+ minp_pipeline,+ minp_applied_logits,gumbel_pipeline,gumbel_applied_logits,argmax_implementation,@@ -315,6 +333,7 @@ impl SamplingKernel {temperature,top_k,top_p,+ min_p,} = sampling_method{if let Some(temperature) = temperature {@@ -353,6 +372,18 @@ impl SamplingKernel {last_logits_buffer = &self.topp_applied_logits;}+ if let Some(min_p) = min_p {+ self.encode_minp(+ last_logits_buffer,+ &self.minp_applied_logits,+ batch_size as u32,+ vocab_size as u32,+ min_p,+ compute_encoder,+ )?;+ last_logits_buffer = &self.minp_applied_logits;+ }+self.encode_gumbel(last_logits_buffer,seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?,@@ -525,6 +556,38 @@ impl SamplingKernel {Ok(())}+ pub fn encode_minp(+ &self,+ logits_buffer: &MTLBuffer,+ processed_logits_buffer: &MTLBuffer,+ batch_size: u32,+ vocab_size: u32,+ min_p: f32,+ compute_encoder: &ComputeCommandEncoderRef,+ ) -> Result<(), SamplingError> {+ compute_encoder.set_compute_pipeline_state(&self.minp_pipeline);++ compute_encoder.set_buffer(0, Some(logits_buffer), 0);+ compute_encoder.set_buffer(1, Some(processed_logits_buffer), 0);+ compute_encoder.set_bytes(+ 2,+ size_of::<u32>() as u64,+ &vocab_size as *const u32 as *const std::ffi::c_void,+ );+ compute_encoder.set_bytes(+ 3,+ size_of::<f32>() as u64,+ &min_p as *const f32 as *const std::ffi::c_void,+ );++ compute_encoder.dispatch_thread_groups(+ MTLSize::new(batch_size as u64, 1, 1),+ MTLSize::new(BLOCK_SIZE as u64, 1, 1),+ );++ Ok(())+ }+pub fn encode_gumbel(&self,logits_buffer: &MTLBuffer,diff --git a/crates/uzu/src/config/generation.rs b/crates/uzu/src/config/generation.rsindex 5dce2219c..badb2ce7b 100644--- a/crates/uzu/src/config/generation.rs+++ b/crates/uzu/src/config/generation.rs@@ -6,5 +6,6 @@ pub struct GenerationConfig {pub temperature: Option<f32>,pub top_k: Option<u32>,pub top_p: Option<f32>,+ pub min_p: Option<f32>,pub banned_tokens: Option<Vec<u32>>,}diff --git a/crates/uzu/src/session/parameter/sampling_policy.rs b/crates/uzu/src/session/parameter/sampling_policy.rsindex f4c5745f1..070d95ced 100644--- a/crates/uzu/src/session/parameter/sampling_policy.rs+++ b/crates/uzu/src/session/parameter/sampling_policy.rs@@ -9,6 +9,7 @@ pub enum SamplingMethod {temperature: Option<f32>,top_k: Option<u32>,top_p: Option<f32>,+ min_p: Option<f32>,},}@@ -45,6 +46,7 @@ impl ConfigResolvableValue<LanguageModelConfig, SamplingMethod>temperature: generation_config.temperature,top_k: generation_config.top_k,top_p: generation_config.top_p,+ min_p: generation_config.min_p,},SamplingPolicy::Custom {value,diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rsindex 0ce9d1424..03b07c34b 100644--- a/crates/uzu/tests/sampling_test.rs+++ b/crates/uzu/tests/sampling_test.rs@@ -59,6 +59,26 @@ fn cpu_reference_top_p(dist}+fn cpu_reference_min_p(+ row_logits: &[f32],+ min_p: f32,+) -> Vec<f32> {+ let max_logit =+ row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);+ let threshold = max_logit + min_p.ln();++ row_logits+ .iter()+ .map(|&logit| {+ if logit >= threshold {+ logit+ } else {+ f32::NEG_INFINITY+ }+ })+ .collect()+}+fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {let context = match create_test_context() {Ok(ctx) => ctx,@@ -256,6 +276,7 @@ fn test_topp_sampling_from_prob_exact_match(temperature: None,top_k: None,top_p: Some(p),+ min_p: None,},batch_size,vocab_size,@@ -385,6 +406,7 @@ fn test_topp_sampling_statistical_large() {temperature: None,top_k: None,top_p: Some(TOP_P),+ min_p: None,},BATCH,VOCAB,@@ -495,6 +517,7 @@ fn perf_topp_128k_vocab() {temperature: None,top_k: None,top_p: Some(TOP_P),+ min_p: None,},BATCH,VOCAB,@@ -725,6 +748,7 @@ fn test_categorical_sampling() {temperature: None,top_k: None,top_p: None,+ min_p: None,},batch_size,vocab_size,@@ -870,6 +894,7 @@ fn test_categorical_sampling_statistical() {temperature: None,top_k: None,top_p: None,+ min_p: None,},BATCH,VOCAB,@@ -978,6 +1003,7 @@ fn perf_categorical_128k_vocab() {temperature: None,top_k: None,top_p: None,+ min_p: None,},BATCH,VOCAB,@@ -1260,6 +1286,216 @@ fn test_topp_gpu_cpu_match() {println!("✓ Topp processor gpu cpu match (topp={})", TOPP);}+#[test]+fn test_minp_gpu_cpu_match() {+ use rand::{Rng, SeedableRng, rngs::StdRng};++ let context = match create_test_context() {+ Ok(ctx) => ctx,+ Err(e) => {+ println!("Skipping minp gpu cpu match test: {}", e);+ return;+ },+ };++ const BATCH: usize = 4;+ const VOCAB: usize = 1024;+ const MINP: f32 = 0.1;++ let kernel =+ SamplingKernel::new(&context, KernelDataType::Float32, BATCH, VOCAB)+ .expect("Failed to create sampling kernel");++ let mut rng = StdRng::seed_from_u64(42);+ let mut logits = vec![0.0f32; BATCH * VOCAB];+ for x in logits.iter_mut() {+ *x = rng.random_range(-16.0f32..16.0f32);+ }++ let logits_buffer = context.device.new_buffer_with_data(+ logits.as_ptr() as *const _,+ (logits.len() * std::mem::size_of::<f32>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let processed_buffer = context.device.new_buffer(+ (logits.len() * std::mem::size_of::<f32>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let command_buffer_ref = context.command_queue.new_command_buffer();+ let command_buffer = command_buffer_ref.to_owned();++ let compute_encoder = command_buffer.new_compute_command_encoder();+ kernel+ .encode_minp(+ &logits_buffer,+ &processed_buffer,+ BATCH as u32,+ VOCAB as u32,+ MINP,+ compute_encoder,+ )+ .expect("encode_minp");+ compute_encoder.end_encoding();++ command_buffer_ref.commit();+ command_buffer_ref.wait_until_completed();++ let results_ptr = processed_buffer.contents() as *const f32;+ let all_results =+ unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };++ for batch_idx in 0..BATCH {+ let cpu_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];+ let cpu_processed_logits = cpu_reference_min_p(cpu_logits, MINP);++ let gpu_processed_logits =+ &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];++ assert_eq!(&cpu_processed_logits, gpu_processed_logits);+ }++ println!("✓ Minp processor gpu cpu match (minp={})", MINP);+}++fn test_minp_sampling_exact_match(+ batch_size: usize,+ min_p: f32,+ vocab_size: usize,+) {+ use std::collections::HashSet;++ use rand::{SeedableRng, rngs::StdRng};++ let context = match create_test_context() {+ Ok(ctx) => ctx,+ Err(e) => {+ eprintln!("Skipping minp exact match test: {}", e);+ return;+ },+ };++ let kernel = SamplingKernel::new(+ &context,+ KernelDataType::Float32,+ batch_size,+ vocab_size,+ )+ .expect("Failed to create sampling kernel");++ // Build logits where some tokens have high probability and others have low+ // For min_p filtering, tokens with probability < min_p * max_prob are masked+ // We set k tokens to have probability 0.1 each (high), rest get very low probability+ let k = 10;+ let high_logit = 0.0f32; // exp(0) = 1+ let low_logit = -50.0f32; // exp(-50) ≈ 0++ let mut logits = vec![low_logit; batch_size * vocab_size];+ let mut high_prob_token_sets: Vec<HashSet<usize>> =+ Vec::with_capacity(batch_size);++ let mut rng = StdRng::seed_from_u64(42);+ let mut all_token_ids: Vec<usize> = (0..vocab_size).collect();++ for b in 0..batch_size {+ all_token_ids.shuffle(&mut rng);+ let high_prob_tokens: HashSet<usize> =+ all_token_ids[..k].iter().cloned().collect();++ for &token_id in &high_prob_tokens {+ logits[b * vocab_size + token_id] = high_logit;+ }+ high_prob_token_sets.push(high_prob_tokens);+ }++ let logits_buf = context.device.new_buffer_with_data(+ logits.as_ptr() as *const _,+ (logits.len() * std::mem::size_of::<f32>()) as u64,+ metal::MTLResourceOptions::StorageModeShared,+ );+ let output_buf = context.device.new_buffer(+ (batch_size * std::mem::size_of::<u32>()) as u64,+ metal::MTLResourceOptions::StorageModeShared,+ );++ let num_samples = 1000;+ let mut counter = vec![0i32; batch_size * vocab_size];++ for draw in 0..num_samples {+ let seeds_buf = context.device.new_buffer_with_data(+ vec![TEST_SAMPLING_SEED + draw as u64; batch_size].as_ptr()+ as *const _,+ (batch_size * std::mem::size_of::<u64>()) as u64,+ metal::MTLResourceOptions::StorageModeShared,+ );++ let cb_ref = context.command_queue.new_command_buffer();+ let cb = cb_ref.to_owned();+ kernel+ .encode(+ &logits_buf,+ Some(&seeds_buf),+ 0,+ None,+ &output_buf,+ SamplingMethod::Stochastic {+ temperature: None,+ top_k: None,+ top_p: None,+ min_p: Some(min_p),+ },+ batch_size,+ vocab_size,+ &cb,+ )+ .expect("encode");+ cb_ref.commit();+ cb_ref.wait_until_completed();++ let ptr = output_buf.contents() as *const u32;+ let sampled_ids =+ unsafe { std::slice::from_raw_parts(ptr, batch_size) };++ for (i, &sampled_id) in sampled_ids.iter().enumerate() {+ assert!(+ (sampled_id as usize) < vocab_size,+ "Sampled token out of range"+ );+ counter[i * vocab_size + sampled_id as usize] += 1;+ }+ }++ for i in 0..batch_size {+ for j in 0..vocab_size {+ if counter[i * vocab_size + j] > 0 {+ assert!(+ high_prob_token_sets[i].contains(&j),+ "high_prob_token_sets[{}] does not contain {} (appeared {} times)",+ i,+ j,+ counter[i * vocab_size + j]+ );+ }+ }+ }++ println!(+ "batch_size: {}, min_p: {:.2}, vocab_size: {}, accuracy test passed.",+ batch_size, min_p, vocab_size+ );+}++#[test]+fn test_minp_sampling_match_small() {+ test_minp_sampling_exact_match(8, 0.1, 1024);+}++#[test]+fn test_minp_sampling_match_large() {+ test_minp_sampling_exact_match(32, 0.05, 4096);+}+#[test]fn test_gumbel_gpu_cpu_match() {let context = match create_test_context() {