Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

min_p (#69)

uuuvn15aa6f4955fe
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
new file mode 100644
index 000000000..94e9e2d47
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
@@ -0,0 +1,87 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+
+#define BLOCK_SIZE 1024
+#define BLOCK_SIZE_IN_SIMDS (BLOCK_SIZE / 32)
+
+template <typename T>
+void batched_minp(
+ device const T* logits_data,
+ device T* processed_logits,
+ threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],
+ constant uint& vocab_size,
+ constant float& min_p,
+ uint batch_idx,
+ uint thread_idx
+) {
+ uint batch_start = batch_idx * vocab_size;
+
+ // Find maximum logit
+ float local_max = -INFINITY;
+#pragma unroll(4)
+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
+ float logit_value = float(logits_data[batch_start + i]);
+ local_max = fmax(local_max, logit_value);
+ }
+ float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(
+ local_max,
+ shared_reduce_buffer,
+ thread_idx
+ );
+
+ // Then the threshold is just max_logit + log(min_p), mask everything strictly below it
+ T t_threshold = T(max_logit + log(min_p));
+#pragma unroll(4)
+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
+ T logit_value = logits_data[batch_start + i];
+ processed_logits[batch_start + i] =
+ select(T(-INFINITY), logit_value, logit_value >= t_threshold);
+ }
+}
+
+#define outerArguments(T) \
+ (device const T* logits_data [[buffer(0)]], \
+ device T* processed_logits [[buffer(1)]], \
+ constant uint& vocab_size [[buffer(2)]], \
+ constant float& min_p [[buffer(3)]], \
+ uint3 threadgroup_idx [[threadgroup_position_in_grid]], \
+ uint3 thread_idx [[thread_position_in_threadgroup]])
+
+#define innerArguments \
+ (logits_data, \
+ processed_logits, \
+ shared_reduce_buffer, \
+ vocab_size, \
+ min_p, \
+ threadgroup_idx.x, \
+ thread_idx.x)
+
+#define generateMinpKernel(functionName, scalarType, outerArgs, innerArgs) \
+ [[max_total_threads_per_threadgroup( \
+ 1024 \
+ )]] kernel void functionName##_##scalarType outerArgs { \
+ threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \
+ functionName innerArgs; \
+ }
+
+#define generateMinpKernels(functionName) \
+ generateMinpKernel( \
+ functionName, \
+ float, \
+ outerArguments(float), \
+ innerArguments \
+ ); \
+ generateMinpKernel( \
+ functionName, \
+ bfloat, \
+ outerArguments(bfloat), \
+ innerArguments \
+ ); \
+ generateMinpKernel(functionName, half, outerArguments(half), innerArguments);
+
+generateMinpKernels(batched_minp)
+
+#undef outerArguments
+#undef innerArguments
+#undef generateMinpKernel
+#undef generateMinpKernels
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
index aedbd34bd..b5dcf636d 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
@@ -59,6 +59,8 @@ pub struct SamplingKernel {
topk_applied_logits: MTLBuffer,
topp_pipeline: MTLComputePipelineState,
topp_applied_logits: MTLBuffer,
+ minp_pipeline: MTLComputePipelineState,
+ minp_applied_logits: MTLBuffer,
gumbel_pipeline: MTLComputePipelineState,
gumbel_applied_logits: MTLBuffer,
argmax_implementation: ArgmaxImplementation,
@@ -162,6 +164,20 @@ impl SamplingKernel {
MTLResourceOptions::StorageModeShared,
);
+ let minp_pipeline = context
+ .compute_pipeline_state_with_reflection(
+ &format!("batched_minp_{}", data_suffix),
+ None,
+ )
+ .map(|(pipeline, _)| pipeline)
+ .map_err(SamplingError::MetalError)?;
+
+ let minp_applied_logits = context.device.new_buffer(
+ (max_elements * Into::<DataType>::into(data_type).size_in_bytes())
+ as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
let gumbel_pipeline = context
.compute_pipeline_state_with_reflection(
&format!("batched_gumbel_{}", data_suffix),
@@ -236,6 +252,8 @@ impl SamplingKernel {
topk_applied_logits,
topp_pipeline,
topp_applied_logits,
+ minp_pipeline,
+ minp_applied_logits,
gumbel_pipeline,
gumbel_applied_logits,
argmax_implementation,
@@ -315,6 +333,7 @@ impl SamplingKernel {
temperature,
top_k,
top_p,
+ min_p,
} = sampling_method
{
if let Some(temperature) = temperature {
@@ -353,6 +372,18 @@ impl SamplingKernel {
last_logits_buffer = &self.topp_applied_logits;
}
+ if let Some(min_p) = min_p {
+ self.encode_minp(
+ last_logits_buffer,
+ &self.minp_applied_logits,
+ batch_size as u32,
+ vocab_size as u32,
+ min_p,
+ compute_encoder,
+ )?;
+ last_logits_buffer = &self.minp_applied_logits;
+ }
+
self.encode_gumbel(
last_logits_buffer,
seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?,
@@ -525,6 +556,38 @@ impl SamplingKernel {
Ok(())
}
+ pub fn encode_minp(
+ &self,
+ logits_buffer: &MTLBuffer,
+ processed_logits_buffer: &MTLBuffer,
+ batch_size: u32,
+ vocab_size: u32,
+ min_p: f32,
+ compute_encoder: &ComputeCommandEncoderRef,
+ ) -> Result<(), SamplingError> {
+ compute_encoder.set_compute_pipeline_state(&self.minp_pipeline);
+
+ compute_encoder.set_buffer(0, Some(logits_buffer), 0);
+ compute_encoder.set_buffer(1, Some(processed_logits_buffer), 0);
+ compute_encoder.set_bytes(
+ 2,
+ size_of::<u32>() as u64,
+ &vocab_size as *const u32 as *const std::ffi::c_void,
+ );
+ compute_encoder.set_bytes(
+ 3,
+ size_of::<f32>() as u64,
+ &min_p as *const f32 as *const std::ffi::c_void,
+ );
+
+ compute_encoder.dispatch_thread_groups(
+ MTLSize::new(batch_size as u64, 1, 1),
+ MTLSize::new(BLOCK_SIZE as u64, 1, 1),
+ );
+
+ Ok(())
+ }
+
pub fn encode_gumbel(
&self,
logits_buffer: &MTLBuffer,
diff --git a/crates/uzu/src/config/generation.rs b/crates/uzu/src/config/generation.rs
index 5dce2219c..badb2ce7b 100644
--- a/crates/uzu/src/config/generation.rs
+++ b/crates/uzu/src/config/generation.rs
@@ -6,5 +6,6 @@ pub struct GenerationConfig {
pub temperature: Option<f32>,
pub top_k: Option<u32>,
pub top_p: Option<f32>,
+ pub min_p: Option<f32>,
pub banned_tokens: Option<Vec<u32>>,
}
diff --git a/crates/uzu/src/session/parameter/sampling_policy.rs b/crates/uzu/src/session/parameter/sampling_policy.rs
index f4c5745f1..070d95ced 100644
--- a/crates/uzu/src/session/parameter/sampling_policy.rs
+++ b/crates/uzu/src/session/parameter/sampling_policy.rs
@@ -9,6 +9,7 @@ pub enum SamplingMethod {
temperature: Option<f32>,
top_k: Option<u32>,
top_p: Option<f32>,
+ min_p: Option<f32>,
},
}
@@ -45,6 +46,7 @@ impl ConfigResolvableValue<LanguageModelConfig, SamplingMethod>
temperature: generation_config.temperature,
top_k: generation_config.top_k,
top_p: generation_config.top_p,
+ min_p: generation_config.min_p,
},
SamplingPolicy::Custom {
value,
diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rs
index 0ce9d1424..03b07c34b 100644
--- a/crates/uzu/tests/sampling_test.rs
+++ b/crates/uzu/tests/sampling_test.rs
@@ -59,6 +59,26 @@ fn cpu_reference_top_p(
dist
}
+fn cpu_reference_min_p(
+ row_logits: &[f32],
+ min_p: f32,
+) -> Vec<f32> {
+ let max_logit =
+ row_logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
+ let threshold = max_logit + min_p.ln();
+
+ row_logits
+ .iter()
+ .map(|&logit| {
+ if logit >= threshold {
+ logit
+ } else {
+ f32::NEG_INFINITY
+ }
+ })
+ .collect()
+}
+
fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
let context = match create_test_context() {
Ok(ctx) => ctx,
@@ -256,6 +276,7 @@ fn test_topp_sampling_from_prob_exact_match(
temperature: None,
top_k: None,
top_p: Some(p),
+ min_p: None,
},
batch_size,
vocab_size,
@@ -385,6 +406,7 @@ fn test_topp_sampling_statistical_large() {
temperature: None,
top_k: None,
top_p: Some(TOP_P),
+ min_p: None,
},
BATCH,
VOCAB,
@@ -495,6 +517,7 @@ fn perf_topp_128k_vocab() {
temperature: None,
top_k: None,
top_p: Some(TOP_P),
+ min_p: None,
},
BATCH,
VOCAB,
@@ -725,6 +748,7 @@ fn test_categorical_sampling() {
temperature: None,
top_k: None,
top_p: None,
+ min_p: None,
},
batch_size,
vocab_size,
@@ -870,6 +894,7 @@ fn test_categorical_sampling_statistical() {
temperature: None,
top_k: None,
top_p: None,
+ min_p: None,
},
BATCH,
VOCAB,
@@ -978,6 +1003,7 @@ fn perf_categorical_128k_vocab() {
temperature: None,
top_k: None,
top_p: None,
+ min_p: None,
},
BATCH,
VOCAB,
@@ -1260,6 +1286,216 @@ fn test_topp_gpu_cpu_match() {
println!("✓ Topp processor gpu cpu match (topp={})", TOPP);
}
+#[test]
+fn test_minp_gpu_cpu_match() {
+ use rand::{Rng, SeedableRng, rngs::StdRng};
+
+ let context = match create_test_context() {
+ Ok(ctx) => ctx,
+ Err(e) => {
+ println!("Skipping minp gpu cpu match test: {}", e);
+ return;
+ },
+ };
+
+ const BATCH: usize = 4;
+ const VOCAB: usize = 1024;
+ const MINP: f32 = 0.1;
+
+ let kernel =
+ SamplingKernel::new(&context, KernelDataType::Float32, BATCH, VOCAB)
+ .expect("Failed to create sampling kernel");
+
+ let mut rng = StdRng::seed_from_u64(42);
+ let mut logits = vec![0.0f32; BATCH * VOCAB];
+ for x in logits.iter_mut() {
+ *x = rng.random_range(-16.0f32..16.0f32);
+ }
+
+ let logits_buffer = context.device.new_buffer_with_data(
+ logits.as_ptr() as *const _,
+ (logits.len() * std::mem::size_of::<f32>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let processed_buffer = context.device.new_buffer(
+ (logits.len() * std::mem::size_of::<f32>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let command_buffer_ref = context.command_queue.new_command_buffer();
+ let command_buffer = command_buffer_ref.to_owned();
+
+ let compute_encoder = command_buffer.new_compute_command_encoder();
+ kernel
+ .encode_minp(
+ &logits_buffer,
+ &processed_buffer,
+ BATCH as u32,
+ VOCAB as u32,
+ MINP,
+ compute_encoder,
+ )
+ .expect("encode_minp");
+ compute_encoder.end_encoding();
+
+ command_buffer_ref.commit();
+ command_buffer_ref.wait_until_completed();
+
+ let results_ptr = processed_buffer.contents() as *const f32;
+ let all_results =
+ unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };
+
+ for batch_idx in 0..BATCH {
+ let cpu_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
+ let cpu_processed_logits = cpu_reference_min_p(cpu_logits, MINP);
+
+ let gpu_processed_logits =
+ &all_results[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
+
+ assert_eq!(&cpu_processed_logits, gpu_processed_logits);
+ }
+
+ println!("✓ Minp processor gpu cpu match (minp={})", MINP);
+}
+
+fn test_minp_sampling_exact_match(
+ batch_size: usize,
+ min_p: f32,
+ vocab_size: usize,
+) {
+ use std::collections::HashSet;
+
+ use rand::{SeedableRng, rngs::StdRng};
+
+ let context = match create_test_context() {
+ Ok(ctx) => ctx,
+ Err(e) => {
+ eprintln!("Skipping minp exact match test: {}", e);
+ return;
+ },
+ };
+
+ let kernel = SamplingKernel::new(
+ &context,
+ KernelDataType::Float32,
+ batch_size,
+ vocab_size,
+ )
+ .expect("Failed to create sampling kernel");
+
+ // Build logits where some tokens have high probability and others have low
+ // For min_p filtering, tokens with probability < min_p * max_prob are masked
+ // We set k tokens to have probability 0.1 each (high), rest get very low probability
+ let k = 10;
+ let high_logit = 0.0f32; // exp(0) = 1
+ let low_logit = -50.0f32; // exp(-50) ≈ 0
+
+ let mut logits = vec![low_logit; batch_size * vocab_size];
+ let mut high_prob_token_sets: Vec<HashSet<usize>> =
+ Vec::with_capacity(batch_size);
+
+ let mut rng = StdRng::seed_from_u64(42);
+ let mut all_token_ids: Vec<usize> = (0..vocab_size).collect();
+
+ for b in 0..batch_size {
+ all_token_ids.shuffle(&mut rng);
+ let high_prob_tokens: HashSet<usize> =
+ all_token_ids[..k].iter().cloned().collect();
+
+ for &token_id in &high_prob_tokens {
+ logits[b * vocab_size + token_id] = high_logit;
+ }
+ high_prob_token_sets.push(high_prob_tokens);
+ }
+
+ let logits_buf = context.device.new_buffer_with_data(
+ logits.as_ptr() as *const _,
+ (logits.len() * std::mem::size_of::<f32>()) as u64,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+ let output_buf = context.device.new_buffer(
+ (batch_size * std::mem::size_of::<u32>()) as u64,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+
+ let num_samples = 1000;
+ let mut counter = vec![0i32; batch_size * vocab_size];
+
+ for draw in 0..num_samples {
+ let seeds_buf = context.device.new_buffer_with_data(
+ vec![TEST_SAMPLING_SEED + draw as u64; batch_size].as_ptr()
+ as *const _,
+ (batch_size * std::mem::size_of::<u64>()) as u64,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+
+ let cb_ref = context.command_queue.new_command_buffer();
+ let cb = cb_ref.to_owned();
+ kernel
+ .encode(
+ &logits_buf,
+ Some(&seeds_buf),
+ 0,
+ None,
+ &output_buf,
+ SamplingMethod::Stochastic {
+ temperature: None,
+ top_k: None,
+ top_p: None,
+ min_p: Some(min_p),
+ },
+ batch_size,
+ vocab_size,
+ &cb,
+ )
+ .expect("encode");
+ cb_ref.commit();
+ cb_ref.wait_until_completed();
+
+ let ptr = output_buf.contents() as *const u32;
+ let sampled_ids =
+ unsafe { std::slice::from_raw_parts(ptr, batch_size) };
+
+ for (i, &sampled_id) in sampled_ids.iter().enumerate() {
+ assert!(
+ (sampled_id as usize) < vocab_size,
+ "Sampled token out of range"
+ );
+ counter[i * vocab_size + sampled_id as usize] += 1;
+ }
+ }
+
+ for i in 0..batch_size {
+ for j in 0..vocab_size {
+ if counter[i * vocab_size + j] > 0 {
+ assert!(
+ high_prob_token_sets[i].contains(&j),
+ "high_prob_token_sets[{}] does not contain {} (appeared {} times)",
+ i,
+ j,
+ counter[i * vocab_size + j]
+ );
+ }
+ }
+ }
+
+ println!(
+ "batch_size: {}, min_p: {:.2}, vocab_size: {}, accuracy test passed.",
+ batch_size, min_p, vocab_size
+ );
+}
+
+#[test]
+fn test_minp_sampling_match_small() {
+ test_minp_sampling_exact_match(8, 0.1, 1024);
+}
+
+#[test]
+fn test_minp_sampling_match_large() {
+ test_minp_sampling_exact_match(32, 0.05, 4096);
+}
+
#[test]
fn test_gumbel_gpu_cpu_match() {
let context = match create_test_context() {