A high-performance inference engine for AI models
fix sampling memory usage (#77)
uuuvn6c3b1fa3ea98
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs b/crates/uzu/src/backends/metal/kernel/sampling/mod.rsindex aaa944f97..7d56878e1 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs@@ -7,7 +7,6 @@ use metal::{use thiserror::Error;use crate::{- DataType,backends::metal::{KernelDataType, MTLContext, MTLError,kernel::dsl::{@@ -54,17 +53,11 @@ enum ArgmaxImplementation {pub struct SamplingKernel {bitmask: BitmaskKernel,- bitmask_applied_logits: MTLBuffer,temperature: TemperatureKernel,- temperature_applied_logits: MTLBuffer,topk: TopKKernel,- topk_applied_logits: MTLBuffer,topp: TopPKernel,- topp_applied_logits: MTLBuffer,minp: MinPKernel,- minp_applied_logits: MTLBuffer,gumbel: GumbelKernel,- gumbel_applied_logits: MTLBuffer,argmax_implementation: ArgmaxImplementation,max_batch_size: usize,max_vocab_size: usize,@@ -107,56 +100,18 @@ impl SamplingKernel {max_vocab_size: usize,argmax_strategy: ArgmaxStrategy,) -> Result<Self, SamplingError> {- let max_elements = max_batch_size * max_vocab_size;-let bitmask = BitmaskKernel::new(context, data_type)?;- let bitmask_applied_logits = context.device.new_buffer(- (max_elements * Into::<DataType>::into(data_type).size_in_bytes())- as u64,- MTLResourceOptions::StorageModeShared,- );-let temperature = TemperatureKernel::new(context, data_type)?;- let temperature_applied_logits = context.device.new_buffer(- (max_elements * Into::<DataType>::into(data_type).size_in_bytes())- as u64,- MTLResourceOptions::StorageModeShared,- );-let topk = TopKKernel::new(context, data_type)?;- let topk_applied_logits = context.device.new_buffer(- (max_elements * Into::<DataType>::into(data_type).size_in_bytes())- as u64,- MTLResourceOptions::StorageModeShared,- );-let topp = TopPKernel::new(context, data_type)?;- let topp_applied_logits = context.device.new_buffer(- (max_elements * Into::<DataType>::into(data_type).size_in_bytes())- as u64,- MTLResourceOptions::StorageModeShared,- );-let minp = MinPKernel::new(context, data_type)?;- let minp_applied_logits = context.device.new_buffer(- (max_elements * Into::<DataType>::into(data_type).size_in_bytes())- as u64,- MTLResourceOptions::StorageModeShared,- );-let gumbel = GumbelKernel::new(context, data_type)?;- let gumbel_applied_logits = context.device.new_buffer(- (max_elements * Into::<DataType>::into(data_type).size_in_bytes())- as u64,- MTLResourceOptions::StorageModeShared,- );-let argmax_implementation = match argmax_strategy {ArgmaxStrategy::SinglePass => {let kernel = ArgmaxSingleKernel::new(context, data_type)?;@@ -194,17 +149,11 @@ impl SamplingKernel {Ok(Self {bitmask,- bitmask_applied_logits,temperature,- temperature_applied_logits,topk,- topk_applied_logits,topp,- topp_applied_logits,minp,- minp_applied_logits,gumbel,- gumbel_applied_logits,argmax_implementation,max_batch_size,max_vocab_size,@@ -267,19 +216,16 @@ impl SamplingKernel {));}- let mut last_logits_buffer = logits_buffer;-if let Some(bitmask_buffer) = bitmask_buffer {self.bitmask.encode(- last_logits_buffer,+ logits_buffer,bitmask_buffer,bitmask_offset as u32,- &self.bitmask_applied_logits,+ logits_buffer,batch_size as u32,vocab_size as u32,compute_encoder,);- last_logits_buffer = &self.bitmask_applied_logits;}if let SamplingMethod::Stochastic {@@ -291,62 +237,57 @@ impl SamplingKernel {{if let Some(temperature) = temperature {self.temperature.encode(- last_logits_buffer,- &self.temperature_applied_logits,+ logits_buffer,+ logits_buffer,batch_size as u32,vocab_size as u32,temperature,compute_encoder,);- last_logits_buffer = &self.temperature_applied_logits;}if let Some(top_k) = top_k {self.topk.encode(- last_logits_buffer,- &self.topk_applied_logits,+ logits_buffer,+ logits_buffer,batch_size as u32,vocab_size as u32,top_k,compute_encoder,);- last_logits_buffer = &self.topk_applied_logits;}if let Some(top_p) = top_p {self.topp.encode(- last_logits_buffer,- &self.topp_applied_logits,+ logits_buffer,+ logits_buffer,batch_size as u32,vocab_size as u32,top_p,compute_encoder,);- last_logits_buffer = &self.topp_applied_logits;}if let Some(min_p) = min_p {self.minp.encode(- last_logits_buffer,- &self.minp_applied_logits,+ logits_buffer,+ logits_buffer,batch_size as u32,vocab_size as u32,min_p,compute_encoder,);- last_logits_buffer = &self.minp_applied_logits;}self.gumbel.encode(- last_logits_buffer,+ logits_buffer,seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?,- &self.gumbel_applied_logits,+ logits_buffer,batch_size as u32,vocab_size as u32,(seeds_offset / size_of::<u64>()) as u32,compute_encoder,);- last_logits_buffer = &self.gumbel_applied_logits;}match &self.argmax_implementation {@@ -354,7 +295,7 @@ impl SamplingKernel {kernel,} => {kernel.encode(- last_logits_buffer,+ logits_buffer,sampled_tokens_buffer,batch_size as u32,vocab_size as u32,@@ -367,7 +308,7 @@ impl SamplingKernel {partial_results_buffer,} => {main_kernel.encode(- last_logits_buffer,+ logits_buffer,partial_results_buffer,batch_size as u32,vocab_size as u32,