A high-performance inference engine for AI models
Lalamo 0.13.0 (with very slop repetition penalty) (#518)
uuuvn113394947db9
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/sampling/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/sampling/mod.rsindex 5171b2be1..6a251ef64 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/sampling/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/sampling/mod.rs@@ -1 +1,2 @@+pub mod repetition_penalty;pub mod unified_sampling;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/sampling/repetition_penalty.rs b/crates/backend-uzu/src/backends/cpu/kernel/sampling/repetition_penalty.rsnew file mode 100644index 000000000..7546210c3--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/sampling/repetition_penalty.rs@@ -0,0 +1,46 @@+use half::bf16;+use num_traits::{Float, NumCast};+use proc_macros::kernel;++use crate::{array::ArrayElement, backends::common::gpu_types::ring::RingParams};++#[kernel(RepetitionPenalty)]+#[variants(T, f32, bf16)]+pub fn repetition_penalty<T: ArrayElement + Float + NumCast>(+ original_logits: *const T,+ logits_copy: *mut T,+ context_ring: *const u32,+ token_ids: *const u64,+ repetition_penalty: f32,+ suffix_repetition_length: u32,+) {+ fn apply<T: ArrayElement + Float + NumCast>(+ original_logits: *const T,+ logits_copy: *mut T,+ token_id: usize,+ repetition_penalty: f32,+ ) {+ let logit = unsafe { (*original_logits.add(token_id)).to_f32().unwrap() };+ let penalized = if logit > 0.0 {+ logit / repetition_penalty+ } else {+ logit * repetition_penalty+ };+ unsafe {+ *logits_copy.add(token_id) = NumCast::from(penalized).unwrap();+ }+ }++ let ring = unsafe { &*context_ring.cast::<RingParams>() };+ let ring_tokens = unsafe { context_ring.add(2) };++ apply(original_logits, logits_copy, unsafe { *token_ids } as usize, repetition_penalty);++ for ring_index in 0..suffix_repetition_length {+ if ring_index >= ring.ring_length {+ continue;+ }+ let slot = (ring.ring_offset + ring_index) % suffix_repetition_length;+ apply(original_logits, logits_copy, unsafe { *ring_tokens.add(slot as usize) } as usize, repetition_penalty)+ }+}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/token_copy/token_copy.rs b/crates/backend-uzu/src/backends/cpu/kernel/token_copy/token_copy.rsindex 73a4355a2..693ce9517 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/token_copy/token_copy.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/token_copy/token_copy.rs@@ -1,12 +1,36 @@use proc_macros::kernel;+use crate::backends::common::gpu_types::ring::RingParams;+#[kernel(TokenCopySampled)]pub fn token_copy_sampled(#[allow(unused)] src: *const u32,#[allow(unused)] dst: *mut u64,+ #[optional(has_context_ring)] context_ring: Option<*mut u32>,+ #[optional(has_context_ring)] suffix_repetition_length: Option<u32>,+ #[specialize] has_context_ring: bool,) {unsafe {- *dst.add(0) = *src.add(0) as u64;+ let token = *src.add(0);+ *dst.add(0) = token as u64;++ if has_context_ring {+ let suffix_repetition_length = suffix_repetition_length.unwrap();++ let context_ring = context_ring.unwrap();+ let ring = &mut *context_ring.cast::<RingParams>();+ let ring_tokens = context_ring.add(2);+ let slot = if ring.ring_length < suffix_repetition_length {+ let slot = (ring.ring_offset + ring.ring_length) % suffix_repetition_length;+ ring.ring_length += 1;+ slot+ } else {+ let slot = ring.ring_offset;+ ring.ring_offset = (ring.ring_offset + 1) % suffix_repetition_length;+ slot+ };+ *ring_tokens.add(slot as usize) = token;+ }}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/repetition_penalty.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/repetition_penalty.metalnew file mode 100644index 000000000..265c9bde5--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/repetition_penalty.metal@@ -0,0 +1,37 @@+#include <metal_stdlib>+#include "../common/dsl.h"+#include "../generated/ring.h"++using namespace metal;+using namespace uzu::ring;++template <typename T>+VARIANTS(T, float, bfloat)+PUBLIC KERNEL(RepetitionPenalty)(+ const device T* original_logits,+ device T* logits_copy,+ const device uint32_t* context_ring,+ const device uint64_t* token_ids,+ const constant float& repetition_penalty,+ const constant uint32_t& suffix_repetition_length,+ const uint32_t ring_index AXIS(suffix_repetition_length, 256)+) {+ const uint32_t input_token_id = static_cast<uint32_t>(token_ids[0]);+ if (ring_index == 0) {+ const float logit = static_cast<float>(original_logits[input_token_id]);+ const float penalized = logit > 0.0 ? logit / repetition_penalty : logit * repetition_penalty;+ logits_copy[input_token_id] = T(penalized);+ }++ const RingParams ring = *reinterpret_cast<const device RingParams*>(context_ring);+ if (ring_index >= ring.ring_length) {+ return;+ }++ const device uint32_t* ring_tokens = context_ring + 2;+ const uint32_t slot = (ring.ring_offset + ring_index) % suffix_repetition_length;+ const uint32_t token_id = ring_tokens[slot];+ const float logit = static_cast<float>(original_logits[token_id]);+ const float penalized = logit > 0.0 ? logit / repetition_penalty : logit * repetition_penalty;+ logits_copy[token_id] = T(penalized);+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metalindex 72d0e3c0e..eeb4ed2d9 100644--- a/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal@@ -1,10 +1,35 @@#include <metal_stdlib>#include "../common/dsl.h"+#include "../generated/ring.h"++using namespace uzu::ring;/// Copies a single sampled token (u32) to the token_ids buffer (u64)./// Used in async pipeline to pass sampled token to next forward pass.-PUBLIC KERNEL(TokenCopySampled)(device const uint32_t* src, device uint64_t* dst) {- dst[0] = static_cast<uint64_t>(src[0]);+PUBLIC KERNEL(TokenCopySampled)(+ device const uint32_t* src,+ device uint64_t* dst,+ device uint32_t* context_ring OPTIONAL(has_context_ring),+ const constant uint32_t& suffix_repetition_length OPTIONAL(has_context_ring),+ const bool has_context_ring SPECIALIZE+) {+ const uint32_t token = src[0];+ dst[0] = static_cast<uint64_t>(token);++ if (has_context_ring) {+ RingParams ring = *reinterpret_cast<device RingParams*>(context_ring);+ device uint32_t* ring_tokens = context_ring + 2;+ uint32_t slot;+ if (ring.ring_length < suffix_repetition_length) {+ slot = (ring.ring_offset + ring.ring_length) % suffix_repetition_length;+ ring.ring_length += 1;+ } else {+ slot = ring.ring_offset;+ ring.ring_offset = (ring.ring_offset + 1) % suffix_repetition_length;+ }+ ring_tokens[slot] = token;+ *reinterpret_cast<device RingParams*>(context_ring) = ring;+ }}/// Copies a single sampled token (u32) to an indexed results buffer (u32).diff --git a/crates/backend-uzu/src/config/model/generation.rs b/crates/backend-uzu/src/config/model/generation.rsindex feddaab5e..2be8ff950 100644--- a/crates/backend-uzu/src/config/model/generation.rs+++ b/crates/backend-uzu/src/config/model/generation.rs@@ -10,8 +10,8 @@ pub struct GenerationConfig {pub top_p: Option<f32>,pub min_p: Option<f32>,pub banned_tokens: Option<Unsupported>,- pub repetition_penalty: Option<Unsupported>,+ pub repetition_penalty: Option<f32>,pub presence_penalty: Option<Unsupported>,pub frequency_penalty: Option<Unsupported>,- pub suffix_repetition_length: Option<Unsupported>,+ pub suffix_repetition_length: Option<usize>,}diff --git a/crates/backend-uzu/src/config/token_codec/chat_codec.rs b/crates/backend-uzu/src/config/token_codec/chat_codec.rsindex 3881185fe..8c170fba9 100644--- a/crates/backend-uzu/src/config/token_codec/chat_codec.rs+++ b/crates/backend-uzu/src/config/token_codec/chat_codec.rs@@ -9,5 +9,6 @@ pub struct ChatCodecConfig {pub assistant_role_name: String,pub eos_token: Option<String>,pub bos_token: Option<String>,+ pub end_of_thinking_tag: Option<String>,pub default_system_prompt: Option<String>,}diff --git a/crates/backend-uzu/src/encodable_block/sampling.rs b/crates/backend-uzu/src/encodable_block/sampling.rsindex 2dd038862..15303a34f 100644--- a/crates/backend-uzu/src/encodable_block/sampling.rs+++ b/crates/backend-uzu/src/encodable_block/sampling.rs@@ -5,7 +5,10 @@ use std::{};use crate::{- backends::common::{Allocation, AllocationType, Backend, Context, Encoder, Kernels, kernel::UnifiedSamplingKernel},+ backends::common::{+ Allocation, AllocationType, AsBufferRangeRef, Backend, Context, Encoder, Kernels,+ kernel::{RepetitionPenaltyKernel, TensorCopyKernel, UnifiedSamplingKernel},+ },data_type::DataType,session::parameter::{SamplingMethod, SamplingProcessingOrder},};@@ -44,18 +47,31 @@ impl<B: Backend> Sampling<B> {logits: &Allocation<B>,seeds: Option<&Allocation<B>>,bitmask: Option<&Allocation<B>>,+ context_ring: Option<&Allocation<B>>,+ token_ids: Option<&Allocation<B>>,sampling_method: SamplingMethod,batch_size: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {// TODO: reject seeds with greedy- let (seeds, temperature, temperature_after_filters, top_k, top_p, min_p) = match sampling_method {- SamplingMethod::Greedy => (None, None, false, None, None, None),+ let (+ seeds,+ temperature,+ temperature_after_filters,+ top_k,+ top_p,+ min_p,+ repetition_penalty,+ suffix_repetition_length,+ ) = match sampling_method {+ SamplingMethod::Greedy => (None, None, false, None, None, None, None, None),SamplingMethod::Stochastic {temperature,top_k,top_p,min_p,+ repetition_penalty,+ suffix_repetition_length,processing_order,} => (Some(seeds.unwrap()),@@ -67,6 +83,8 @@ impl<B: Backend> Sampling<B> {top_k,top_p,min_p,+ repetition_penalty,+ suffix_repetition_length,),};@@ -79,6 +97,33 @@ impl<B: Backend> Sampling<B> {has_top_p: top_p.is_some(),has_min_p: min_p.is_some(),};++ let penalized_logits = if let Some(repetition_penalty) = repetition_penalty {+ let suffix_repetition_length =+ suffix_repetition_length.expect("suffix_repetition_length is required for repetition_penalty");+ assert_eq!(batch_size, 1, "repetition_penalty currently only supports batch_size == 1");++ let mut logits_copy = encoder.allocate_scratch(logits.as_buffer_range_ref().range().len())?;+ let tensor_copy = <B::Kernels as Kernels>::TensorCopyKernel::new(encoder.context(), self.data_type)?;+ tensor_copy.encode(logits, &mut logits_copy, (self.vocab_size * batch_size) as u32, encoder);++ let repetition_penalty_kernel =+ <B::Kernels as Kernels>::RepetitionPenaltyKernel::new(encoder.context(), self.data_type)?;+ repetition_penalty_kernel.encode(+ logits,+ &mut logits_copy,+ context_ring.expect("context_ring is required for repetition_penalty"),+ token_ids.expect("token_ids is required for repetition_penalty"),+ repetition_penalty,+ suffix_repetition_length as u32,+ encoder,+ );+ Some(logits_copy)+ } else {+ None+ };+ let logits = penalized_logits.as_ref().unwrap_or(logits);+let mut unified_kernels = self.unified_kernels.borrow_mut();let entry = unified_kernels.entry(key);let kernel = match entry {diff --git a/crates/backend-uzu/src/inference/chat/bridging.rs b/crates/backend-uzu/src/inference/chat/bridging.rsindex 73466ad70..e8420aaf4 100644--- a/crates/backend-uzu/src/inference/chat/bridging.rs+++ b/crates/backend-uzu/src/inference/chat/bridging.rs@@ -155,6 +155,8 @@ fn build_sampling_policy(sampling_policy: &ShojiSamplingPolicy) -> SamplingPolictop_k: top_k.map(|value| value.max(0) as u32),top_p: top_p.map(|value| value as f32),min_p: min_p.map(|value| value as f32),+ repetition_penalty: None,+ suffix_repetition_length: None,processing_order: SamplingProcessingOrder::TemperatureThenFilters,},},diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rsindex 2ddef4405..e9a78b512 100644--- a/crates/backend-uzu/src/language_model/language_model_generator.rs+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs@@ -1,6 +1,7 @@use std::{any::Any,iter::repeat_n,+ mem::size_of,ops::Range,path::Path,time::{Duration, Instant},@@ -462,6 +463,20 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {.create_allocation(std::mem::size_of_val(&sampling_seed), AllocationType::Global).expect("Failed to create sampling seed allocation");sampling_seeds.copyin(&sampling_seed);+ let repetition_penalty = Self::repetition_penalty_config(sampling_method);+ if pass_idx == 0+ && let Some((_, suffix_repetition_length)) = repetition_penalty+ {+ let capacity = self.context.async_buffers.repetition_context_ring_capacity;+ Self::copy_repetition_context_ring(+ &mut self.context.async_buffers.repetition_context_ring,+ &self.tokens,+ suffix_repetition_length,+ capacity,+ );+ }+ let sampling_context_ring =+ repetition_penalty.as_ref().map(|_| &self.context.async_buffers.repetition_context_ring);let is_first_decode = !is_continuation;let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);@@ -488,6 +503,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {Some(sampling_method),Some(sampling_seeds),None,+ sampling_context_ring,)?;// Copy sampled token: sampling_output → token_ids (for next pass)@@ -501,11 +517,23 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {(async_token_ids_buffer_range.buffer().cpu_ptr().as_ptr() as *const u64).add(async_token_ids_range.start / std::mem::size_of::<u64>())});- self.context.token_copy_sampled.encode(- resources.sampling_output.as_ref().expect("Sampling output must exist"),- &mut async_token_ids_allocation,- &mut encoder,- );+ if let Some((_, suffix_repetition_length)) = repetition_penalty {+ self.context.token_copy_sampled_context_ring.encode(+ resources.sampling_output.as_ref().expect("Sampling output must exist"),+ &mut async_token_ids_allocation,+ Some(&mut self.context.async_buffers.repetition_context_ring),+ Some(suffix_repetition_length as u32),+ &mut encoder,+ );+ } else {+ self.context.token_copy_sampled.encode(+ resources.sampling_output.as_ref().expect("Sampling output must exist"),+ &mut async_token_ids_allocation,+ None::<&mut Allocation<B>>,+ None,+ &mut encoder,+ );+ }self.async_token_ids = Some(unsafe {Array::from_allocation(async_token_ids_allocation, 0, &token_ids_shape, token_ids_data_type)});@@ -662,6 +690,35 @@ impl<B: Backend> LanguageModelGenerator<B> {})}+ fn repetition_penalty_config(sampling_method: SamplingMethod) -> Option<(f32, usize)> {+ match sampling_method {+ SamplingMethod::Stochastic {+ repetition_penalty: Some(repetition_penalty),+ suffix_repetition_length,+ ..+ } => Some((+ repetition_penalty,+ suffix_repetition_length.expect("suffix_repetition_length is required for repetition_penalty"),+ )),+ _ => None,+ }+ }++ fn copy_repetition_context_ring(+ context_ring: &mut Allocation<B>,+ tokens: &[u64],+ suffix_repetition_length: usize,+ capacity: usize,+ ) {+ let ring_length = tokens.len().min(suffix_repetition_length);+ let mut packed_ring = vec![0u32; 2 + capacity];+ packed_ring[1] = ring_length as u32;+ for (index, token) in tokens[tokens.len() - ring_length..].iter().enumerate() {+ packed_ring[2 + index] = *token as u32;+ }+ context_ring.copyin(&packed_ring);+ }+pub fn get_generate_task(&self,flat_trie: &FlatTrie,@@ -727,6 +784,27 @@ impl<B: Backend> LanguageModelGenerator<B> {let batch_dim = task.active_row_count;let sampling_start = task.sampling_start;let sampling_length = task.sampling_length;+ let repetition_penalty = if sample {+ Self::repetition_penalty_config(sampling_method)+ } else {+ None+ };+ let sampling_context_ring = if let Some((_, suffix_repetition_length)) = repetition_penalty {+ let mut allocation = self+ .context+ .context+ .create_allocation((2 + suffix_repetition_length) * size_of::<u32>(), AllocationType::Global)+ .map_err(|e| Error::UnableToCreateContext(e.into()))?;+ Self::copy_repetition_context_ring(+ &mut allocation,+ &self.tokens,+ suffix_repetition_length,+ suffix_repetition_length,+ );+ Some(allocation)+ } else {+ None+ };let (sampling_seeds, sampling_bitmask) = if sample {let seeds = &task.token_seeds[task.sampling_start..task.sampling_start + task.sampling_length];let mut seed_allocation = self@@ -778,6 +856,7 @@ impl<B: Backend> LanguageModelGenerator<B> {sample.then_some(sampling_method),sampling_seeds,sampling_bitmask,+ sampling_context_ring.as_ref(),)?;let pending = encoder.end_encoding().submit();@@ -820,6 +899,7 @@ impl<B: Backend> LanguageModelGenerator<B> {sampling_method: Option<SamplingMethod>,sampling_seeds: Option<Allocation<B>>,sampling_bitmask: Option<Allocation<B>>,+ sampling_context_ring: Option<&Allocation<B>>,) -> Result<ForwardPassResources<B>, Error> {let mut sampling_output = None;let mut logits = None;@@ -859,6 +939,8 @@ impl<B: Backend> LanguageModelGenerator<B> {&retained_logits,Some(sampling_seeds),sampling_bitmask.as_ref(),+ sampling_context_ring,+ Some(token_inputs.token_ids()),sampling_method.expect("Sampling requires method"),sampling_length,encoder,diff --git a/crates/backend-uzu/src/language_model/language_model_generator_context.rs b/crates/backend-uzu/src/language_model/language_model_generator_context.rsindex 9fd45ca9d..c491ecb29 100644--- a/crates/backend-uzu/src/language_model/language_model_generator_context.rs+++ b/crates/backend-uzu/src/language_model/language_model_generator_context.rs@@ -2,13 +2,14 @@ use std::{cell::{Cell, RefCell},cmp::min,fs::File,+ mem::size_of,path::Path,rc::Rc,};use crate::{array::{Array, ArrayContextExt},- backends::common::{Backend, Context, Kernels, kernel::TokenCopySampledKernel},+ backends::common::{Allocation, AllocationType, Backend, Context, Kernels, kernel::TokenCopySampledKernel},config::{decoder::DecoderConfig, model::language_model::LanguageModelConfig},data_type::DataType,encodable_block::{Decoder, KVCacheUpdate, Sampling},@@ -38,6 +39,9 @@ pub struct AsyncBuffers<B: Backend> {pub prefill_count: Cell<usize>,/// Batch size (number of passes to keep in flight)pub batch_size: usize,+ /// Packed [ring_offset, ring_length, token...].+ pub repetition_context_ring: Allocation<B>,+ pub repetition_context_ring_capacity: usize,}impl<B: Backend> AsyncBuffers<B> {@@ -48,6 +52,10 @@ impl<B: Backend> AsyncBuffers<B> {) -> Self {let positions = context.create_array_uninitialized(&[max_tokens], DataType::I32);let seeds = context.create_array_uninitialized(&[max_tokens], DataType::U64);+ let repetition_context_ring_capacity = max_tokens;+ let repetition_context_ring = context+ .create_allocation((2 + repetition_context_ring_capacity) * size_of::<u32>(), AllocationType::Global)+ .expect("Failed to create repetition context ring allocation");Self {positions,@@ -55,6 +63,8 @@ impl<B: Backend> AsyncBuffers<B> {counter: Cell::new(0),prefill_count: Cell::new(0),batch_size,+ repetition_context_ring,+ repetition_context_ring_capacity,}}@@ -106,6 +116,7 @@ pub struct LanguageModelGeneratorContext<B: Backend> {/// Kernels for copying sampled tokens in async pipelinepub token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,+ pub token_copy_sampled_context_ring: <B::Kernels as Kernels>::TokenCopySampledKernel,/// Pre-allocated buffers for async generationpub async_buffers: AsyncBuffers<B>,}@@ -149,7 +160,9 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {let gpu_sampler = Sampling::<B>::new(model_shape.data_type, model_config.decoder_config.vocab_size);- let token_copy_sampled = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context)+ let token_copy_sampled = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context, false)+ .map_err(|e| Error::UnableToCreateContext(e.into()))?;+ let token_copy_sampled_context_ring = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context, true).map_err(|e| Error::UnableToCreateContext(e.into()))?;let async_batch_size = decoding_config@@ -172,6 +185,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {gpu_sampler,seed,token_copy_sampled,+ token_copy_sampled_context_ring,async_buffers,})}diff --git a/crates/backend-uzu/src/session/parameter/sampling_policy.rs b/crates/backend-uzu/src/session/parameter/sampling_policy.rsindex 23238ce99..a6b96cf98 100644--- a/crates/backend-uzu/src/session/parameter/sampling_policy.rs+++ b/crates/backend-uzu/src/session/parameter/sampling_policy.rs@@ -14,6 +14,8 @@ pub enum SamplingMethod {top_k: Option<u32>,top_p: Option<f32>,min_p: Option<f32>,+ repetition_penalty: Option<f32>,+ suffix_repetition_length: Option<usize>,processing_order: SamplingProcessingOrder,},}@@ -38,6 +40,8 @@ impl ConfigResolvableValue<LanguageModelConfig, SamplingMethod> for SamplingPolitop_k: generation_config.top_k,top_p: generation_config.top_p,min_p: generation_config.min_p,+ repetition_penalty: generation_config.repetition_penalty,+ suffix_repetition_length: generation_config.suffix_repetition_length,processing_order: SamplingProcessingOrder::TemperatureThenFilters,},SamplingPolicy::Custom {diff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rsindex fa633f6b7..4a9b5f33f 100644--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs@@ -46,8 +46,8 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {).map_err(|error| Error::UnableToCreateDecoder(Box::new(error)))?;let sampler = GpuSampling::new(model_shape.data_type, decoder_config.vocab_size);- let token_copy_sampled =- <B::Kernels as Kernels>::TokenCopySampledKernel::new(context.as_ref()).map_err(unable_to_create_context)?;+ let token_copy_sampled = <B::Kernels as Kernels>::TokenCopySampledKernel::new(context.as_ref(), false)+ .map_err(unable_to_create_context)?;Ok(Self {executables,@@ -281,7 +281,7 @@ impl<B: Backend> TokenDecoderRunner<B> {let sampling_bitmask = bitmask_index.map(|index| &pending_sampling_allocations[index]);self.ctx.sampler- .encode(logits, Some(sampling_seeds), sampling_bitmask, sampling_method, batch_size, encoder)+ .encode(logits, Some(sampling_seeds), sampling_bitmask, None, None, sampling_method, batch_size, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))}@@ -482,7 +482,13 @@ impl<B: Backend> TokenDecoderRunner<B> {let token_ids_data_type = DataType::U64;let mut next_token_ids =self.ctx.context.create_array_uninitialized(&token_ids_shape, token_ids_data_type).into_allocation();- self.ctx.token_copy_sampled.encode(&sampling_output, &mut next_token_ids, encoder);+ self.ctx.token_copy_sampled.encode(+ &sampling_output,+ &mut next_token_ids,+ None::<&mut Allocation<B>>,+ None,+ encoder,+ );Some(unsafe { Array::from_allocation(next_token_ids, 0, &token_ids_shape, token_ids_data_type) })} else {None@@ -588,7 +594,13 @@ impl<B: Backend> TokenDecoderRunner<B> {.context.create_array_uninitialized(&token_ids_shape, token_ids_data_type).into_allocation();- self.ctx.token_copy_sampled.encode(&sampling_output, &mut updated_token_ids, encoder);+ self.ctx.token_copy_sampled.encode(+ &sampling_output,+ &mut updated_token_ids,+ None::<&mut Allocation<B>>,+ None,+ encoder,+ );next_token_ids = Some(unsafe {Array::from_allocation(updated_token_ids, 0, &token_ids_shape, token_ids_data_type)});diff --git a/crates/backend-uzu/src/session/tts_session/decoder_support.rs b/crates/backend-uzu/src/session/tts_session/decoder_support.rsindex 2dcbfcda9..eb93ab76a 100644--- a/crates/backend-uzu/src/session/tts_session/decoder_support.rs+++ b/crates/backend-uzu/src/session/tts_session/decoder_support.rs@@ -37,6 +37,8 @@ impl TextSamplingState {top_k: None,top_p: Some(top_p),min_p: None,+ repetition_penalty: None,+ suffix_repetition_length: None,processing_order: SamplingProcessingOrder::FiltersThenTemperature,}};diff --git a/crates/backend-uzu/src/utils/version.rs b/crates/backend-uzu/src/utils/version.rsindex 2c39d1571..5b23307cd 100644--- a/crates/backend-uzu/src/utils/version.rs+++ b/crates/backend-uzu/src/utils/version.rs@@ -1,2 +1,2 @@pub const VERSION: &str = env!("CARGO_PKG_VERSION");-pub const TOOLCHAIN_VERSION: &str = "0.12.1";+pub const TOOLCHAIN_VERSION: &str = "0.13.0";diff --git a/crates/backend-uzu/tests/integration/session/chat_session/sampling_test.rs b/crates/backend-uzu/tests/integration/session/chat_session/sampling_test.rsindex f21af965b..59f537ac2 100644--- a/crates/backend-uzu/tests/integration/session/chat_session/sampling_test.rs+++ b/crates/backend-uzu/tests/integration/session/chat_session/sampling_test.rs@@ -48,6 +48,8 @@ fn stochastic(temperature: f32) -> SamplingPolicy {top_k: None,top_p: None,min_p: None,+ repetition_penalty: None, // TODO+ suffix_repetition_length: None, // TODOprocessing_order: SamplingProcessingOrder::TemperatureThenFilters,},}diff --git a/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs b/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rsindex 61a1bd99d..f3030f00a 100644--- a/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs@@ -70,6 +70,8 @@ fn do_sampling_backend<B: Backend, T: ArrayElement + Float>(&logits_allocation,seeds_allocation.as_ref(),bitmask_allocation.as_ref(),+ None, // TODO+ None, // TODOmethod,batch_size,&mut encoder,@@ -147,6 +149,8 @@ fn sampling_case() -> impl Strategy<Value = SamplingCase> {top_k,top_p,min_p,+ repetition_penalty: None, // TODO+ suffix_repetition_length: None, // TODOprocessing_order,}},diff --git a/crates/backend-uzu/tests/unit/kernel/token_copy_test.rs b/crates/backend-uzu/tests/unit/kernel/token_copy_test.rsindex e9713c9d1..45fba0c1a 100644--- a/crates/backend-uzu/tests/unit/kernel/token_copy_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/token_copy_test.rs@@ -1,5 +1,5 @@use backend_uzu::backends::common::{- Backend, Context, Encoder, Kernels,+ Allocation, Backend, Context, Encoder, Kernels,kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel},};@@ -11,14 +11,14 @@ use crate::{fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {let context = B::Context::new().expect("Failed to create Context");- let kernel = <<B as Backend>::Kernels as Kernels>::TokenCopySampledKernel::new(&context)+ let kernel = <<B as Backend>::Kernels as Kernels>::TokenCopySampledKernel::new(&context, false).expect("Failed to create TokenCopySampledKernel");let src_allocation = alloc_allocation_with_data::<B, u32>(&context, &[src_value]);let mut dst_allocation = alloc_allocation::<B, u64>(&context, 1);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(&src_allocation, &mut dst_allocation, &mut encoder);+ kernel.encode(&src_allocation, &mut dst_allocation, None::<&mut Allocation<B>>, None, &mut encoder);encoder.end_encoding().submit().wait_until_completed().unwrap();let output = allocation_to_vec::<B, u64>(&dst_allocation);