Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Lalamo 0.13.0 (with very slop repetition penalty) (#518)

uuuvn113394947db9
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/sampling/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/sampling/mod.rs
index 5171b2be1..6a251ef64 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/sampling/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/sampling/mod.rs
@@ -1 +1,2 @@
+pub mod repetition_penalty;
pub mod unified_sampling;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/sampling/repetition_penalty.rs b/crates/backend-uzu/src/backends/cpu/kernel/sampling/repetition_penalty.rs
new file mode 100644
index 000000000..7546210c3
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/sampling/repetition_penalty.rs
@@ -0,0 +1,46 @@
+use half::bf16;
+use num_traits::{Float, NumCast};
+use proc_macros::kernel;
+
+use crate::{array::ArrayElement, backends::common::gpu_types::ring::RingParams};
+
+#[kernel(RepetitionPenalty)]
+#[variants(T, f32, bf16)]
+pub fn repetition_penalty<T: ArrayElement + Float + NumCast>(
+ original_logits: *const T,
+ logits_copy: *mut T,
+ context_ring: *const u32,
+ token_ids: *const u64,
+ repetition_penalty: f32,
+ suffix_repetition_length: u32,
+) {
+ fn apply<T: ArrayElement + Float + NumCast>(
+ original_logits: *const T,
+ logits_copy: *mut T,
+ token_id: usize,
+ repetition_penalty: f32,
+ ) {
+ let logit = unsafe { (*original_logits.add(token_id)).to_f32().unwrap() };
+ let penalized = if logit > 0.0 {
+ logit / repetition_penalty
+ } else {
+ logit * repetition_penalty
+ };
+ unsafe {
+ *logits_copy.add(token_id) = NumCast::from(penalized).unwrap();
+ }
+ }
+
+ let ring = unsafe { &*context_ring.cast::<RingParams>() };
+ let ring_tokens = unsafe { context_ring.add(2) };
+
+ apply(original_logits, logits_copy, unsafe { *token_ids } as usize, repetition_penalty);
+
+ for ring_index in 0..suffix_repetition_length {
+ if ring_index >= ring.ring_length {
+ continue;
+ }
+ let slot = (ring.ring_offset + ring_index) % suffix_repetition_length;
+ apply(original_logits, logits_copy, unsafe { *ring_tokens.add(slot as usize) } as usize, repetition_penalty)
+ }
+}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/token_copy/token_copy.rs b/crates/backend-uzu/src/backends/cpu/kernel/token_copy/token_copy.rs
index 73a4355a2..693ce9517 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/token_copy/token_copy.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/token_copy/token_copy.rs
@@ -1,12 +1,36 @@
use proc_macros::kernel;
+use crate::backends::common::gpu_types::ring::RingParams;
+
#[kernel(TokenCopySampled)]
pub fn token_copy_sampled(
#[allow(unused)] src: *const u32,
#[allow(unused)] dst: *mut u64,
+ #[optional(has_context_ring)] context_ring: Option<*mut u32>,
+ #[optional(has_context_ring)] suffix_repetition_length: Option<u32>,
+ #[specialize] has_context_ring: bool,
) {
unsafe {
- *dst.add(0) = *src.add(0) as u64;
+ let token = *src.add(0);
+ *dst.add(0) = token as u64;
+
+ if has_context_ring {
+ let suffix_repetition_length = suffix_repetition_length.unwrap();
+
+ let context_ring = context_ring.unwrap();
+ let ring = &mut *context_ring.cast::<RingParams>();
+ let ring_tokens = context_ring.add(2);
+ let slot = if ring.ring_length < suffix_repetition_length {
+ let slot = (ring.ring_offset + ring.ring_length) % suffix_repetition_length;
+ ring.ring_length += 1;
+ slot
+ } else {
+ let slot = ring.ring_offset;
+ ring.ring_offset = (ring.ring_offset + 1) % suffix_repetition_length;
+ slot
+ };
+ *ring_tokens.add(slot as usize) = token;
+ }
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/repetition_penalty.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/repetition_penalty.metal
new file mode 100644
index 000000000..265c9bde5
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/repetition_penalty.metal
@@ -0,0 +1,37 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+#include "../generated/ring.h"
+
+using namespace metal;
+using namespace uzu::ring;
+
+template <typename T>
+VARIANTS(T, float, bfloat)
+PUBLIC KERNEL(RepetitionPenalty)(
+ const device T* original_logits,
+ device T* logits_copy,
+ const device uint32_t* context_ring,
+ const device uint64_t* token_ids,
+ const constant float& repetition_penalty,
+ const constant uint32_t& suffix_repetition_length,
+ const uint32_t ring_index AXIS(suffix_repetition_length, 256)
+) {
+ const uint32_t input_token_id = static_cast<uint32_t>(token_ids[0]);
+ if (ring_index == 0) {
+ const float logit = static_cast<float>(original_logits[input_token_id]);
+ const float penalized = logit > 0.0 ? logit / repetition_penalty : logit * repetition_penalty;
+ logits_copy[input_token_id] = T(penalized);
+ }
+
+ const RingParams ring = *reinterpret_cast<const device RingParams*>(context_ring);
+ if (ring_index >= ring.ring_length) {
+ return;
+ }
+
+ const device uint32_t* ring_tokens = context_ring + 2;
+ const uint32_t slot = (ring.ring_offset + ring_index) % suffix_repetition_length;
+ const uint32_t token_id = ring_tokens[slot];
+ const float logit = static_cast<float>(original_logits[token_id]);
+ const float penalized = logit > 0.0 ? logit / repetition_penalty : logit * repetition_penalty;
+ logits_copy[token_id] = T(penalized);
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal
index 72d0e3c0e..eeb4ed2d9 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal
@@ -1,10 +1,35 @@
#include <metal_stdlib>
#include "../common/dsl.h"
+#include "../generated/ring.h"
+
+using namespace uzu::ring;
/// Copies a single sampled token (u32) to the token_ids buffer (u64).
/// Used in async pipeline to pass sampled token to next forward pass.
-PUBLIC KERNEL(TokenCopySampled)(device const uint32_t* src, device uint64_t* dst) {
- dst[0] = static_cast<uint64_t>(src[0]);
+PUBLIC KERNEL(TokenCopySampled)(
+ device const uint32_t* src,
+ device uint64_t* dst,
+ device uint32_t* context_ring OPTIONAL(has_context_ring),
+ const constant uint32_t& suffix_repetition_length OPTIONAL(has_context_ring),
+ const bool has_context_ring SPECIALIZE
+) {
+ const uint32_t token = src[0];
+ dst[0] = static_cast<uint64_t>(token);
+
+ if (has_context_ring) {
+ RingParams ring = *reinterpret_cast<device RingParams*>(context_ring);
+ device uint32_t* ring_tokens = context_ring + 2;
+ uint32_t slot;
+ if (ring.ring_length < suffix_repetition_length) {
+ slot = (ring.ring_offset + ring.ring_length) % suffix_repetition_length;
+ ring.ring_length += 1;
+ } else {
+ slot = ring.ring_offset;
+ ring.ring_offset = (ring.ring_offset + 1) % suffix_repetition_length;
+ }
+ ring_tokens[slot] = token;
+ *reinterpret_cast<device RingParams*>(context_ring) = ring;
+ }
}
/// Copies a single sampled token (u32) to an indexed results buffer (u32).
diff --git a/crates/backend-uzu/src/config/model/generation.rs b/crates/backend-uzu/src/config/model/generation.rs
index feddaab5e..2be8ff950 100644
--- a/crates/backend-uzu/src/config/model/generation.rs
+++ b/crates/backend-uzu/src/config/model/generation.rs
@@ -10,8 +10,8 @@ pub struct GenerationConfig {
pub top_p: Option<f32>,
pub min_p: Option<f32>,
pub banned_tokens: Option<Unsupported>,
- pub repetition_penalty: Option<Unsupported>,
+ pub repetition_penalty: Option<f32>,
pub presence_penalty: Option<Unsupported>,
pub frequency_penalty: Option<Unsupported>,
- pub suffix_repetition_length: Option<Unsupported>,
+ pub suffix_repetition_length: Option<usize>,
}
diff --git a/crates/backend-uzu/src/config/token_codec/chat_codec.rs b/crates/backend-uzu/src/config/token_codec/chat_codec.rs
index 3881185fe..8c170fba9 100644
--- a/crates/backend-uzu/src/config/token_codec/chat_codec.rs
+++ b/crates/backend-uzu/src/config/token_codec/chat_codec.rs
@@ -9,5 +9,6 @@ pub struct ChatCodecConfig {
pub assistant_role_name: String,
pub eos_token: Option<String>,
pub bos_token: Option<String>,
+ pub end_of_thinking_tag: Option<String>,
pub default_system_prompt: Option<String>,
}
diff --git a/crates/backend-uzu/src/encodable_block/sampling.rs b/crates/backend-uzu/src/encodable_block/sampling.rs
index 2dd038862..15303a34f 100644
--- a/crates/backend-uzu/src/encodable_block/sampling.rs
+++ b/crates/backend-uzu/src/encodable_block/sampling.rs
@@ -5,7 +5,10 @@ use std::{
};
use crate::{
- backends::common::{Allocation, AllocationType, Backend, Context, Encoder, Kernels, kernel::UnifiedSamplingKernel},
+ backends::common::{
+ Allocation, AllocationType, AsBufferRangeRef, Backend, Context, Encoder, Kernels,
+ kernel::{RepetitionPenaltyKernel, TensorCopyKernel, UnifiedSamplingKernel},
+ },
data_type::DataType,
session::parameter::{SamplingMethod, SamplingProcessingOrder},
};
@@ -44,18 +47,31 @@ impl<B: Backend> Sampling<B> {
logits: &Allocation<B>,
seeds: Option<&Allocation<B>>,
bitmask: Option<&Allocation<B>>,
+ context_ring: Option<&Allocation<B>>,
+ token_ids: Option<&Allocation<B>>,
sampling_method: SamplingMethod,
batch_size: usize,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
// TODO: reject seeds with greedy
- let (seeds, temperature, temperature_after_filters, top_k, top_p, min_p) = match sampling_method {
- SamplingMethod::Greedy => (None, None, false, None, None, None),
+ let (
+ seeds,
+ temperature,
+ temperature_after_filters,
+ top_k,
+ top_p,
+ min_p,
+ repetition_penalty,
+ suffix_repetition_length,
+ ) = match sampling_method {
+ SamplingMethod::Greedy => (None, None, false, None, None, None, None, None),
SamplingMethod::Stochastic {
temperature,
top_k,
top_p,
min_p,
+ repetition_penalty,
+ suffix_repetition_length,
processing_order,
} => (
Some(seeds.unwrap()),
@@ -67,6 +83,8 @@ impl<B: Backend> Sampling<B> {
top_k,
top_p,
min_p,
+ repetition_penalty,
+ suffix_repetition_length,
),
};
@@ -79,6 +97,33 @@ impl<B: Backend> Sampling<B> {
has_top_p: top_p.is_some(),
has_min_p: min_p.is_some(),
};
+
+ let penalized_logits = if let Some(repetition_penalty) = repetition_penalty {
+ let suffix_repetition_length =
+ suffix_repetition_length.expect("suffix_repetition_length is required for repetition_penalty");
+ assert_eq!(batch_size, 1, "repetition_penalty currently only supports batch_size == 1");
+
+ let mut logits_copy = encoder.allocate_scratch(logits.as_buffer_range_ref().range().len())?;
+ let tensor_copy = <B::Kernels as Kernels>::TensorCopyKernel::new(encoder.context(), self.data_type)?;
+ tensor_copy.encode(logits, &mut logits_copy, (self.vocab_size * batch_size) as u32, encoder);
+
+ let repetition_penalty_kernel =
+ <B::Kernels as Kernels>::RepetitionPenaltyKernel::new(encoder.context(), self.data_type)?;
+ repetition_penalty_kernel.encode(
+ logits,
+ &mut logits_copy,
+ context_ring.expect("context_ring is required for repetition_penalty"),
+ token_ids.expect("token_ids is required for repetition_penalty"),
+ repetition_penalty,
+ suffix_repetition_length as u32,
+ encoder,
+ );
+ Some(logits_copy)
+ } else {
+ None
+ };
+ let logits = penalized_logits.as_ref().unwrap_or(logits);
+
let mut unified_kernels = self.unified_kernels.borrow_mut();
let entry = unified_kernels.entry(key);
let kernel = match entry {
diff --git a/crates/backend-uzu/src/inference/chat/bridging.rs b/crates/backend-uzu/src/inference/chat/bridging.rs
index 73466ad70..e8420aaf4 100644
--- a/crates/backend-uzu/src/inference/chat/bridging.rs
+++ b/crates/backend-uzu/src/inference/chat/bridging.rs
@@ -155,6 +155,8 @@ fn build_sampling_policy(sampling_policy: &ShojiSamplingPolicy) -> SamplingPolic
top_k: top_k.map(|value| value.max(0) as u32),
top_p: top_p.map(|value| value as f32),
min_p: min_p.map(|value| value as f32),
+ repetition_penalty: None,
+ suffix_repetition_length: None,
processing_order: SamplingProcessingOrder::TemperatureThenFilters,
},
},
diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rs
index 2ddef4405..e9a78b512 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs
@@ -1,6 +1,7 @@
use std::{
any::Any,
iter::repeat_n,
+ mem::size_of,
ops::Range,
path::Path,
time::{Duration, Instant},
@@ -462,6 +463,20 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
.create_allocation(std::mem::size_of_val(&sampling_seed), AllocationType::Global)
.expect("Failed to create sampling seed allocation");
sampling_seeds.copyin(&sampling_seed);
+ let repetition_penalty = Self::repetition_penalty_config(sampling_method);
+ if pass_idx == 0
+ && let Some((_, suffix_repetition_length)) = repetition_penalty
+ {
+ let capacity = self.context.async_buffers.repetition_context_ring_capacity;
+ Self::copy_repetition_context_ring(
+ &mut self.context.async_buffers.repetition_context_ring,
+ &self.tokens,
+ suffix_repetition_length,
+ capacity,
+ );
+ }
+ let sampling_context_ring =
+ repetition_penalty.as_ref().map(|_| &self.context.async_buffers.repetition_context_ring);
let is_first_decode = !is_continuation;
let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);
@@ -488,6 +503,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
Some(sampling_method),
Some(sampling_seeds),
None,
+ sampling_context_ring,
)?;
// Copy sampled token: sampling_output → token_ids (for next pass)
@@ -501,11 +517,23 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
(async_token_ids_buffer_range.buffer().cpu_ptr().as_ptr() as *const u64)
.add(async_token_ids_range.start / std::mem::size_of::<u64>())
});
- self.context.token_copy_sampled.encode(
- resources.sampling_output.as_ref().expect("Sampling output must exist"),
- &mut async_token_ids_allocation,
- &mut encoder,
- );
+ if let Some((_, suffix_repetition_length)) = repetition_penalty {
+ self.context.token_copy_sampled_context_ring.encode(
+ resources.sampling_output.as_ref().expect("Sampling output must exist"),
+ &mut async_token_ids_allocation,
+ Some(&mut self.context.async_buffers.repetition_context_ring),
+ Some(suffix_repetition_length as u32),
+ &mut encoder,
+ );
+ } else {
+ self.context.token_copy_sampled.encode(
+ resources.sampling_output.as_ref().expect("Sampling output must exist"),
+ &mut async_token_ids_allocation,
+ None::<&mut Allocation<B>>,
+ None,
+ &mut encoder,
+ );
+ }
self.async_token_ids = Some(unsafe {
Array::from_allocation(async_token_ids_allocation, 0, &token_ids_shape, token_ids_data_type)
});
@@ -662,6 +690,35 @@ impl<B: Backend> LanguageModelGenerator<B> {
})
}
+ fn repetition_penalty_config(sampling_method: SamplingMethod) -> Option<(f32, usize)> {
+ match sampling_method {
+ SamplingMethod::Stochastic {
+ repetition_penalty: Some(repetition_penalty),
+ suffix_repetition_length,
+ ..
+ } => Some((
+ repetition_penalty,
+ suffix_repetition_length.expect("suffix_repetition_length is required for repetition_penalty"),
+ )),
+ _ => None,
+ }
+ }
+
+ fn copy_repetition_context_ring(
+ context_ring: &mut Allocation<B>,
+ tokens: &[u64],
+ suffix_repetition_length: usize,
+ capacity: usize,
+ ) {
+ let ring_length = tokens.len().min(suffix_repetition_length);
+ let mut packed_ring = vec![0u32; 2 + capacity];
+ packed_ring[1] = ring_length as u32;
+ for (index, token) in tokens[tokens.len() - ring_length..].iter().enumerate() {
+ packed_ring[2 + index] = *token as u32;
+ }
+ context_ring.copyin(&packed_ring);
+ }
+
pub fn get_generate_task(
&self,
flat_trie: &FlatTrie,
@@ -727,6 +784,27 @@ impl<B: Backend> LanguageModelGenerator<B> {
let batch_dim = task.active_row_count;
let sampling_start = task.sampling_start;
let sampling_length = task.sampling_length;
+ let repetition_penalty = if sample {
+ Self::repetition_penalty_config(sampling_method)
+ } else {
+ None
+ };
+ let sampling_context_ring = if let Some((_, suffix_repetition_length)) = repetition_penalty {
+ let mut allocation = self
+ .context
+ .context
+ .create_allocation((2 + suffix_repetition_length) * size_of::<u32>(), AllocationType::Global)
+ .map_err(|e| Error::UnableToCreateContext(e.into()))?;
+ Self::copy_repetition_context_ring(
+ &mut allocation,
+ &self.tokens,
+ suffix_repetition_length,
+ suffix_repetition_length,
+ );
+ Some(allocation)
+ } else {
+ None
+ };
let (sampling_seeds, sampling_bitmask) = if sample {
let seeds = &task.token_seeds[task.sampling_start..task.sampling_start + task.sampling_length];
let mut seed_allocation = self
@@ -778,6 +856,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
sample.then_some(sampling_method),
sampling_seeds,
sampling_bitmask,
+ sampling_context_ring.as_ref(),
)?;
let pending = encoder.end_encoding().submit();
@@ -820,6 +899,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
sampling_method: Option<SamplingMethod>,
sampling_seeds: Option<Allocation<B>>,
sampling_bitmask: Option<Allocation<B>>,
+ sampling_context_ring: Option<&Allocation<B>>,
) -> Result<ForwardPassResources<B>, Error> {
let mut sampling_output = None;
let mut logits = None;
@@ -859,6 +939,8 @@ impl<B: Backend> LanguageModelGenerator<B> {
&retained_logits,
Some(sampling_seeds),
sampling_bitmask.as_ref(),
+ sampling_context_ring,
+ Some(token_inputs.token_ids()),
sampling_method.expect("Sampling requires method"),
sampling_length,
encoder,
diff --git a/crates/backend-uzu/src/language_model/language_model_generator_context.rs b/crates/backend-uzu/src/language_model/language_model_generator_context.rs
index 9fd45ca9d..c491ecb29 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator_context.rs
@@ -2,13 +2,14 @@ use std::{
cell::{Cell, RefCell},
cmp::min,
fs::File,
+ mem::size_of,
path::Path,
rc::Rc,
};
use crate::{
array::{Array, ArrayContextExt},
- backends::common::{Backend, Context, Kernels, kernel::TokenCopySampledKernel},
+ backends::common::{Allocation, AllocationType, Backend, Context, Kernels, kernel::TokenCopySampledKernel},
config::{decoder::DecoderConfig, model::language_model::LanguageModelConfig},
data_type::DataType,
encodable_block::{Decoder, KVCacheUpdate, Sampling},
@@ -38,6 +39,9 @@ pub struct AsyncBuffers<B: Backend> {
pub prefill_count: Cell<usize>,
/// Batch size (number of passes to keep in flight)
pub batch_size: usize,
+ /// Packed [ring_offset, ring_length, token...].
+ pub repetition_context_ring: Allocation<B>,
+ pub repetition_context_ring_capacity: usize,
}
impl<B: Backend> AsyncBuffers<B> {
@@ -48,6 +52,10 @@ impl<B: Backend> AsyncBuffers<B> {
) -> Self {
let positions = context.create_array_uninitialized(&[max_tokens], DataType::I32);
let seeds = context.create_array_uninitialized(&[max_tokens], DataType::U64);
+ let repetition_context_ring_capacity = max_tokens;
+ let repetition_context_ring = context
+ .create_allocation((2 + repetition_context_ring_capacity) * size_of::<u32>(), AllocationType::Global)
+ .expect("Failed to create repetition context ring allocation");
Self {
positions,
@@ -55,6 +63,8 @@ impl<B: Backend> AsyncBuffers<B> {
counter: Cell::new(0),
prefill_count: Cell::new(0),
batch_size,
+ repetition_context_ring,
+ repetition_context_ring_capacity,
}
}
@@ -106,6 +116,7 @@ pub struct LanguageModelGeneratorContext<B: Backend> {
/// Kernels for copying sampled tokens in async pipeline
pub token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,
+ pub token_copy_sampled_context_ring: <B::Kernels as Kernels>::TokenCopySampledKernel,
/// Pre-allocated buffers for async generation
pub async_buffers: AsyncBuffers<B>,
}
@@ -149,7 +160,9 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
let gpu_sampler = Sampling::<B>::new(model_shape.data_type, model_config.decoder_config.vocab_size);
- let token_copy_sampled = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context)
+ let token_copy_sampled = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context, false)
+ .map_err(|e| Error::UnableToCreateContext(e.into()))?;
+ let token_copy_sampled_context_ring = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context, true)
.map_err(|e| Error::UnableToCreateContext(e.into()))?;
let async_batch_size = decoding_config
@@ -172,6 +185,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
gpu_sampler,
seed,
token_copy_sampled,
+ token_copy_sampled_context_ring,
async_buffers,
})
}
diff --git a/crates/backend-uzu/src/session/parameter/sampling_policy.rs b/crates/backend-uzu/src/session/parameter/sampling_policy.rs
index 23238ce99..a6b96cf98 100644
--- a/crates/backend-uzu/src/session/parameter/sampling_policy.rs
+++ b/crates/backend-uzu/src/session/parameter/sampling_policy.rs
@@ -14,6 +14,8 @@ pub enum SamplingMethod {
top_k: Option<u32>,
top_p: Option<f32>,
min_p: Option<f32>,
+ repetition_penalty: Option<f32>,
+ suffix_repetition_length: Option<usize>,
processing_order: SamplingProcessingOrder,
},
}
@@ -38,6 +40,8 @@ impl ConfigResolvableValue<LanguageModelConfig, SamplingMethod> for SamplingPoli
top_k: generation_config.top_k,
top_p: generation_config.top_p,
min_p: generation_config.min_p,
+ repetition_penalty: generation_config.repetition_penalty,
+ suffix_repetition_length: generation_config.suffix_repetition_length,
processing_order: SamplingProcessingOrder::TemperatureThenFilters,
},
SamplingPolicy::Custom {
diff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
index fa633f6b7..4a9b5f33f 100644
--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
@@ -46,8 +46,8 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {
)
.map_err(|error| Error::UnableToCreateDecoder(Box::new(error)))?;
let sampler = GpuSampling::new(model_shape.data_type, decoder_config.vocab_size);
- let token_copy_sampled =
- <B::Kernels as Kernels>::TokenCopySampledKernel::new(context.as_ref()).map_err(unable_to_create_context)?;
+ let token_copy_sampled = <B::Kernels as Kernels>::TokenCopySampledKernel::new(context.as_ref(), false)
+ .map_err(unable_to_create_context)?;
Ok(Self {
executables,
@@ -281,7 +281,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
let sampling_bitmask = bitmask_index.map(|index| &pending_sampling_allocations[index]);
self.ctx
.sampler
- .encode(logits, Some(sampling_seeds), sampling_bitmask, sampling_method, batch_size, encoder)
+ .encode(logits, Some(sampling_seeds), sampling_bitmask, None, None, sampling_method, batch_size, encoder)
.map_err(|err| Error::EncodeFailed(Box::new(err)))
}
@@ -482,7 +482,13 @@ impl<B: Backend> TokenDecoderRunner<B> {
let token_ids_data_type = DataType::U64;
let mut next_token_ids =
self.ctx.context.create_array_uninitialized(&token_ids_shape, token_ids_data_type).into_allocation();
- self.ctx.token_copy_sampled.encode(&sampling_output, &mut next_token_ids, encoder);
+ self.ctx.token_copy_sampled.encode(
+ &sampling_output,
+ &mut next_token_ids,
+ None::<&mut Allocation<B>>,
+ None,
+ encoder,
+ );
Some(unsafe { Array::from_allocation(next_token_ids, 0, &token_ids_shape, token_ids_data_type) })
} else {
None
@@ -588,7 +594,13 @@ impl<B: Backend> TokenDecoderRunner<B> {
.context
.create_array_uninitialized(&token_ids_shape, token_ids_data_type)
.into_allocation();
- self.ctx.token_copy_sampled.encode(&sampling_output, &mut updated_token_ids, encoder);
+ self.ctx.token_copy_sampled.encode(
+ &sampling_output,
+ &mut updated_token_ids,
+ None::<&mut Allocation<B>>,
+ None,
+ encoder,
+ );
next_token_ids = Some(unsafe {
Array::from_allocation(updated_token_ids, 0, &token_ids_shape, token_ids_data_type)
});
diff --git a/crates/backend-uzu/src/session/tts_session/decoder_support.rs b/crates/backend-uzu/src/session/tts_session/decoder_support.rs
index 2dcbfcda9..eb93ab76a 100644
--- a/crates/backend-uzu/src/session/tts_session/decoder_support.rs
+++ b/crates/backend-uzu/src/session/tts_session/decoder_support.rs
@@ -37,6 +37,8 @@ impl TextSamplingState {
top_k: None,
top_p: Some(top_p),
min_p: None,
+ repetition_penalty: None,
+ suffix_repetition_length: None,
processing_order: SamplingProcessingOrder::FiltersThenTemperature,
}
};
diff --git a/crates/backend-uzu/src/utils/version.rs b/crates/backend-uzu/src/utils/version.rs
index 2c39d1571..5b23307cd 100644
--- a/crates/backend-uzu/src/utils/version.rs
+++ b/crates/backend-uzu/src/utils/version.rs
@@ -1,2 +1,2 @@
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
-pub const TOOLCHAIN_VERSION: &str = "0.12.1";
+pub const TOOLCHAIN_VERSION: &str = "0.13.0";
diff --git a/crates/backend-uzu/tests/integration/session/chat_session/sampling_test.rs b/crates/backend-uzu/tests/integration/session/chat_session/sampling_test.rs
index f21af965b..59f537ac2 100644
--- a/crates/backend-uzu/tests/integration/session/chat_session/sampling_test.rs
+++ b/crates/backend-uzu/tests/integration/session/chat_session/sampling_test.rs
@@ -48,6 +48,8 @@ fn stochastic(temperature: f32) -> SamplingPolicy {
top_k: None,
top_p: None,
min_p: None,
+ repetition_penalty: None, // TODO
+ suffix_repetition_length: None, // TODO
processing_order: SamplingProcessingOrder::TemperatureThenFilters,
},
}
diff --git a/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs b/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs
index 61a1bd99d..f3030f00a 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs
@@ -70,6 +70,8 @@ fn do_sampling_backend<B: Backend, T: ArrayElement + Float>(
&logits_allocation,
seeds_allocation.as_ref(),
bitmask_allocation.as_ref(),
+ None, // TODO
+ None, // TODO
method,
batch_size,
&mut encoder,
@@ -147,6 +149,8 @@ fn sampling_case() -> impl Strategy<Value = SamplingCase> {
top_k,
top_p,
min_p,
+ repetition_penalty: None, // TODO
+ suffix_repetition_length: None, // TODO
processing_order,
}
},
diff --git a/crates/backend-uzu/tests/unit/kernel/token_copy_test.rs b/crates/backend-uzu/tests/unit/kernel/token_copy_test.rs
index e9713c9d1..45fba0c1a 100644
--- a/crates/backend-uzu/tests/unit/kernel/token_copy_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/token_copy_test.rs
@@ -1,5 +1,5 @@
use backend_uzu::backends::common::{
- Backend, Context, Encoder, Kernels,
+ Allocation, Backend, Context, Encoder, Kernels,
kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel},
};
@@ -11,14 +11,14 @@ use crate::{
fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {
let context = B::Context::new().expect("Failed to create Context");
- let kernel = <<B as Backend>::Kernels as Kernels>::TokenCopySampledKernel::new(&context)
+ let kernel = <<B as Backend>::Kernels as Kernels>::TokenCopySampledKernel::new(&context, false)
.expect("Failed to create TokenCopySampledKernel");
let src_allocation = alloc_allocation_with_data::<B, u32>(&context, &[src_value]);
let mut dst_allocation = alloc_allocation::<B, u64>(&context, 1);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(&src_allocation, &mut dst_allocation, &mut encoder);
+ kernel.encode(&src_allocation, &mut dst_allocation, None::<&mut Allocation<B>>, None, &mut encoder);
encoder.end_encoding().submit().wait_until_completed().unwrap();
let output = allocation_to_vec::<B, u64>(&dst_allocation);