Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Unify token ids to be u32 in both embedding lookup and sampling (#616)

uuuvn2d8cd6f70e7d
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/embedding/full_precision_embedding.rs b/crates/backend-uzu/src/backends/cpu/kernel/embedding/full_precision_embedding.rs
index 89faecac5..3245146fb 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/embedding/full_precision_embedding.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/embedding/full_precision_embedding.rs
@@ -7,7 +7,7 @@ use crate::array::ArrayElement;
#[kernel(FullPrecisionEmbeddingLookup)]
#[variants(T, f32, f16, bf16)]
pub fn full_precision_embedding_lookup<T: ArrayElement + Float>(
- #[allow(unused)] token_ids: *const u64,
+ #[allow(unused)] token_ids: *const u32,
#[allow(unused)] weights: *const T,
#[allow(unused)] output: *mut T,
#[allow(unused)] batch_size: u32,
@@ -19,11 +19,11 @@ pub fn full_precision_embedding_lookup<T: ArrayElement + Float>(
let token_id = unsafe { *token_ids.add(batch_idx as usize) };
for dim_idx in 0..model_dim {
let output_idx = (batch_idx * model_dim + dim_idx) as usize;
- if token_id >= vocab_size as u64 {
+ if token_id >= vocab_size {
unsafe { *output.add(output_idx) = T::zero() };
} else {
unsafe {
- let weights_idx = (token_id * (model_dim as u64) + dim_idx as u64) as usize;
+ let weights_idx = token_id as usize * model_dim as usize + dim_idx as usize;
*output.add(output_idx) = *weights.add(weights_idx) * T::from(input_scale).unwrap();
};
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
index 7864642fd..9d97c24e1 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
@@ -11,7 +11,7 @@ use crate::{
#[kernel(QuantizedEmbeddingLookup)]
#[variants(T, f32, f16, bf16)]
pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
- #[allow(unused)] token_ids: *const u64,
+ #[allow(unused)] token_ids: *const u32,
#[allow(unused)] weights: *const u8,
#[allow(unused)] scales: *const T,
#[allow(unused)]
@@ -60,17 +60,17 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
for dim_idx in 0..model_dim {
let out_idx = (batch_idx * model_dim + dim_idx) as usize;
- if token_id >= vocab_size as u64 {
+ if token_id >= vocab_size {
*output.add(out_idx) = T::zero();
continue;
}
let group_idx = dim_idx / group_size;
- let scale = *scales.add((token_id as u32 * num_groups + group_idx) as usize);
+ let scale = *scales.add((token_id * num_groups + group_idx) as usize);
let quantized_value: i32 = match quantization_mode {
QuantizationMode::U4 => {
- let byte_idx = (token_id as u32 * weights_stride + dim_idx / 2) as usize;
+ let byte_idx = (token_id * weights_stride + dim_idx / 2) as usize;
let packed = *weights.add(byte_idx);
if (dim_idx & 1) == 0 {
(packed & 0x0F) as i32
@@ -79,12 +79,12 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
}
},
QuantizationMode::I8 => {
- let elem_idx = (token_id as u32 * weights_stride + dim_idx) as usize;
+ let elem_idx = (token_id * weights_stride + dim_idx) as usize;
let weights_i8 = weights as *const i8;
*weights_i8.add(elem_idx) as i32
},
QuantizationMode::U8 => {
- let elem_idx = (token_id as u32 * weights_stride + dim_idx) as usize;
+ let elem_idx = (token_id * weights_stride + dim_idx) as usize;
*weights.add(elem_idx) as i32
},
};
@@ -92,7 +92,7 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
let bias = match quantization_method {
QuantizationMethod::ScaleBias => biases
.expect("ScaleBias quantized embedding requires biases")
- .add((token_id as u32 * num_groups + group_idx) as usize)
+ .add((token_id * num_groups + group_idx) as usize)
.read()
.to_f32()
.unwrap(),
@@ -100,7 +100,7 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
let zero_points = zero_points.expect("ScaleZeroPoint quantized embedding requires zero_points");
let zero_point = match quantization_mode {
QuantizationMode::U4 => {
- let byte_idx = (token_id as u32 * zero_points_stride + group_idx / 2) as usize;
+ let byte_idx = (token_id * zero_points_stride + group_idx / 2) as usize;
let packed = *zero_points.add(byte_idx);
if (group_idx & 1) == 0 {
packed & 0x0F
@@ -109,7 +109,7 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
}
},
QuantizationMode::I8 | QuantizationMode::U8 => {
- *zero_points.add((token_id as u32 * zero_points_stride + group_idx) as usize)
+ *zero_points.add((token_id * zero_points_stride + group_idx) as usize)
},
};
-scale.to_f32().unwrap() * zero_point as f32
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
index 60227949b..c140738d9 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
@@ -21,7 +21,6 @@ mod tensor_add_bias;
mod tensor_add_scale;
mod tensor_add_swap;
mod tensor_copy;
-mod token_copy;
include!(concat!(env!("OUT_DIR"), "/cpu/dsl.rs"));
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/sampling/context_ring_update.rs b/crates/backend-uzu/src/backends/cpu/kernel/sampling/context_ring_update.rs
index 219d76ed6..00b1fd360 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/sampling/context_ring_update.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/sampling/context_ring_update.rs
@@ -4,7 +4,7 @@ use crate::backends::common::gpu_types::ring::RingParams;
#[kernel(ContextRingUpdate)]
pub fn context_ring_update(
- input: *const u64,
+ input: *const u32,
context_ring: *mut u32,
suffix_repetition_length: u32,
input_length: u32,
@@ -23,7 +23,7 @@ pub fn context_ring_update(
ring.ring_offset = (ring.ring_offset + 1) % suffix_repetition_length;
slot
};
- *ring_tokens.add(slot as usize) = *input.add(input_index as usize) as u32;
+ *ring_tokens.add(slot as usize) = *input.add(input_index as usize);
}
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/sampling/repetition_penalty.rs b/crates/backend-uzu/src/backends/cpu/kernel/sampling/repetition_penalty.rs
index 1b5f3e78b..8571e6c52 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/sampling/repetition_penalty.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/sampling/repetition_penalty.rs
@@ -10,7 +10,7 @@ pub fn repetition_penalty<T: ArrayElement + Float + NumCast>(
original_logits: *const T,
logits_copy: *mut T,
context_ring: *const u32,
- token_ids: *const u64,
+ token_ids: *const u32,
repetition_penalty: f32,
suffix_repetition_length: u32,
vocab_size: u32,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/token_copy/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/token_copy/mod.rs
deleted file mode 100644
index 90375d281..000000000
--- a/crates/backend-uzu/src/backends/cpu/kernel/token_copy/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub mod token_copy;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/token_copy/token_copy.rs b/crates/backend-uzu/src/backends/cpu/kernel/token_copy/token_copy.rs
deleted file mode 100644
index 9fbbb175c..000000000
--- a/crates/backend-uzu/src/backends/cpu/kernel/token_copy/token_copy.rs
+++ /dev/null
@@ -1,12 +0,0 @@
-use proc_macros::kernel;
-
-#[kernel(TokenCopySampled)]
-pub fn token_copy_sampled(
- src: *const u32,
- dst: *mut u64,
-) {
- unsafe {
- let token = *src.add(0);
- *dst.add(0) = token as u64;
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal b/crates/backend-uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal
index c45f05e24..4baaf37e3 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal
@@ -6,7 +6,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
PUBLIC KERNEL(FullPrecisionEmbeddingLookup) (
- const device uint64_t* token_ids, // [batch_size]
+ const device uint32_t* token_ids, // [batch_size]
const device T* weights, // [vocab_size, model_dim]
device T* output, // [batch_size, model_dim]
constant uint32_t& batch_size,
@@ -16,7 +16,7 @@ PUBLIC KERNEL(FullPrecisionEmbeddingLookup) (
uint dim_idx AXIS(model_dim, BLOCK_SIZE),
uint batch_idx AXIS(batch_size, 1)
) {
- const uint64_t token_id = token_ids[batch_idx];
+ const uint32_t token_id = token_ids[batch_idx];
const uint output_idx = batch_idx * model_dim + dim_idx;
if (token_id >= vocab_size) {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index 5a885f0ce..c3998a8eb 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -10,7 +10,7 @@ using namespace uzu::quantization;
template <typename T>
VARIANTS(T, float, half, bfloat)
PUBLIC KERNEL(QuantizedEmbeddingLookup) (
- const device uint64_t* token_ids, // [batch_size]
+ const device uint32_t* token_ids, // [batch_size]
const device uint8_t* weights, // [vocab_size, model_dim/packing_divisor] packed
const device T* scales, // [vocab_size, num_groups]
const device uint8_t* zero_points OPTIONAL(quantization_method == QuantizationMethod::ScaleZeroPoint),
@@ -29,7 +29,7 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
const uint batch_idx AXIS(batch_size, 1)
) {
const uint thread_position_in_grid = batch_idx * model_dim + dim_idx;
- const uint64_t token_id = token_ids[batch_idx];
+ const uint32_t token_id = token_ids[batch_idx];
if (token_id >= vocab_size) {
output[thread_position_in_grid] = T(0);
return;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/context_ring_update.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/context_ring_update.metal
index 3b67f8729..471fa3232 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/context_ring_update.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/context_ring_update.metal
@@ -7,7 +7,7 @@ using namespace uzu::ring;
#define THREADGROUP_SIZE 256
PUBLIC KERNEL(ContextRingUpdate)(
- const device uint64_t* input,
+ const device uint32_t* input,
device uint32_t* context_ring,
const constant uint32_t& suffix_repetition_length,
const constant uint32_t& input_length,
@@ -20,7 +20,7 @@ PUBLIC KERNEL(ContextRingUpdate)(
for (uint32_t input_index = write_start + thread_index; input_index < input_length; input_index += THREADGROUP_SIZE) {
const uint32_t slot = (ring.ring_offset + ring.ring_length + input_index) % suffix_repetition_length;
- ring_tokens[slot] = static_cast<uint32_t>(input[input_index]);
+ ring_tokens[slot] = input[input_index];
}
threadgroup_barrier(mem_flags::mem_none);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/repetition_penalty.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/repetition_penalty.metal
index 95a00ec67..0173f74a1 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/repetition_penalty.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/repetition_penalty.metal
@@ -11,7 +11,7 @@ PUBLIC KERNEL(RepetitionPenalty)(
const device T* original_logits,
device T* logits_copy,
const device uint32_t* context_ring,
- const device uint64_t* token_ids,
+ const device uint32_t* token_ids,
const constant float& repetition_penalty,
const constant uint32_t& suffix_repetition_length,
const constant uint32_t& vocab_size,
@@ -37,7 +37,7 @@ PUBLIC KERNEL(RepetitionPenalty)(
const uint32_t slot = (ring.ring_offset + source_index) % suffix_repetition_length;
token_id = ring_tokens[slot];
} else {
- token_id = static_cast<uint32_t>(token_ids[source_index - ring.ring_length]);
+ token_id = token_ids[source_index - ring.ring_length];
}
const uint32_t logit_offset = sample_index * vocab_size + token_id;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal
deleted file mode 100644
index 61db88105..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal
+++ /dev/null
@@ -1,8 +0,0 @@
-#include <metal_stdlib>
-#include "../common/dsl.h"
-
-/// Copies a single sampled token (u32) to the token_ids buffer (u64).
-/// Used in decoding loop to pass sampled token to next forward pass.
-PUBLIC KERNEL(TokenCopySampled)(device const uint32_t* src, device uint64_t* dst) {
- dst[0] = static_cast<uint64_t>(src[0]);
-}
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index ae7f9790d..9f62c86f4 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -112,8 +112,6 @@ struct LogitSoftCap<B: Backend> {
kernel: <B::Kernels as Kernels>::LogitSoftCapKernel,
}
-// TODO: embedding lookup dtype (u64) should match sampling (u32)
-
impl<B: Backend> Embedding<B> {
pub fn new(
context: &B::Context,
diff --git a/crates/backend-uzu/src/engine/classifier_model.rs b/crates/backend-uzu/src/engine/classifier_model.rs
index 4073cd254..d096937a9 100644
--- a/crates/backend-uzu/src/engine/classifier_model.rs
+++ b/crates/backend-uzu/src/engine/classifier_model.rs
@@ -108,9 +108,10 @@ impl<B: Backend> ClassifierModel<B> {
let mut encoder = Encoder::<B>::new(&self.context).map_err(ClassifierModelClassifyError::Backend)?;
- let mut token_ids =
- encoder.allocate_constant(size_of_val(input)).map_err(ClassifierModelClassifyError::Backend)?;
- token_ids.copyin(input);
+ let mut token_ids = encoder
+ .allocate_constant(input.len() * DataType::U32.size_in_bytes())
+ .map_err(ClassifierModelClassifyError::Backend)?;
+ token_ids.copyin(&input.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>());
let logits = self.classifier.encode(&token_ids, input.len(), &mut encoder)?;
diff --git a/crates/backend-uzu/src/engine/language_model/mod.rs b/crates/backend-uzu/src/engine/language_model/mod.rs
index 4f9f0835c..85225a363 100644
--- a/crates/backend-uzu/src/engine/language_model/mod.rs
+++ b/crates/backend-uzu/src/engine/language_model/mod.rs
@@ -3,10 +3,7 @@ use std::{fs::File, io, io::BufReader, path::Path, sync::Arc};
use thiserror::Error;
use crate::{
- backends::common::{
- Backend, Context, Kernels,
- kernel::{ContextRingUpdateKernel, TokenCopySampledKernel},
- },
+ backends::common::{Backend, Context, Kernels, kernel::ContextRingUpdateKernel},
config::model::{generation::GenerationConfig, language_model::LanguageModelConfig},
data_type::DataType,
encodable_block::{
@@ -25,7 +22,6 @@ pub struct LanguageModel<B: Backend> {
context: Arc<B::Context>,
decoder: Decoder<B>,
sampling: Sampling<B>,
- token_copy: <B::Kernels as Kernels>::TokenCopySampledKernel,
context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,
generation_config: GenerationConfig,
vocab_size: usize,
@@ -93,8 +89,6 @@ impl<B: Backend> Engine<B> {
let sampling = Sampling::new(data_type, config.decoder_config.vocab_size);
- let token_copy = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context)
- .map_err(EngineLoadLanguageModelError::Backend)?;
let context_ring_update = <B::Kernels as Kernels>::ContextRingUpdateKernel::new(&context)
.map_err(EngineLoadLanguageModelError::Backend)?;
@@ -108,7 +102,6 @@ impl<B: Backend> Engine<B> {
context,
decoder,
sampling,
- token_copy,
context_ring_update,
generation_config,
vocab_size,
diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
index a2a9da7ab..25818906b 100644
--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs
+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
@@ -9,8 +9,7 @@ use shoji::traits::backend::chat_token::TokenStreamMetrics;
use crate::{
backends::common::{
Allocation, AllocationPool, AllocationType, Backend, Context, Encoder, Pending,
- gpu_types::trie::TrieNode as GpuTrieNode,
- kernel::{ContextRingUpdateKernel, TokenCopySampledKernel},
+ gpu_types::trie::TrieNode as GpuTrieNode, kernel::ContextRingUpdateKernel,
},
data_type::DataType,
encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod},
@@ -185,9 +184,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let input_flat_trie = input_trie.linearize();
let mut token_ids = encoder
- .allocate_constant(input_chunk.len() * DataType::U64.size_in_bytes())
+ .allocate_constant(input_chunk.len() * DataType::U32.size_in_bytes())
.map_err(LanguageModelStreamError::Backend)?;
- token_ids.copyin(input_chunk);
+ token_ids.copyin(&input_chunk.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>());
let input_flat_trie_nodes = input_flat_trie.token_subtrie_ranges().collect::<Box<[GpuTrieNode]>>();
let batch_dim = BatchTopology::new(&input_flat_trie_nodes, true);
@@ -369,9 +368,14 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length()
{
let mut accepted_input_token_ids_const = encoder
- .allocate_constant(full.len() * DataType::U64.size_in_bytes())
+ .allocate_constant(full.len() * DataType::U32.size_in_bytes())
.map_err(LanguageModelStreamError::Backend)?;
- accepted_input_token_ids_const.copyin(&accepted_input_token_ids);
+ accepted_input_token_ids_const.copyin(
+ &accepted_input_token_ids
+ .iter()
+ .map(|token_id| *token_id as u32)
+ .collect::<Box<[u32]>>(),
+ );
self.model.context_ring_update.encode(
&accepted_input_token_ids_const,
self.context_ring.as_mut().unwrap(),
@@ -429,14 +433,14 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let token_ids = if let Some(chain_copy) = chain_copy {
let mut token_ids =
- encoder.allocate_scratch(DataType::U64.size_in_bytes()).map_err(LanguageModelStreamError::Backend)?;
- self.model.token_copy.encode(chain_copy, &mut token_ids, &mut encoder);
+ encoder.allocate_scratch(DataType::U32.size_in_bytes()).map_err(LanguageModelStreamError::Backend)?;
+ encoder.encode_copy(chain_copy, .., &mut token_ids, ..);
token_ids
} else {
let mut token_ids = encoder
- .allocate_constant(input_flat_trie.len() * DataType::U64.size_in_bytes())
+ .allocate_constant(input_flat_trie.len() * DataType::U32.size_in_bytes())
.map_err(LanguageModelStreamError::Backend)?;
- token_ids.copyin(&input_flat_trie.token_ids().collect::<Box<[u64]>>());
+ token_ids.copyin(&input_flat_trie.token_ids().map(|token_id| token_id as u32).collect::<Box<[u32]>>());
token_ids
};
diff --git a/crates/backend-uzu/unit/backends/common/kernel/embedding/full_precision_embedding_test.rs b/crates/backend-uzu/unit/backends/common/kernel/embedding/full_precision_embedding_test.rs
index dfa2c1a1c..4070bef07 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/embedding/full_precision_embedding_test.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/embedding/full_precision_embedding_test.rs
@@ -12,7 +12,7 @@ use crate::{
};
struct Input<T: ArrayElement + Float> {
- token_ids: Box<[u64]>,
+ token_ids: Box<[u32]>,
weights: Box<[T]>,
batch_size: usize,
vocab_size: usize,
@@ -26,7 +26,7 @@ fn get_test_data<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
let batch_size = 7;
let input_scale = 2.0_f32;
- let token_ids: Box<[u64]> = Box::new([3, 7, 1, 10, 0, 5, 8]);
+ let token_ids: Box<[u32]> = Box::new([3, 7, 1, 10, 0, 5, 8]);
let weights: Vec<T> =
(0..vocab_size * model_dim).map(|i| T::from((i as f32).sin() * 30.0f32).unwrap()).collect::<Vec<_>>();
let mut expected: Vec<T> = Vec::with_capacity(batch_size * model_dim);
@@ -56,7 +56,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
<<B as Backend>::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(&context, T::data_type())
.expect("Failed to create FullPrecisionEmbeddingLookupKernel");
- let token_ids_allocation = alloc_allocation_with_data::<B, u64>(&context, &input.token_ids);
+ let token_ids_allocation = alloc_allocation_with_data::<B, u32>(&context, &input.token_ids);
let weights_allocation = alloc_allocation_with_data::<B, T>(&context, &input.weights);
let mut output = alloc_allocation::<B, T>(&context, input.batch_size * input.model_dim);
diff --git a/crates/backend-uzu/unit/backends/common/kernel/embedding/quant_embedding_test.rs b/crates/backend-uzu/unit/backends/common/kernel/embedding/quant_embedding_test.rs
index e59542304..035415a10 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/embedding/quant_embedding_test.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/embedding/quant_embedding_test.rs
@@ -23,7 +23,7 @@ use crate::{
};
struct Input<T: ArrayElement + Float> {
- token_ids: Box<[u64]>,
+ token_ids: Box<[u32]>,
weights: Box<[u8]>,
scales: Box<[T]>,
zero_points: Option<Box<[u8]>>,
@@ -44,7 +44,7 @@ fn get_test_data<T: ArrayElement + Float>(quant_mode: QuantizationMode) -> (Inpu
let group_size = 32u32;
let input_scale = 1.5f32;
- let token_ids: Box<[u64]> = Box::new([2, 5, 0]);
+ let token_ids: Box<[u32]> = Box::new([2, 5, 0]);
let packing_divisor = quant_mode.packing_divisor() as u32;
let weights_stride = model_dim / packing_divisor;
@@ -87,7 +87,7 @@ fn get_test_data_zero_point_group16<T: ArrayElement + Float>() -> (Input<T>, Vec
let input_scale = 1.5f32;
let quant_mode = QuantizationMode::U4;
- let token_ids: Box<[u64]> = Box::new([2, 5, 0]);
+ let token_ids: Box<[u32]> = Box::new([2, 5, 0]);
let packing_divisor = quant_mode.packing_divisor() as u32;
let weights_stride = model_dim / packing_divisor;
@@ -131,7 +131,7 @@ fn get_test_data_symmetric_u8<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
let input_scale = 1.5f32;
let quant_mode = QuantizationMode::U8;
- let token_ids: Box<[u64]> = Box::new([2, 5, 0]);
+ let token_ids: Box<[u32]> = Box::new([2, 5, 0]);
let weights: Vec<u8> =
(0..vocab_size as usize * model_dim as usize).map(|i| ((i * 37 + 11) & 0xFF) as u8).collect();
@@ -168,7 +168,7 @@ fn get_test_data_oob<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
let input_scale = 1.0f32;
// token_id 1 is valid, token_id 10 is out of bounds
- let token_ids: Box<[u64]> = Box::new([1, 10]);
+ let token_ids: Box<[u32]> = Box::new([1, 10]);
let weights: Vec<u8> = (0..vocab_size as usize * model_dim as usize).map(|i| (i % 200) as u8).collect();
@@ -210,7 +210,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
)
.expect("Failed to create QuantizedEmbeddingLookupKernel");
- let token_ids_allocation = alloc_allocation_with_data::<B, u64>(&context, &input.token_ids);
+ let token_ids_allocation = alloc_allocation_with_data::<B, u32>(&context, &input.token_ids);
let weights_allocation = alloc_allocation_with_data::<B, u8>(&context, &input.weights);
let scales_allocation = alloc_allocation_with_data::<B, T>(&context, &input.scales);
let zero_points_allocation =
diff --git a/crates/backend-uzu/unit/backends/common/kernel/mod.rs b/crates/backend-uzu/unit/backends/common/kernel/mod.rs
index 442f83e34..5ac3ed84b 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/mod.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/mod.rs
@@ -16,4 +16,3 @@ mod tensor_add_bias_test;
mod tensor_add_scale_test;
mod tensor_add_swap_test;
mod tensor_copy_test;
-mod token_copy_test;
diff --git a/crates/backend-uzu/unit/backends/common/kernel/token_copy_test.rs b/crates/backend-uzu/unit/backends/common/kernel/token_copy_test.rs
deleted file mode 100644
index 14a4c05d1..000000000
--- a/crates/backend-uzu/unit/backends/common/kernel/token_copy_test.rs
+++ /dev/null
@@ -1,33 +0,0 @@
-use proc_macros::uzu_test;
-use test_runner::for_each_backend;
-
-use crate::{
- backends::common::{Backend, Context, Encoder, Kernels, kernel::TokenCopySampledKernel},
- tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
-};
-
-fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {
- let context = B::Context::new().expect("Failed to create Context");
-
- let kernel = <<B as Backend>::Kernels as Kernels>::TokenCopySampledKernel::new(&context)
- .expect("Failed to create TokenCopySampledKernel");
-
- let src_allocation = alloc_allocation_with_data::<B, u32>(&context, &[src_value]);
- let mut dst_allocation = alloc_allocation::<B, u64>(&context, 1);
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(&src_allocation, &mut dst_allocation, &mut encoder);
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- let output = allocation_to_vec::<B, u64>(&dst_allocation);
- assert_eq!(output[0], src_value as u64, "TokenCopySampled failed for backend {}", std::any::type_name::<B>());
-}
-
-#[uzu_test]
-fn test_token_copy_sampled() {
- for_each_backend!(|B| {
- test_token_copy_sampled_impl::<B>(0);
- test_token_copy_sampled_impl::<B>(42);
- test_token_copy_sampled_impl::<B>(u32::MAX);
- });
-}