A high-performance inference engine for AI models
Unify token ids to be u32 in both embedding lookup and sampling (#616)
uuuvn2d8cd6f70e7d
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/embedding/full_precision_embedding.rs b/crates/backend-uzu/src/backends/cpu/kernel/embedding/full_precision_embedding.rsindex 89faecac5..3245146fb 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/embedding/full_precision_embedding.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/embedding/full_precision_embedding.rs@@ -7,7 +7,7 @@ use crate::array::ArrayElement;#[kernel(FullPrecisionEmbeddingLookup)]#[variants(T, f32, f16, bf16)]pub fn full_precision_embedding_lookup<T: ArrayElement + Float>(- #[allow(unused)] token_ids: *const u64,+ #[allow(unused)] token_ids: *const u32,#[allow(unused)] weights: *const T,#[allow(unused)] output: *mut T,#[allow(unused)] batch_size: u32,@@ -19,11 +19,11 @@ pub fn full_precision_embedding_lookup<T: ArrayElement + Float>(let token_id = unsafe { *token_ids.add(batch_idx as usize) };for dim_idx in 0..model_dim {let output_idx = (batch_idx * model_dim + dim_idx) as usize;- if token_id >= vocab_size as u64 {+ if token_id >= vocab_size {unsafe { *output.add(output_idx) = T::zero() };} else {unsafe {- let weights_idx = (token_id * (model_dim as u64) + dim_idx as u64) as usize;+ let weights_idx = token_id as usize * model_dim as usize + dim_idx as usize;*output.add(output_idx) = *weights.add(weights_idx) * T::from(input_scale).unwrap();};}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rsindex 7864642fd..9d97c24e1 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs@@ -11,7 +11,7 @@ use crate::{#[kernel(QuantizedEmbeddingLookup)]#[variants(T, f32, f16, bf16)]pub fn quantized_embedding_lookup<T: ArrayElement + Float>(- #[allow(unused)] token_ids: *const u64,+ #[allow(unused)] token_ids: *const u32,#[allow(unused)] weights: *const u8,#[allow(unused)] scales: *const T,#[allow(unused)]@@ -60,17 +60,17 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(for dim_idx in 0..model_dim {let out_idx = (batch_idx * model_dim + dim_idx) as usize;- if token_id >= vocab_size as u64 {+ if token_id >= vocab_size {*output.add(out_idx) = T::zero();continue;}let group_idx = dim_idx / group_size;- let scale = *scales.add((token_id as u32 * num_groups + group_idx) as usize);+ let scale = *scales.add((token_id * num_groups + group_idx) as usize);let quantized_value: i32 = match quantization_mode {QuantizationMode::U4 => {- let byte_idx = (token_id as u32 * weights_stride + dim_idx / 2) as usize;+ let byte_idx = (token_id * weights_stride + dim_idx / 2) as usize;let packed = *weights.add(byte_idx);if (dim_idx & 1) == 0 {(packed & 0x0F) as i32@@ -79,12 +79,12 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(}},QuantizationMode::I8 => {- let elem_idx = (token_id as u32 * weights_stride + dim_idx) as usize;+ let elem_idx = (token_id * weights_stride + dim_idx) as usize;let weights_i8 = weights as *const i8;*weights_i8.add(elem_idx) as i32},QuantizationMode::U8 => {- let elem_idx = (token_id as u32 * weights_stride + dim_idx) as usize;+ let elem_idx = (token_id * weights_stride + dim_idx) as usize;*weights.add(elem_idx) as i32},};@@ -92,7 +92,7 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(let bias = match quantization_method {QuantizationMethod::ScaleBias => biases.expect("ScaleBias quantized embedding requires biases")- .add((token_id as u32 * num_groups + group_idx) as usize)+ .add((token_id * num_groups + group_idx) as usize).read().to_f32().unwrap(),@@ -100,7 +100,7 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(let zero_points = zero_points.expect("ScaleZeroPoint quantized embedding requires zero_points");let zero_point = match quantization_mode {QuantizationMode::U4 => {- let byte_idx = (token_id as u32 * zero_points_stride + group_idx / 2) as usize;+ let byte_idx = (token_id * zero_points_stride + group_idx / 2) as usize;let packed = *zero_points.add(byte_idx);if (group_idx & 1) == 0 {packed & 0x0F@@ -109,7 +109,7 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(}},QuantizationMode::I8 | QuantizationMode::U8 => {- *zero_points.add((token_id as u32 * zero_points_stride + group_idx) as usize)+ *zero_points.add((token_id * zero_points_stride + group_idx) as usize)},};-scale.to_f32().unwrap() * zero_point as f32diff --git a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/mod.rsindex 60227949b..c140738d9 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs@@ -21,7 +21,6 @@ mod tensor_add_bias;mod tensor_add_scale;mod tensor_add_swap;mod tensor_copy;-mod token_copy;include!(concat!(env!("OUT_DIR"), "/cpu/dsl.rs"));diff --git a/crates/backend-uzu/src/backends/cpu/kernel/sampling/context_ring_update.rs b/crates/backend-uzu/src/backends/cpu/kernel/sampling/context_ring_update.rsindex 219d76ed6..00b1fd360 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/sampling/context_ring_update.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/sampling/context_ring_update.rs@@ -4,7 +4,7 @@ use crate::backends::common::gpu_types::ring::RingParams;#[kernel(ContextRingUpdate)]pub fn context_ring_update(- input: *const u64,+ input: *const u32,context_ring: *mut u32,suffix_repetition_length: u32,input_length: u32,@@ -23,7 +23,7 @@ pub fn context_ring_update(ring.ring_offset = (ring.ring_offset + 1) % suffix_repetition_length;slot};- *ring_tokens.add(slot as usize) = *input.add(input_index as usize) as u32;+ *ring_tokens.add(slot as usize) = *input.add(input_index as usize);}}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/sampling/repetition_penalty.rs b/crates/backend-uzu/src/backends/cpu/kernel/sampling/repetition_penalty.rsindex 1b5f3e78b..8571e6c52 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/sampling/repetition_penalty.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/sampling/repetition_penalty.rs@@ -10,7 +10,7 @@ pub fn repetition_penalty<T: ArrayElement + Float + NumCast>(original_logits: *const T,logits_copy: *mut T,context_ring: *const u32,- token_ids: *const u64,+ token_ids: *const u32,repetition_penalty: f32,suffix_repetition_length: u32,vocab_size: u32,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/token_copy/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/token_copy/mod.rsdeleted file mode 100644index 90375d281..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/token_copy/mod.rs+++ /dev/null@@ -1 +0,0 @@-pub mod token_copy;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/token_copy/token_copy.rs b/crates/backend-uzu/src/backends/cpu/kernel/token_copy/token_copy.rsdeleted file mode 100644index 9fbbb175c..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/token_copy/token_copy.rs+++ /dev/null@@ -1,12 +0,0 @@-use proc_macros::kernel;--#[kernel(TokenCopySampled)]-pub fn token_copy_sampled(- src: *const u32,- dst: *mut u64,-) {- unsafe {- let token = *src.add(0);- *dst.add(0) = token as u64;- }-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal b/crates/backend-uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metalindex c45f05e24..4baaf37e3 100644--- a/crates/backend-uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal@@ -6,7 +6,7 @@template <typename T>VARIANTS(T, float, half, bfloat)PUBLIC KERNEL(FullPrecisionEmbeddingLookup) (- const device uint64_t* token_ids, // [batch_size]+ const device uint32_t* token_ids, // [batch_size]const device T* weights, // [vocab_size, model_dim]device T* output, // [batch_size, model_dim]constant uint32_t& batch_size,@@ -16,7 +16,7 @@ PUBLIC KERNEL(FullPrecisionEmbeddingLookup) (uint dim_idx AXIS(model_dim, BLOCK_SIZE),uint batch_idx AXIS(batch_size, 1)) {- const uint64_t token_id = token_ids[batch_idx];+ const uint32_t token_id = token_ids[batch_idx];const uint output_idx = batch_idx * model_dim + dim_idx;if (token_id >= vocab_size) {diff --git a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex 5a885f0ce..c3998a8eb 100644--- a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -10,7 +10,7 @@ using namespace uzu::quantization;template <typename T>VARIANTS(T, float, half, bfloat)PUBLIC KERNEL(QuantizedEmbeddingLookup) (- const device uint64_t* token_ids, // [batch_size]+ const device uint32_t* token_ids, // [batch_size]const device uint8_t* weights, // [vocab_size, model_dim/packing_divisor] packedconst device T* scales, // [vocab_size, num_groups]const device uint8_t* zero_points OPTIONAL(quantization_method == QuantizationMethod::ScaleZeroPoint),@@ -29,7 +29,7 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (const uint batch_idx AXIS(batch_size, 1)) {const uint thread_position_in_grid = batch_idx * model_dim + dim_idx;- const uint64_t token_id = token_ids[batch_idx];+ const uint32_t token_id = token_ids[batch_idx];if (token_id >= vocab_size) {output[thread_position_in_grid] = T(0);return;diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/context_ring_update.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/context_ring_update.metalindex 3b67f8729..471fa3232 100644--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/context_ring_update.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/context_ring_update.metal@@ -7,7 +7,7 @@ using namespace uzu::ring;#define THREADGROUP_SIZE 256PUBLIC KERNEL(ContextRingUpdate)(- const device uint64_t* input,+ const device uint32_t* input,device uint32_t* context_ring,const constant uint32_t& suffix_repetition_length,const constant uint32_t& input_length,@@ -20,7 +20,7 @@ PUBLIC KERNEL(ContextRingUpdate)(for (uint32_t input_index = write_start + thread_index; input_index < input_length; input_index += THREADGROUP_SIZE) {const uint32_t slot = (ring.ring_offset + ring.ring_length + input_index) % suffix_repetition_length;- ring_tokens[slot] = static_cast<uint32_t>(input[input_index]);+ ring_tokens[slot] = input[input_index];}threadgroup_barrier(mem_flags::mem_none);diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/repetition_penalty.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/repetition_penalty.metalindex 95a00ec67..0173f74a1 100644--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/repetition_penalty.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/repetition_penalty.metal@@ -11,7 +11,7 @@ PUBLIC KERNEL(RepetitionPenalty)(const device T* original_logits,device T* logits_copy,const device uint32_t* context_ring,- const device uint64_t* token_ids,+ const device uint32_t* token_ids,const constant float& repetition_penalty,const constant uint32_t& suffix_repetition_length,const constant uint32_t& vocab_size,@@ -37,7 +37,7 @@ PUBLIC KERNEL(RepetitionPenalty)(const uint32_t slot = (ring.ring_offset + source_index) % suffix_repetition_length;token_id = ring_tokens[slot];} else {- token_id = static_cast<uint32_t>(token_ids[source_index - ring.ring_length]);+ token_id = token_ids[source_index - ring.ring_length];}const uint32_t logit_offset = sample_index * vocab_size + token_id;diff --git a/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metaldeleted file mode 100644index 61db88105..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal+++ /dev/null@@ -1,8 +0,0 @@-#include <metal_stdlib>-#include "../common/dsl.h"--/// Copies a single sampled token (u32) to the token_ids buffer (u64).-/// Used in decoding loop to pass sampled token to next forward pass.-PUBLIC KERNEL(TokenCopySampled)(device const uint32_t* src, device uint64_t* dst) {- dst[0] = static_cast<uint64_t>(src[0]);-}diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex ae7f9790d..9f62c86f4 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -112,8 +112,6 @@ struct LogitSoftCap<B: Backend> {kernel: <B::Kernels as Kernels>::LogitSoftCapKernel,}-// TODO: embedding lookup dtype (u64) should match sampling (u32)-impl<B: Backend> Embedding<B> {pub fn new(context: &B::Context,diff --git a/crates/backend-uzu/src/engine/classifier_model.rs b/crates/backend-uzu/src/engine/classifier_model.rsindex 4073cd254..d096937a9 100644--- a/crates/backend-uzu/src/engine/classifier_model.rs+++ b/crates/backend-uzu/src/engine/classifier_model.rs@@ -108,9 +108,10 @@ impl<B: Backend> ClassifierModel<B> {let mut encoder = Encoder::<B>::new(&self.context).map_err(ClassifierModelClassifyError::Backend)?;- let mut token_ids =- encoder.allocate_constant(size_of_val(input)).map_err(ClassifierModelClassifyError::Backend)?;- token_ids.copyin(input);+ let mut token_ids = encoder+ .allocate_constant(input.len() * DataType::U32.size_in_bytes())+ .map_err(ClassifierModelClassifyError::Backend)?;+ token_ids.copyin(&input.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>());let logits = self.classifier.encode(&token_ids, input.len(), &mut encoder)?;diff --git a/crates/backend-uzu/src/engine/language_model/mod.rs b/crates/backend-uzu/src/engine/language_model/mod.rsindex 4f9f0835c..85225a363 100644--- a/crates/backend-uzu/src/engine/language_model/mod.rs+++ b/crates/backend-uzu/src/engine/language_model/mod.rs@@ -3,10 +3,7 @@ use std::{fs::File, io, io::BufReader, path::Path, sync::Arc};use thiserror::Error;use crate::{- backends::common::{- Backend, Context, Kernels,- kernel::{ContextRingUpdateKernel, TokenCopySampledKernel},- },+ backends::common::{Backend, Context, Kernels, kernel::ContextRingUpdateKernel},config::model::{generation::GenerationConfig, language_model::LanguageModelConfig},data_type::DataType,encodable_block::{@@ -25,7 +22,6 @@ pub struct LanguageModel<B: Backend> {context: Arc<B::Context>,decoder: Decoder<B>,sampling: Sampling<B>,- token_copy: <B::Kernels as Kernels>::TokenCopySampledKernel,context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,generation_config: GenerationConfig,vocab_size: usize,@@ -93,8 +89,6 @@ impl<B: Backend> Engine<B> {let sampling = Sampling::new(data_type, config.decoder_config.vocab_size);- let token_copy = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context)- .map_err(EngineLoadLanguageModelError::Backend)?;let context_ring_update = <B::Kernels as Kernels>::ContextRingUpdateKernel::new(&context).map_err(EngineLoadLanguageModelError::Backend)?;@@ -108,7 +102,6 @@ impl<B: Backend> Engine<B> {context,decoder,sampling,- token_copy,context_ring_update,generation_config,vocab_size,diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rsindex a2a9da7ab..25818906b 100644--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs@@ -9,8 +9,7 @@ use shoji::traits::backend::chat_token::TokenStreamMetrics;use crate::{backends::common::{Allocation, AllocationPool, AllocationType, Backend, Context, Encoder, Pending,- gpu_types::trie::TrieNode as GpuTrieNode,- kernel::{ContextRingUpdateKernel, TokenCopySampledKernel},+ gpu_types::trie::TrieNode as GpuTrieNode, kernel::ContextRingUpdateKernel,},data_type::DataType,encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod},@@ -185,9 +184,9 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let input_flat_trie = input_trie.linearize();let mut token_ids = encoder- .allocate_constant(input_chunk.len() * DataType::U64.size_in_bytes())+ .allocate_constant(input_chunk.len() * DataType::U32.size_in_bytes()).map_err(LanguageModelStreamError::Backend)?;- token_ids.copyin(input_chunk);+ token_ids.copyin(&input_chunk.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>());let input_flat_trie_nodes = input_flat_trie.token_subtrie_ranges().collect::<Box<[GpuTrieNode]>>();let batch_dim = BatchTopology::new(&input_flat_trie_nodes, true);@@ -369,9 +368,14 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {if let Some(suffix_repetition_length) = self.options.sampling_method.suffix_repetition_length(){let mut accepted_input_token_ids_const = encoder- .allocate_constant(full.len() * DataType::U64.size_in_bytes())+ .allocate_constant(full.len() * DataType::U32.size_in_bytes()).map_err(LanguageModelStreamError::Backend)?;- accepted_input_token_ids_const.copyin(&accepted_input_token_ids);+ accepted_input_token_ids_const.copyin(+ &accepted_input_token_ids+ .iter()+ .map(|token_id| *token_id as u32)+ .collect::<Box<[u32]>>(),+ );self.model.context_ring_update.encode(&accepted_input_token_ids_const,self.context_ring.as_mut().unwrap(),@@ -429,14 +433,14 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {let token_ids = if let Some(chain_copy) = chain_copy {let mut token_ids =- encoder.allocate_scratch(DataType::U64.size_in_bytes()).map_err(LanguageModelStreamError::Backend)?;- self.model.token_copy.encode(chain_copy, &mut token_ids, &mut encoder);+ encoder.allocate_scratch(DataType::U32.size_in_bytes()).map_err(LanguageModelStreamError::Backend)?;+ encoder.encode_copy(chain_copy, .., &mut token_ids, ..);token_ids} else {let mut token_ids = encoder- .allocate_constant(input_flat_trie.len() * DataType::U64.size_in_bytes())+ .allocate_constant(input_flat_trie.len() * DataType::U32.size_in_bytes()).map_err(LanguageModelStreamError::Backend)?;- token_ids.copyin(&input_flat_trie.token_ids().collect::<Box<[u64]>>());+ token_ids.copyin(&input_flat_trie.token_ids().map(|token_id| token_id as u32).collect::<Box<[u32]>>());token_ids};diff --git a/crates/backend-uzu/unit/backends/common/kernel/embedding/full_precision_embedding_test.rs b/crates/backend-uzu/unit/backends/common/kernel/embedding/full_precision_embedding_test.rsindex dfa2c1a1c..4070bef07 100644--- a/crates/backend-uzu/unit/backends/common/kernel/embedding/full_precision_embedding_test.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/embedding/full_precision_embedding_test.rs@@ -12,7 +12,7 @@ use crate::{};struct Input<T: ArrayElement + Float> {- token_ids: Box<[u64]>,+ token_ids: Box<[u32]>,weights: Box<[T]>,batch_size: usize,vocab_size: usize,@@ -26,7 +26,7 @@ fn get_test_data<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {let batch_size = 7;let input_scale = 2.0_f32;- let token_ids: Box<[u64]> = Box::new([3, 7, 1, 10, 0, 5, 8]);+ let token_ids: Box<[u32]> = Box::new([3, 7, 1, 10, 0, 5, 8]);let weights: Vec<T> =(0..vocab_size * model_dim).map(|i| T::from((i as f32).sin() * 30.0f32).unwrap()).collect::<Vec<_>>();let mut expected: Vec<T> = Vec::with_capacity(batch_size * model_dim);@@ -56,7 +56,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {<<B as Backend>::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(&context, T::data_type()).expect("Failed to create FullPrecisionEmbeddingLookupKernel");- let token_ids_allocation = alloc_allocation_with_data::<B, u64>(&context, &input.token_ids);+ let token_ids_allocation = alloc_allocation_with_data::<B, u32>(&context, &input.token_ids);let weights_allocation = alloc_allocation_with_data::<B, T>(&context, &input.weights);let mut output = alloc_allocation::<B, T>(&context, input.batch_size * input.model_dim);diff --git a/crates/backend-uzu/unit/backends/common/kernel/embedding/quant_embedding_test.rs b/crates/backend-uzu/unit/backends/common/kernel/embedding/quant_embedding_test.rsindex e59542304..035415a10 100644--- a/crates/backend-uzu/unit/backends/common/kernel/embedding/quant_embedding_test.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/embedding/quant_embedding_test.rs@@ -23,7 +23,7 @@ use crate::{};struct Input<T: ArrayElement + Float> {- token_ids: Box<[u64]>,+ token_ids: Box<[u32]>,weights: Box<[u8]>,scales: Box<[T]>,zero_points: Option<Box<[u8]>>,@@ -44,7 +44,7 @@ fn get_test_data<T: ArrayElement + Float>(quant_mode: QuantizationMode) -> (Inpulet group_size = 32u32;let input_scale = 1.5f32;- let token_ids: Box<[u64]> = Box::new([2, 5, 0]);+ let token_ids: Box<[u32]> = Box::new([2, 5, 0]);let packing_divisor = quant_mode.packing_divisor() as u32;let weights_stride = model_dim / packing_divisor;@@ -87,7 +87,7 @@ fn get_test_data_zero_point_group16<T: ArrayElement + Float>() -> (Input<T>, Veclet input_scale = 1.5f32;let quant_mode = QuantizationMode::U4;- let token_ids: Box<[u64]> = Box::new([2, 5, 0]);+ let token_ids: Box<[u32]> = Box::new([2, 5, 0]);let packing_divisor = quant_mode.packing_divisor() as u32;let weights_stride = model_dim / packing_divisor;@@ -131,7 +131,7 @@ fn get_test_data_symmetric_u8<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {let input_scale = 1.5f32;let quant_mode = QuantizationMode::U8;- let token_ids: Box<[u64]> = Box::new([2, 5, 0]);+ let token_ids: Box<[u32]> = Box::new([2, 5, 0]);let weights: Vec<u8> =(0..vocab_size as usize * model_dim as usize).map(|i| ((i * 37 + 11) & 0xFF) as u8).collect();@@ -168,7 +168,7 @@ fn get_test_data_oob<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {let input_scale = 1.0f32;// token_id 1 is valid, token_id 10 is out of bounds- let token_ids: Box<[u64]> = Box::new([1, 10]);+ let token_ids: Box<[u32]> = Box::new([1, 10]);let weights: Vec<u8> = (0..vocab_size as usize * model_dim as usize).map(|i| (i % 200) as u8).collect();@@ -210,7 +210,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {).expect("Failed to create QuantizedEmbeddingLookupKernel");- let token_ids_allocation = alloc_allocation_with_data::<B, u64>(&context, &input.token_ids);+ let token_ids_allocation = alloc_allocation_with_data::<B, u32>(&context, &input.token_ids);let weights_allocation = alloc_allocation_with_data::<B, u8>(&context, &input.weights);let scales_allocation = alloc_allocation_with_data::<B, T>(&context, &input.scales);let zero_points_allocation =diff --git a/crates/backend-uzu/unit/backends/common/kernel/mod.rs b/crates/backend-uzu/unit/backends/common/kernel/mod.rsindex 442f83e34..5ac3ed84b 100644--- a/crates/backend-uzu/unit/backends/common/kernel/mod.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/mod.rs@@ -16,4 +16,3 @@ mod tensor_add_bias_test;mod tensor_add_scale_test;mod tensor_add_swap_test;mod tensor_copy_test;-mod token_copy_test;diff --git a/crates/backend-uzu/unit/backends/common/kernel/token_copy_test.rs b/crates/backend-uzu/unit/backends/common/kernel/token_copy_test.rsdeleted file mode 100644index 14a4c05d1..000000000--- a/crates/backend-uzu/unit/backends/common/kernel/token_copy_test.rs+++ /dev/null@@ -1,33 +0,0 @@-use proc_macros::uzu_test;-use test_runner::for_each_backend;--use crate::{- backends::common::{Backend, Context, Encoder, Kernels, kernel::TokenCopySampledKernel},- tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},-};--fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {- let context = B::Context::new().expect("Failed to create Context");-- let kernel = <<B as Backend>::Kernels as Kernels>::TokenCopySampledKernel::new(&context)- .expect("Failed to create TokenCopySampledKernel");-- let src_allocation = alloc_allocation_with_data::<B, u32>(&context, &[src_value]);- let mut dst_allocation = alloc_allocation::<B, u64>(&context, 1);-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(&src_allocation, &mut dst_allocation, &mut encoder);- encoder.end_encoding().submit().wait_until_completed().unwrap();-- let output = allocation_to_vec::<B, u64>(&dst_allocation);- assert_eq!(output[0], src_value as u64, "TokenCopySampled failed for backend {}", std::any::type_name::<B>());-}--#[uzu_test]-fn test_token_copy_sampled() {- for_each_backend!(|B| {- test_token_copy_sampled_impl::<B>(0);- test_token_copy_sampled_impl::<B>(42);- test_token_copy_sampled_impl::<B>(u32::MAX);- });-}