A high-performance inference engine for AI models
CPU token_* and tensor_* kernels (#234)
Aleksandr Golokoz92a57d8d67d0
diff --git a/crates/uzu/src/array.rs b/crates/uzu/src/array.rsindex fd7199515..20b7f16e1 100644--- a/crates/uzu/src/array.rs+++ b/crates/uzu/src/array.rs@@ -270,7 +270,7 @@ pub trait ArrayContextExt {data.len(),size_from_shape,"Shape size {} and data size {} are not equal",- shape.len(),+ size_from_shape,data.len());diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rsindex 3767ad7d9..6a590fc17 100644--- a/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs@@ -18,5 +18,14 @@ pub fn tensor_add_bias<T: ArrayElement + Float>(#[specialize]in_place: bool,) {- todo!()+ for i in 0usize..(length as usize) {+ let bias_position = (i % num_cols as usize);+ unsafe {+ if let Some(in_data) = input {+ *output.add(i) = *in_data.add(i) + *bias.add(bias_position);+ } else {+ *output.add(i) = *output.add(i) + *bias.add(bias_position);+ }+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/tensor_add_swap.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/tensor_add_swap.rsindex c4f234600..6f4e1e371 100644--- a/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/tensor_add_swap.rs+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/tensor_add_swap.rs@@ -11,5 +11,11 @@ pub fn tensor_add_swap<T: ArrayElement + Float>(#[allow(unused)] main_buffer: *mut T,#[allow(unused)] length: u32,) {- todo!()+ unsafe {+ for i in 0..length {+ let result = *skip_buffer.add(i as usize) + *main_buffer.add(i as usize);+ *skip_buffer.add(i as usize) = result;+ *main_buffer.add(i as usize) = result;+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_copy/tensor_copy.rs b/crates/uzu/src/backends/cpu/kernel/tensor_copy/tensor_copy.rsindex c2e19054a..1459b590d 100644--- a/crates/uzu/src/backends/cpu/kernel/tensor_copy/tensor_copy.rs+++ b/crates/uzu/src/backends/cpu/kernel/tensor_copy/tensor_copy.rs@@ -11,5 +11,9 @@ pub fn tensor_copy<T: ArrayElement + Float>(#[allow(unused)] dst_buffer: *mut T,#[allow(unused)] length: u32,) {- todo!()+ unsafe {+ for i in 0usize..(length as usize) {+ *dst_buffer.offset(i as isize) = *src_buffer.offset(i as isize);+ }+ }}diff --git a/crates/uzu/src/backends/cpu/kernel/token_copy/token_copy.rs b/crates/uzu/src/backends/cpu/kernel/token_copy/token_copy.rsindex 9da03daaa..f97649550 100644--- a/crates/uzu/src/backends/cpu/kernel/token_copy/token_copy.rs+++ b/crates/uzu/src/backends/cpu/kernel/token_copy/token_copy.rs@@ -8,7 +8,9 @@ pub fn token_copy_sampled(#[allow(unused)] src: *const u32,#[allow(unused)] dst: *mut u64,) {- todo!()+ unsafe {+ *dst.add(0) = *src.add(0) as u64;+ }}#[kernel(TokenCopyToResults)]@@ -16,5 +18,7 @@ pub fn token_copy_to_results(#[allow(unused)] src: *const u32,#[allow(unused)] dst: *mut u32,) {- todo!()+ unsafe {+ *dst.add(0) = *src.add(0);+ }}diff --git a/crates/uzu/tests/kernel/full_precision_embedding.rs b/crates/uzu/tests/kernel/full_precision_embedding_test.rssimilarity index 74%rename from crates/uzu/tests/kernel/full_precision_embedding.rsrename to crates/uzu/tests/kernel/full_precision_embedding_test.rsindex e68836b64..2202a09bf 100644--- a/crates/uzu/tests/kernel/full_precision_embedding.rs+++ b/crates/uzu/tests/kernel/full_precision_embedding_test.rs@@ -1,5 +1,9 @@-use std::ops::{Deref, DerefMut};+use std::{+ fmt::Debug,+ ops::{Deref, DerefMut},+};+use half::{bf16, f16};use num_traits::Float;use uzu::{ArrayElement,@@ -16,28 +20,26 @@ struct Input<T: ArrayElement + Float> {input_scale: f32,}-fn get_test_data() -> (Input<f32>, Vec<f32>) {+fn get_test_data<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {let vocab_size = 11;let model_dim = 513;let batch_size = 7;let input_scale = 2.0_f32;let token_ids: Box<[u64]> = Box::new([3, 7, 1, 10, 0, 5, 8]);-- let weights: Box<[f32]> =- (0..vocab_size * model_dim).map(|i| (i as f32 + 1.0) * 0.01).collect::<Vec<_>>().into_boxed_slice();-- let mut expected = Vec::with_capacity(batch_size * model_dim);+ let weights: Vec<T> =+ (0..vocab_size * model_dim).map(|i| T::from((i as f32).sin() * 30.0f32).unwrap()).collect::<Vec<_>>();+ let mut expected: Vec<T> = Vec::with_capacity(batch_size * model_dim);for &tid in token_ids.iter() {let row_start = tid as usize * model_dim;for j in 0..model_dim {- expected.push(weights[row_start + j] * input_scale);+ expected.push(weights[row_start + j] * T::from(input_scale).unwrap());}}let input = Input {token_ids,- weights,+ weights: weights.iter().map(|&v| T::from(v).unwrap()).collect(),batch_size,vocab_size,model_dim,@@ -77,11 +79,25 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {output_array.as_slice().to_vec()}-#[test]-fn test() {- let (input, expected) = get_test_data();+fn test<T: ArrayElement + Float + Debug>() {+ let (input, expected) = get_test_data::<T>();for_each_backend!(|B| {- let output = get_output::<f32, B>(&input);+ let output = get_output::<T, B>(&input);assert_eq!(output, expected, "Results are not equals for backend {}", std::any::type_name::<B>());});}++#[test]+fn test_f32() {+ test::<f32>()+}++#[test]+fn test_f16() {+ test::<f16>()+}++#[test]+fn test_bf16() {+ test::<bf16>()+}diff --git a/crates/uzu/tests/kernel/mod.rs b/crates/uzu/tests/kernel/mod.rsindex 199b69466..6029a0067 100644--- a/crates/uzu/tests/kernel/mod.rs+++ b/crates/uzu/tests/kernel/mod.rs@@ -2,5 +2,8 @@#[path = "../common/mod.rs"]mod common;-#[path = "full_precision_embedding.rs"]-mod full_precision_embedding;+mod full_precision_embedding_test;+mod tensor_add_bias_test;+mod tensor_add_swap_test;+mod tensor_copy_test;+mod token_copy_test;diff --git a/crates/uzu/tests/kernel/tensor_add_bias_test.rs b/crates/uzu/tests/kernel/tensor_add_bias_test.rsnew file mode 100644index 000000000..1baeb5116--- /dev/null+++ b/crates/uzu/tests/kernel/tensor_add_bias_test.rs@@ -0,0 +1,131 @@+use std::{+ fmt::Debug,+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayElement,+ array::ArrayContextExt,+ backends::common::{Backend, CommandBuffer, Context, Kernels, kernel::TensorAddBiasKernel},+};++struct Input<T: ArrayElement + Float> {+ input: Box<[T]>,+ bias: Box<[T]>,+ output: Box<[T]>,+ num_cols: u32,+ length: u32,+}++fn get_test_data<T: ArrayElement + Float>(in_place: bool) -> (Input<T>, Vec<T>) {+ let length = 1025usize;+ let num_cols = 129usize;++ let mut input: Vec<T> = vec![T::zero(); length];+ let mut bias: Vec<T> = vec![T::zero(); num_cols];+ let mut output: Vec<T> = vec![T::zero(); length];+ let mut expected: Vec<T> = vec![T::zero(); length];++ for i in 0..length {+ input[i] = T::from((i as f32).sin() * 30f32).unwrap();+ if i < num_cols {+ bias[i] = T::from((i as f32).cos() * 30f32).unwrap();+ }+ output[i] = T::from((i as f32).sin() * 30f32).unwrap();++ if in_place {+ expected[i] = output[i] + bias[i % num_cols];+ } else {+ expected[i] = input[i] + bias[i % num_cols];+ }+ }++ let input = Input {+ input: input.into_boxed_slice(),+ bias: bias.into_boxed_slice(),+ output: output.into_boxed_slice(),+ num_cols: num_cols as u32,+ length: length as u32,+ };++ (input, expected)+}++fn get_output<T: ArrayElement + Float, B: Backend>(+ input: &Input<T>,+ in_place: bool,+) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(&context, T::data_type(), in_place)+ .expect("Failed to create TensorAddBiasKernel");++ let size = input.length as usize;+ let input_array = context.create_array_from(&[size], &input.input, "");+ let input_array_buffer_rc = input_array.buffer();+ let input_array_borrow = input_array_buffer_rc.borrow();+ let input_array_deref = input_array_borrow.deref();+ let input_buffer = (!in_place).then(|| input_array_deref);++ let bias_array = context.create_array_from(&[input.num_cols as usize], &input.bias, "");+ let output_array = match in_place {+ true => context.create_array_from(&[size], &input.output, ""),+ false => context.create_array_uninitialized(&[size], T::data_type(), ""),+ };++ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");+ command_buffer.with_compute_encoder(|encoder| {+ kernel.encode(+ input_buffer,+ bias_array.buffer().borrow_mut().deref(),+ output_array.buffer().borrow_mut().deref_mut(),+ input.num_cols,+ input.length,+ encoder,+ )+ });+ command_buffer.submit();+ command_buffer.wait_until_completed();++ output_array.as_slice().to_vec()+}++fn test<T: ArrayElement + Float + Debug>(in_place: bool) {+ let (input, expected) = get_test_data::<T>(in_place);+ for_each_backend!(|B| {+ let output = get_output::<T, B>(&input, in_place);+ assert_eq!(output, expected, "Results are not equals for backend {}", std::any::type_name::<B>());+ });+}++#[test]+fn test_f32() {+ test::<f32>(false);+}++#[test]+fn test_f32_in_place() {+ test::<f32>(true);+}++#[test]+fn test_f16() {+ test::<f16>(false);+}++#[test]+fn test_f16_in_place() {+ test::<f16>(true);+}++#[test]+fn test_bf16() {+ test::<bf16>(false);+}++#[test]+fn test_bf16_in_place() {+ test::<bf16>(true);+}diff --git a/crates/uzu/tests/kernel/tensor_add_swap_test.rs b/crates/uzu/tests/kernel/tensor_add_swap_test.rsnew file mode 100644index 000000000..61a8ebdc7--- /dev/null+++ b/crates/uzu/tests/kernel/tensor_add_swap_test.rs@@ -0,0 +1,96 @@+use std::{fmt::Debug, ops::DerefMut};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayElement,+ array::ArrayContextExt,+ backends::common::{Backend, CommandBuffer, Context, Kernels, kernel::TensorAddSwapKernel},+};++struct Input<T: ArrayElement + Float> {+ skip_buffer: Box<[T]>,+ main_buffer: Box<[T]>,+ length: u32,+}++fn get_test_data<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let length = 1025usize;++ let mut skip_buffer: Vec<T> = vec![T::zero(); length];+ let mut main_buffer: Vec<T> = vec![T::zero(); length];+ let mut expected: Vec<T> = vec![T::zero(); length];++ for i in 0..length {+ skip_buffer[i] = T::from((i as f32).sin() * 30f32).unwrap();+ main_buffer[i] = T::from((i as f32).cos() * 30f32).unwrap();+ expected[i] = skip_buffer[i] + main_buffer[i];+ }++ let input = Input {+ skip_buffer: skip_buffer.into_boxed_slice(),+ main_buffer: main_buffer.into_boxed_slice(),+ length: length as u32,+ };++ (input, expected)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::TensorAddSwapKernel::new(&context, T::data_type())+ .expect("Failed to create TensorAddSwapKernel");++ let size = input.length as usize;+ let skip_array = context.create_array_from(&[size], &input.skip_buffer, "");+ let main_array = context.create_array_from(&[size], &input.main_buffer, "");++ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");+ command_buffer.with_compute_encoder(|encoder| {+ kernel.encode(+ skip_array.buffer().borrow_mut().deref_mut(),+ main_array.buffer().borrow_mut().deref_mut(),+ input.length,+ encoder,+ )+ });+ command_buffer.submit();+ command_buffer.wait_until_completed();++ (skip_array.as_slice().to_vec(), main_array.as_slice().to_vec())+}++fn test<T: ArrayElement + Float + Debug>() {+ let (input, expected) = get_test_data::<T>();+ for_each_backend!(|B| {+ let (skip_output, main_output) = get_output::<T, B>(&input);+ assert_eq!(+ skip_output,+ expected,+ "skip_buffer results are not equal for backend {}",+ std::any::type_name::<B>()+ );+ assert_eq!(+ main_output,+ expected,+ "main_buffer results are not equal for backend {}",+ std::any::type_name::<B>()+ );+ });+}++#[test]+fn test_f32() {+ test::<f32>();+}++#[test]+fn test_f16() {+ test::<f16>();+}++#[test]+fn test_bf16() {+ test::<bf16>();+}diff --git a/crates/uzu/tests/kernel/tensor_copy_test.rs b/crates/uzu/tests/kernel/tensor_copy_test.rsnew file mode 100644index 000000000..3053e81a4--- /dev/null+++ b/crates/uzu/tests/kernel/tensor_copy_test.rs@@ -0,0 +1,84 @@+use std::{+ fmt::Debug,+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayElement,+ array::ArrayContextExt,+ backends::common::{Backend, CommandBuffer, Context, Kernels, kernel::TensorCopyKernel},+};++struct Input<T: ArrayElement + Float> {+ src: Box<[T]>,+ length: u32,+}++fn get_test_data<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {+ let length = 1025usize;++ let mut src: Vec<T> = vec![T::zero(); length];+ let mut expected: Vec<T> = vec![T::zero(); length];++ for i in 0..length {+ src[i] = T::from((i as f32).sin() * 30f32).unwrap();+ expected[i] = src[i];+ }++ let input = Input {+ src: src.into_boxed_slice(),+ length: length as u32,+ };++ (input, expected)+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::TensorCopyKernel::new(&context, T::data_type())+ .expect("Failed to create TensorCopyKernel");++ let size = input.length as usize;+ let src_array = context.create_array_from(&[size], &input.src, "");+ let dst_array = context.create_array_uninitialized(&[size], T::data_type(), "");++ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");+ command_buffer.with_compute_encoder(|encoder| {+ kernel.encode(+ src_array.buffer().borrow().deref(),+ dst_array.buffer().borrow_mut().deref_mut(),+ input.length,+ encoder,+ )+ });+ command_buffer.submit();+ command_buffer.wait_until_completed();++ dst_array.as_slice().to_vec()+}++fn test<T: ArrayElement + Float + Debug>() {+ let (input, expected) = get_test_data::<T>();+ for_each_backend!(|B| {+ let output = get_output::<T, B>(&input);+ assert_eq!(output, expected, "Results are not equal for backend {}", std::any::type_name::<B>());+ });+}++#[test]+fn test_f32() {+ test::<f32>();+}++#[test]+fn test_f16() {+ test::<f16>();+}++#[test]+fn test_bf16() {+ test::<bf16>();+}diff --git a/crates/uzu/tests/kernel/token_copy_test.rs b/crates/uzu/tests/kernel/token_copy_test.rsnew file mode 100644index 000000000..d4b64007d--- /dev/null+++ b/crates/uzu/tests/kernel/token_copy_test.rs@@ -0,0 +1,68 @@+use std::ops::{Deref, DerefMut};++use uzu::{+ ArrayElement,+ array::ArrayContextExt,+ backends::common::{+ Backend, CommandBuffer, Context, Kernels,+ kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel},+ },+};++fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::TokenCopySampledKernel::new(&context)+ .expect("Failed to create TokenCopySampledKernel");++ let src_array = context.create_array_from(&[1], &[src_value], "");+ let dst_array = context.create_array_uninitialized(&[1], u64::data_type(), "");++ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");+ command_buffer.with_compute_encoder(|encoder| {+ kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), encoder)+ });+ command_buffer.submit();+ command_buffer.wait_until_completed();++ let output = dst_array.as_slice::<u64>().to_vec();+ assert_eq!(output[0], src_value as u64, "TokenCopySampled failed for backend {}", std::any::type_name::<B>());+}++fn test_token_copy_to_results_impl<B: Backend>(src_value: u32) {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::TokenCopyToResultsKernel::new(&context)+ .expect("Failed to create TokenCopyToResultsKernel");++ let src_array = context.create_array_from(&[1], &[src_value], "");+ let dst_array = context.create_array_uninitialized(&[1], u32::data_type(), "");++ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");+ command_buffer.with_compute_encoder(|encoder| {+ kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), encoder)+ });+ command_buffer.submit();+ command_buffer.wait_until_completed();++ let output = dst_array.as_slice::<u32>().to_vec();+ assert_eq!(output[0], src_value, "TokenCopyToResults failed for backend {}", std::any::type_name::<B>());+}++#[test]+fn test_token_copy_sampled() {+ for_each_backend!(|B| {+ test_token_copy_sampled_impl::<B>(0);+ test_token_copy_sampled_impl::<B>(42);+ test_token_copy_sampled_impl::<B>(u32::MAX);+ });+}++#[test]+fn test_token_copy_to_results() {+ for_each_backend!(|B| {+ test_token_copy_to_results_impl::<B>(0);+ test_token_copy_to_results_impl::<B>(42);+ test_token_copy_to_results_impl::<B>(u32::MAX);+ });+}