Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

CPU token_* and tensor_* kernels (#234)

Aleksandr Golokoz92a57d8d67d0
diff --git a/crates/uzu/src/array.rs b/crates/uzu/src/array.rs
index fd7199515..20b7f16e1 100644
--- a/crates/uzu/src/array.rs
+++ b/crates/uzu/src/array.rs
@@ -270,7 +270,7 @@ pub trait ArrayContextExt {
data.len(),
size_from_shape,
"Shape size {} and data size {} are not equal",
- shape.len(),
+ size_from_shape,
data.len()
);
diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs
index 3767ad7d9..6a590fc17 100644
--- a/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs
+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs
@@ -18,5 +18,14 @@ pub fn tensor_add_bias<T: ArrayElement + Float>(
#[specialize]
in_place: bool,
) {
- todo!()
+ for i in 0usize..(length as usize) {
+ let bias_position = (i % num_cols as usize);
+ unsafe {
+ if let Some(in_data) = input {
+ *output.add(i) = *in_data.add(i) + *bias.add(bias_position);
+ } else {
+ *output.add(i) = *output.add(i) + *bias.add(bias_position);
+ }
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/tensor_add_swap.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/tensor_add_swap.rs
index c4f234600..6f4e1e371 100644
--- a/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/tensor_add_swap.rs
+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_swap/tensor_add_swap.rs
@@ -11,5 +11,11 @@ pub fn tensor_add_swap<T: ArrayElement + Float>(
#[allow(unused)] main_buffer: *mut T,
#[allow(unused)] length: u32,
) {
- todo!()
+ unsafe {
+ for i in 0..length {
+ let result = *skip_buffer.add(i as usize) + *main_buffer.add(i as usize);
+ *skip_buffer.add(i as usize) = result;
+ *main_buffer.add(i as usize) = result;
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_copy/tensor_copy.rs b/crates/uzu/src/backends/cpu/kernel/tensor_copy/tensor_copy.rs
index c2e19054a..1459b590d 100644
--- a/crates/uzu/src/backends/cpu/kernel/tensor_copy/tensor_copy.rs
+++ b/crates/uzu/src/backends/cpu/kernel/tensor_copy/tensor_copy.rs
@@ -11,5 +11,9 @@ pub fn tensor_copy<T: ArrayElement + Float>(
#[allow(unused)] dst_buffer: *mut T,
#[allow(unused)] length: u32,
) {
- todo!()
+ unsafe {
+ for i in 0usize..(length as usize) {
+ *dst_buffer.offset(i as isize) = *src_buffer.offset(i as isize);
+ }
+ }
}
diff --git a/crates/uzu/src/backends/cpu/kernel/token_copy/token_copy.rs b/crates/uzu/src/backends/cpu/kernel/token_copy/token_copy.rs
index 9da03daaa..f97649550 100644
--- a/crates/uzu/src/backends/cpu/kernel/token_copy/token_copy.rs
+++ b/crates/uzu/src/backends/cpu/kernel/token_copy/token_copy.rs
@@ -8,7 +8,9 @@ pub fn token_copy_sampled(
#[allow(unused)] src: *const u32,
#[allow(unused)] dst: *mut u64,
) {
- todo!()
+ unsafe {
+ *dst.add(0) = *src.add(0) as u64;
+ }
}
#[kernel(TokenCopyToResults)]
@@ -16,5 +18,7 @@ pub fn token_copy_to_results(
#[allow(unused)] src: *const u32,
#[allow(unused)] dst: *mut u32,
) {
- todo!()
+ unsafe {
+ *dst.add(0) = *src.add(0);
+ }
}
diff --git a/crates/uzu/tests/kernel/full_precision_embedding.rs b/crates/uzu/tests/kernel/full_precision_embedding_test.rs
similarity index 74%
rename from crates/uzu/tests/kernel/full_precision_embedding.rs
rename to crates/uzu/tests/kernel/full_precision_embedding_test.rs
index e68836b64..2202a09bf 100644
--- a/crates/uzu/tests/kernel/full_precision_embedding.rs
+++ b/crates/uzu/tests/kernel/full_precision_embedding_test.rs
@@ -1,5 +1,9 @@
-use std::ops::{Deref, DerefMut};
+use std::{
+ fmt::Debug,
+ ops::{Deref, DerefMut},
+};
+use half::{bf16, f16};
use num_traits::Float;
use uzu::{
ArrayElement,
@@ -16,28 +20,26 @@ struct Input<T: ArrayElement + Float> {
input_scale: f32,
}
-fn get_test_data() -> (Input<f32>, Vec<f32>) {
+fn get_test_data<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
let vocab_size = 11;
let model_dim = 513;
let batch_size = 7;
let input_scale = 2.0_f32;
let token_ids: Box<[u64]> = Box::new([3, 7, 1, 10, 0, 5, 8]);
-
- let weights: Box<[f32]> =
- (0..vocab_size * model_dim).map(|i| (i as f32 + 1.0) * 0.01).collect::<Vec<_>>().into_boxed_slice();
-
- let mut expected = Vec::with_capacity(batch_size * model_dim);
+ let weights: Vec<T> =
+ (0..vocab_size * model_dim).map(|i| T::from((i as f32).sin() * 30.0f32).unwrap()).collect::<Vec<_>>();
+ let mut expected: Vec<T> = Vec::with_capacity(batch_size * model_dim);
for &tid in token_ids.iter() {
let row_start = tid as usize * model_dim;
for j in 0..model_dim {
- expected.push(weights[row_start + j] * input_scale);
+ expected.push(weights[row_start + j] * T::from(input_scale).unwrap());
}
}
let input = Input {
token_ids,
- weights,
+ weights: weights.iter().map(|&v| T::from(v).unwrap()).collect(),
batch_size,
vocab_size,
model_dim,
@@ -77,11 +79,25 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
output_array.as_slice().to_vec()
}
-#[test]
-fn test() {
- let (input, expected) = get_test_data();
+fn test<T: ArrayElement + Float + Debug>() {
+ let (input, expected) = get_test_data::<T>();
for_each_backend!(|B| {
- let output = get_output::<f32, B>(&input);
+ let output = get_output::<T, B>(&input);
assert_eq!(output, expected, "Results are not equals for backend {}", std::any::type_name::<B>());
});
}
+
+#[test]
+fn test_f32() {
+ test::<f32>()
+}
+
+#[test]
+fn test_f16() {
+ test::<f16>()
+}
+
+#[test]
+fn test_bf16() {
+ test::<bf16>()
+}
diff --git a/crates/uzu/tests/kernel/mod.rs b/crates/uzu/tests/kernel/mod.rs
index 199b69466..6029a0067 100644
--- a/crates/uzu/tests/kernel/mod.rs
+++ b/crates/uzu/tests/kernel/mod.rs
@@ -2,5 +2,8 @@
#[path = "../common/mod.rs"]
mod common;
-#[path = "full_precision_embedding.rs"]
-mod full_precision_embedding;
+mod full_precision_embedding_test;
+mod tensor_add_bias_test;
+mod tensor_add_swap_test;
+mod tensor_copy_test;
+mod token_copy_test;
diff --git a/crates/uzu/tests/kernel/tensor_add_bias_test.rs b/crates/uzu/tests/kernel/tensor_add_bias_test.rs
new file mode 100644
index 000000000..1baeb5116
--- /dev/null
+++ b/crates/uzu/tests/kernel/tensor_add_bias_test.rs
@@ -0,0 +1,131 @@
+use std::{
+ fmt::Debug,
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayElement,
+ array::ArrayContextExt,
+ backends::common::{Backend, CommandBuffer, Context, Kernels, kernel::TensorAddBiasKernel},
+};
+
+struct Input<T: ArrayElement + Float> {
+ input: Box<[T]>,
+ bias: Box<[T]>,
+ output: Box<[T]>,
+ num_cols: u32,
+ length: u32,
+}
+
+fn get_test_data<T: ArrayElement + Float>(in_place: bool) -> (Input<T>, Vec<T>) {
+ let length = 1025usize;
+ let num_cols = 129usize;
+
+ let mut input: Vec<T> = vec![T::zero(); length];
+ let mut bias: Vec<T> = vec![T::zero(); num_cols];
+ let mut output: Vec<T> = vec![T::zero(); length];
+ let mut expected: Vec<T> = vec![T::zero(); length];
+
+ for i in 0..length {
+ input[i] = T::from((i as f32).sin() * 30f32).unwrap();
+ if i < num_cols {
+ bias[i] = T::from((i as f32).cos() * 30f32).unwrap();
+ }
+ output[i] = T::from((i as f32).sin() * 30f32).unwrap();
+
+ if in_place {
+ expected[i] = output[i] + bias[i % num_cols];
+ } else {
+ expected[i] = input[i] + bias[i % num_cols];
+ }
+ }
+
+ let input = Input {
+ input: input.into_boxed_slice(),
+ bias: bias.into_boxed_slice(),
+ output: output.into_boxed_slice(),
+ num_cols: num_cols as u32,
+ length: length as u32,
+ };
+
+ (input, expected)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(
+ input: &Input<T>,
+ in_place: bool,
+) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(&context, T::data_type(), in_place)
+ .expect("Failed to create TensorAddBiasKernel");
+
+ let size = input.length as usize;
+ let input_array = context.create_array_from(&[size], &input.input, "");
+ let input_array_buffer_rc = input_array.buffer();
+ let input_array_borrow = input_array_buffer_rc.borrow();
+ let input_array_deref = input_array_borrow.deref();
+ let input_buffer = (!in_place).then(|| input_array_deref);
+
+ let bias_array = context.create_array_from(&[input.num_cols as usize], &input.bias, "");
+ let output_array = match in_place {
+ true => context.create_array_from(&[size], &input.output, ""),
+ false => context.create_array_uninitialized(&[size], T::data_type(), ""),
+ };
+
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
+ command_buffer.with_compute_encoder(|encoder| {
+ kernel.encode(
+ input_buffer,
+ bias_array.buffer().borrow_mut().deref(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.num_cols,
+ input.length,
+ encoder,
+ )
+ });
+ command_buffer.submit();
+ command_buffer.wait_until_completed();
+
+ output_array.as_slice().to_vec()
+}
+
+fn test<T: ArrayElement + Float + Debug>(in_place: bool) {
+ let (input, expected) = get_test_data::<T>(in_place);
+ for_each_backend!(|B| {
+ let output = get_output::<T, B>(&input, in_place);
+ assert_eq!(output, expected, "Results are not equals for backend {}", std::any::type_name::<B>());
+ });
+}
+
+#[test]
+fn test_f32() {
+ test::<f32>(false);
+}
+
+#[test]
+fn test_f32_in_place() {
+ test::<f32>(true);
+}
+
+#[test]
+fn test_f16() {
+ test::<f16>(false);
+}
+
+#[test]
+fn test_f16_in_place() {
+ test::<f16>(true);
+}
+
+#[test]
+fn test_bf16() {
+ test::<bf16>(false);
+}
+
+#[test]
+fn test_bf16_in_place() {
+ test::<bf16>(true);
+}
diff --git a/crates/uzu/tests/kernel/tensor_add_swap_test.rs b/crates/uzu/tests/kernel/tensor_add_swap_test.rs
new file mode 100644
index 000000000..61a8ebdc7
--- /dev/null
+++ b/crates/uzu/tests/kernel/tensor_add_swap_test.rs
@@ -0,0 +1,96 @@
+use std::{fmt::Debug, ops::DerefMut};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayElement,
+ array::ArrayContextExt,
+ backends::common::{Backend, CommandBuffer, Context, Kernels, kernel::TensorAddSwapKernel},
+};
+
+struct Input<T: ArrayElement + Float> {
+ skip_buffer: Box<[T]>,
+ main_buffer: Box<[T]>,
+ length: u32,
+}
+
+fn get_test_data<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let length = 1025usize;
+
+ let mut skip_buffer: Vec<T> = vec![T::zero(); length];
+ let mut main_buffer: Vec<T> = vec![T::zero(); length];
+ let mut expected: Vec<T> = vec![T::zero(); length];
+
+ for i in 0..length {
+ skip_buffer[i] = T::from((i as f32).sin() * 30f32).unwrap();
+ main_buffer[i] = T::from((i as f32).cos() * 30f32).unwrap();
+ expected[i] = skip_buffer[i] + main_buffer[i];
+ }
+
+ let input = Input {
+ skip_buffer: skip_buffer.into_boxed_slice(),
+ main_buffer: main_buffer.into_boxed_slice(),
+ length: length as u32,
+ };
+
+ (input, expected)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::TensorAddSwapKernel::new(&context, T::data_type())
+ .expect("Failed to create TensorAddSwapKernel");
+
+ let size = input.length as usize;
+ let skip_array = context.create_array_from(&[size], &input.skip_buffer, "");
+ let main_array = context.create_array_from(&[size], &input.main_buffer, "");
+
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
+ command_buffer.with_compute_encoder(|encoder| {
+ kernel.encode(
+ skip_array.buffer().borrow_mut().deref_mut(),
+ main_array.buffer().borrow_mut().deref_mut(),
+ input.length,
+ encoder,
+ )
+ });
+ command_buffer.submit();
+ command_buffer.wait_until_completed();
+
+ (skip_array.as_slice().to_vec(), main_array.as_slice().to_vec())
+}
+
+fn test<T: ArrayElement + Float + Debug>() {
+ let (input, expected) = get_test_data::<T>();
+ for_each_backend!(|B| {
+ let (skip_output, main_output) = get_output::<T, B>(&input);
+ assert_eq!(
+ skip_output,
+ expected,
+ "skip_buffer results are not equal for backend {}",
+ std::any::type_name::<B>()
+ );
+ assert_eq!(
+ main_output,
+ expected,
+ "main_buffer results are not equal for backend {}",
+ std::any::type_name::<B>()
+ );
+ });
+}
+
+#[test]
+fn test_f32() {
+ test::<f32>();
+}
+
+#[test]
+fn test_f16() {
+ test::<f16>();
+}
+
+#[test]
+fn test_bf16() {
+ test::<bf16>();
+}
diff --git a/crates/uzu/tests/kernel/tensor_copy_test.rs b/crates/uzu/tests/kernel/tensor_copy_test.rs
new file mode 100644
index 000000000..3053e81a4
--- /dev/null
+++ b/crates/uzu/tests/kernel/tensor_copy_test.rs
@@ -0,0 +1,84 @@
+use std::{
+ fmt::Debug,
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayElement,
+ array::ArrayContextExt,
+ backends::common::{Backend, CommandBuffer, Context, Kernels, kernel::TensorCopyKernel},
+};
+
+struct Input<T: ArrayElement + Float> {
+ src: Box<[T]>,
+ length: u32,
+}
+
+fn get_test_data<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
+ let length = 1025usize;
+
+ let mut src: Vec<T> = vec![T::zero(); length];
+ let mut expected: Vec<T> = vec![T::zero(); length];
+
+ for i in 0..length {
+ src[i] = T::from((i as f32).sin() * 30f32).unwrap();
+ expected[i] = src[i];
+ }
+
+ let input = Input {
+ src: src.into_boxed_slice(),
+ length: length as u32,
+ };
+
+ (input, expected)
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::TensorCopyKernel::new(&context, T::data_type())
+ .expect("Failed to create TensorCopyKernel");
+
+ let size = input.length as usize;
+ let src_array = context.create_array_from(&[size], &input.src, "");
+ let dst_array = context.create_array_uninitialized(&[size], T::data_type(), "");
+
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
+ command_buffer.with_compute_encoder(|encoder| {
+ kernel.encode(
+ src_array.buffer().borrow().deref(),
+ dst_array.buffer().borrow_mut().deref_mut(),
+ input.length,
+ encoder,
+ )
+ });
+ command_buffer.submit();
+ command_buffer.wait_until_completed();
+
+ dst_array.as_slice().to_vec()
+}
+
+fn test<T: ArrayElement + Float + Debug>() {
+ let (input, expected) = get_test_data::<T>();
+ for_each_backend!(|B| {
+ let output = get_output::<T, B>(&input);
+ assert_eq!(output, expected, "Results are not equal for backend {}", std::any::type_name::<B>());
+ });
+}
+
+#[test]
+fn test_f32() {
+ test::<f32>();
+}
+
+#[test]
+fn test_f16() {
+ test::<f16>();
+}
+
+#[test]
+fn test_bf16() {
+ test::<bf16>();
+}
diff --git a/crates/uzu/tests/kernel/token_copy_test.rs b/crates/uzu/tests/kernel/token_copy_test.rs
new file mode 100644
index 000000000..d4b64007d
--- /dev/null
+++ b/crates/uzu/tests/kernel/token_copy_test.rs
@@ -0,0 +1,68 @@
+use std::ops::{Deref, DerefMut};
+
+use uzu::{
+ ArrayElement,
+ array::ArrayContextExt,
+ backends::common::{
+ Backend, CommandBuffer, Context, Kernels,
+ kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel},
+ },
+};
+
+fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::TokenCopySampledKernel::new(&context)
+ .expect("Failed to create TokenCopySampledKernel");
+
+ let src_array = context.create_array_from(&[1], &[src_value], "");
+ let dst_array = context.create_array_uninitialized(&[1], u64::data_type(), "");
+
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
+ command_buffer.with_compute_encoder(|encoder| {
+ kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), encoder)
+ });
+ command_buffer.submit();
+ command_buffer.wait_until_completed();
+
+ let output = dst_array.as_slice::<u64>().to_vec();
+ assert_eq!(output[0], src_value as u64, "TokenCopySampled failed for backend {}", std::any::type_name::<B>());
+}
+
+fn test_token_copy_to_results_impl<B: Backend>(src_value: u32) {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::TokenCopyToResultsKernel::new(&context)
+ .expect("Failed to create TokenCopyToResultsKernel");
+
+ let src_array = context.create_array_from(&[1], &[src_value], "");
+ let dst_array = context.create_array_uninitialized(&[1], u32::data_type(), "");
+
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
+ command_buffer.with_compute_encoder(|encoder| {
+ kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), encoder)
+ });
+ command_buffer.submit();
+ command_buffer.wait_until_completed();
+
+ let output = dst_array.as_slice::<u32>().to_vec();
+ assert_eq!(output[0], src_value, "TokenCopyToResults failed for backend {}", std::any::type_name::<B>());
+}
+
+#[test]
+fn test_token_copy_sampled() {
+ for_each_backend!(|B| {
+ test_token_copy_sampled_impl::<B>(0);
+ test_token_copy_sampled_impl::<B>(42);
+ test_token_copy_sampled_impl::<B>(u32::MAX);
+ });
+}
+
+#[test]
+fn test_token_copy_to_results() {
+ for_each_backend!(|B| {
+ test_token_copy_to_results_impl::<B>(0);
+ test_token_copy_to_results_impl::<B>(42);
+ test_token_copy_to_results_impl::<B>(u32::MAX);
+ });
+}