A high-performance inference engine for AI models
Add rope.rs (#240)
Aleksandr Golokoz62a812f9d04e
diff --git a/crates/uzu/src/backends/cpu/kernel/rope/rope.rs b/crates/uzu/src/backends/cpu/kernel/rope/rope.rsindex ea5da869f..21e0c84bd 100644--- a/crates/uzu/src/backends/cpu/kernel/rope/rope.rs+++ b/crates/uzu/src/backends/cpu/kernel/rope/rope.rs@@ -4,20 +4,94 @@ use num_traits::Float;use crate::ArrayElement;+fn get_paired_val<T: ArrayElement + Float>(+ values: *const T,+ dim_index: usize,+ half_dim: usize,+ token_index: usize,+ total_heads: usize,+ head_dim: usize,+ head_index: usize,+) -> T {+ if dim_index < half_dim {+ let v =+ unsafe { *values.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index + half_dim) };+ T::zero() - v+ } else {+ unsafe { *values.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index - half_dim) }+ }+}+#[kernel(Rope)]#[variants(T, f32, f16, bf16)]pub fn rope<T: ArrayElement + Float>(- #[allow(unused)] qkv: *const T,- #[allow(unused)] cosines: *const T,- #[allow(unused)] sines: *const T,- #[allow(unused)] token_positions: *const i32,- #[allow(unused)] rotated_queries: *mut T,- #[allow(unused)] rotated_keys: *mut T,- #[allow(unused)] head_dim: u32,- #[allow(unused)] num_heads: u32,- #[allow(unused)] num_groups: u32,- #[allow(unused)] suffix_length: u32,- #[allow(unused)] max_sequence_length: u32,+ qkv: *const T,+ cosines: *const T,+ sines: *const T,+ token_positions: *const i32,+ rotated_queries: *mut T,+ rotated_keys: *mut T,+ head_dim: u32,+ num_heads: u32,+ num_groups: u32,+ suffix_length: u32,+ max_sequence_length: u32,) {- todo!()+ if head_dim & 1 != 0 || num_heads % num_groups != 0 {+ return;+ }++ let head_dim = head_dim as usize;+ let num_heads = num_heads as usize;+ let num_groups = num_groups as usize;+ let suffix_length = suffix_length as usize;+ let max_sequence_length = max_sequence_length as usize;++ let half_dim = head_dim / 2;+ let total_heads = num_heads + 2 * num_groups;+ let heads_per_group = num_heads / num_groups;++ for head_index in 0..num_heads {+ let group_index = head_index / heads_per_group;++ for token_index in 0..suffix_length {+ let raw_position = unsafe { *token_positions.add(token_index) } as usize;+ let absolute_position = if raw_position >= max_sequence_length {+ 0+ } else {+ raw_position+ };++ for dim_index in 0..head_dim {+ let cos_val = unsafe { *cosines.add(absolute_position * head_dim + dim_index) };+ let sin_val = unsafe { *sines.add(absolute_position * head_dim + dim_index) };++ // Query rotation+ let q_val =+ unsafe { *qkv.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index) };+ let q_paired = get_paired_val(qkv, dim_index, half_dim, token_index, total_heads, head_dim, head_index);+ let q_result = q_val * cos_val + q_paired * sin_val;+ unsafe {+ *rotated_queries.add(head_index * suffix_length * head_dim + token_index * head_dim + dim_index) =+ q_result;+ }++ // Key rotation (only first head in each group)+ if head_index == group_index * heads_per_group {+ let key_head_index = num_heads + group_index;+ let k_val = unsafe {+ *qkv.add(token_index * total_heads * head_dim + key_head_index * head_dim + dim_index)+ };+ let k_paired =+ get_paired_val(qkv, dim_index, half_dim, token_index, total_heads, head_dim, key_head_index);+ let k_result = k_val * cos_val + k_paired * sin_val;+ unsafe {+ *rotated_keys+ .add(group_index * suffix_length * head_dim + token_index * head_dim + dim_index) =+ k_result;+ }+ }+ }+ }+ }}diff --git a/crates/uzu/tests/kernel/mod.rs b/crates/uzu/tests/kernel/mod.rsindex fb030b664..455478a42 100644--- a/crates/uzu/tests/kernel/mod.rs+++ b/crates/uzu/tests/kernel/mod.rs@@ -5,6 +5,7 @@ mod common;mod full_precision_embedding_test;mod qk_norm_test;mod rms_norm_test;+mod rope_test;mod tensor_add_bias_test;mod tensor_add_swap_test;mod tensor_copy_test;diff --git a/crates/uzu/tests/kernel/rope_test.rs b/crates/uzu/tests/kernel/rope_test.rsnew file mode 100644index 000000000..be119b7bb--- /dev/null+++ b/crates/uzu/tests/kernel/rope_test.rs@@ -0,0 +1,229 @@+use std::{+ fmt::{Debug, Display},+ ops::{Deref, DerefMut},+};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayElement, DataType,+ array::ArrayContextExt,+ backends::{+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels, kernel::RopeKernel,+ },+ cpu::Cpu,+ },+};++use crate::common::assert::assert_eq_float;++struct Input<T: ArrayElement + Float> {+ qkv: Box<[T]>,+ cosines: Box<[T]>,+ sines: Box<[T]>,+ token_positions: Box<[i32]>,+ head_dim: u32,+ num_heads: u32,+ num_groups: u32,+ suffix_length: u32,+ max_sequence_length: u32,+}++fn get_test_data<T: ArrayElement + Float>(+ num_heads: u32,+ num_groups: u32,+ head_dim: u32,+ suffix_length: u32,+ max_sequence_length: u32,+) -> Input<T> {+ let total_heads = (num_heads + 2 * num_groups) as usize;+ let qkv_size = suffix_length as usize * total_heads * head_dim as usize;+ let cos_sin_size = max_sequence_length as usize * head_dim as usize;++ let mut qkv = vec![T::zero(); qkv_size];+ for i in 0..qkv_size {+ qkv[i] = T::from(((i as f32) * 0.1).sin() * 2.0).unwrap();+ }++ let mut cosines = vec![T::zero(); cos_sin_size];+ let mut sines = vec![T::zero(); cos_sin_size];+ for i in 0..cos_sin_size {+ cosines[i] = T::from(((i as f32) * 0.05).cos()).unwrap();+ sines[i] = T::from(((i as f32) * 0.05).sin()).unwrap();+ }++ let mut token_positions = vec![0i32; suffix_length as usize];+ for i in 0..suffix_length as usize {+ token_positions[i] = i as i32;+ }++ Input {+ qkv: qkv.into_boxed_slice(),+ cosines: cosines.into_boxed_slice(),+ sines: sines.into_boxed_slice(),+ token_positions: token_positions.into_boxed_slice(),+ head_dim,+ num_heads,+ num_groups,+ suffix_length,+ max_sequence_length,+ }+}++fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {+ let context = B::Context::new().expect("Failed to create Context");++ let kernel = <<B as Backend>::Kernels as Kernels>::RopeKernel::new(&context, T::data_type())+ .expect("Failed to create RopeKernel");++ let total_heads = (input.num_heads + 2 * input.num_groups) as usize;+ let qkv_len = input.suffix_length as usize * total_heads * input.head_dim as usize;+ let cos_sin_len = input.max_sequence_length as usize * input.head_dim as usize;+ let queries_len = input.num_heads as usize * input.suffix_length as usize * input.head_dim as usize;+ let keys_len = input.num_groups as usize * input.suffix_length as usize * input.head_dim as usize;++ let qkv_array = context.create_array_from(&[qkv_len], &input.qkv, "");+ let cosines_array = context.create_array_from(&[cos_sin_len], &input.cosines, "");+ let sines_array = context.create_array_from(&[cos_sin_len], &input.sines, "");+ let token_positions_array = context.create_array_from(&[input.suffix_length as usize], &input.token_positions, "");+ let rotated_queries_array = context.create_array_uninitialized(&[queries_len], T::data_type(), "");+ let rotated_keys_array = context.create_array_uninitialized(&[keys_len], T::data_type(), "");++ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(+ qkv_array.buffer().borrow().deref(),+ cosines_array.buffer().borrow().deref(),+ sines_array.buffer().borrow().deref(),+ token_positions_array.buffer().borrow().deref(),+ rotated_queries_array.buffer().borrow_mut().deref_mut(),+ rotated_keys_array.buffer().borrow_mut().deref_mut(),+ input.head_dim,+ input.num_heads,+ input.num_groups,+ input.suffix_length,+ input.max_sequence_length,+ &mut command_buffer,+ );+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();++ (rotated_queries_array.as_slice().to_vec(), rotated_keys_array.as_slice().to_vec())+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(input: &Input<T>) {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.02f32+ } else {+ 1e-5+ };++ let (expected_queries, expected_keys) = get_output::<T, Cpu>(input);++ for_each_backend!(|B| {+ let (queries, keys) = get_output::<T, B>(input);+ let msg = format!("Rope queries test failed with backend={}", std::any::type_name::<B>(),);+ assert_eq_float::<T>(&expected_queries, &queries, eps, &msg);++ let msg = format!("Rope keys test failed with backend={}", std::any::type_name::<B>(),);+ assert_eq_float::<T>(&expected_keys, &keys, eps, &msg);+ });+}++fn test_basic<T: ArrayElement + Float + Debug + Display>() {+ // Typical GQA config: 32 query heads, 8 KV groups, head_dim=128, 4 tokens+ let input = get_test_data::<T>(32, 8, 128, 4, 512);+ test_internal(&input);+}++fn test_mha<T: ArrayElement + Float + Debug + Display>() {+ // MHA: num_heads == num_groups+ let input = get_test_data::<T>(8, 8, 64, 2, 256);+ test_internal(&input);+}++fn test_single_token<T: ArrayElement + Float + Debug + Display>() {+ // Single token (decode)+ let input = get_test_data::<T>(16, 4, 64, 1, 1024);+ test_internal(&input);+}++fn test_small<T: ArrayElement + Float + Debug + Display>() {+ // Minimal config+ let input = get_test_data::<T>(2, 1, 4, 1, 8);+ test_internal(&input);+}++fn test_nonzero_positions<T: ArrayElement + Float + Debug + Display>() {+ let mut input = get_test_data::<T>(4, 2, 8, 3, 64);+ input.token_positions = vec![10, 11, 12].into_boxed_slice();+ test_internal(&input);+}++// f32 tests+#[test]+fn test_basic_f32() {+ test_basic::<f32>();+}++#[test]+fn test_mha_f32() {+ test_mha::<f32>();+}++#[test]+fn test_single_token_f32() {+ test_single_token::<f32>();+}++#[test]+fn test_small_f32() {+ test_small::<f32>();+}++#[test]+fn test_nonzero_positions_f32() {+ test_nonzero_positions::<f32>();+}++// f16 tests+#[test]+fn test_basic_f16() {+ test_basic::<f16>();+}++#[test]+fn test_mha_f16() {+ test_mha::<f16>();+}++#[test]+fn test_single_token_f16() {+ test_single_token::<f16>();+}++#[test]+fn test_small_f16() {+ test_small::<f16>();+}++// bf16 tests+#[test]+fn test_basic_bf16() {+ test_basic::<bf16>();+}++#[test]+fn test_mha_bf16() {+ test_mha::<bf16>();+}++#[test]+fn test_single_token_bf16() {+ test_single_token::<bf16>();+}++#[test]+fn test_small_bf16() {+ test_small::<bf16>();+}