Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Add rope.rs (#240)

Aleksandr Golokoz62a812f9d04e
diff --git a/crates/uzu/src/backends/cpu/kernel/rope/rope.rs b/crates/uzu/src/backends/cpu/kernel/rope/rope.rs
index ea5da869f..21e0c84bd 100644
--- a/crates/uzu/src/backends/cpu/kernel/rope/rope.rs
+++ b/crates/uzu/src/backends/cpu/kernel/rope/rope.rs
@@ -4,20 +4,94 @@ use num_traits::Float;
use crate::ArrayElement;
+fn get_paired_val<T: ArrayElement + Float>(
+ values: *const T,
+ dim_index: usize,
+ half_dim: usize,
+ token_index: usize,
+ total_heads: usize,
+ head_dim: usize,
+ head_index: usize,
+) -> T {
+ if dim_index < half_dim {
+ let v =
+ unsafe { *values.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index + half_dim) };
+ T::zero() - v
+ } else {
+ unsafe { *values.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index - half_dim) }
+ }
+}
+
#[kernel(Rope)]
#[variants(T, f32, f16, bf16)]
pub fn rope<T: ArrayElement + Float>(
- #[allow(unused)] qkv: *const T,
- #[allow(unused)] cosines: *const T,
- #[allow(unused)] sines: *const T,
- #[allow(unused)] token_positions: *const i32,
- #[allow(unused)] rotated_queries: *mut T,
- #[allow(unused)] rotated_keys: *mut T,
- #[allow(unused)] head_dim: u32,
- #[allow(unused)] num_heads: u32,
- #[allow(unused)] num_groups: u32,
- #[allow(unused)] suffix_length: u32,
- #[allow(unused)] max_sequence_length: u32,
+ qkv: *const T,
+ cosines: *const T,
+ sines: *const T,
+ token_positions: *const i32,
+ rotated_queries: *mut T,
+ rotated_keys: *mut T,
+ head_dim: u32,
+ num_heads: u32,
+ num_groups: u32,
+ suffix_length: u32,
+ max_sequence_length: u32,
) {
- todo!()
+ if head_dim & 1 != 0 || num_heads % num_groups != 0 {
+ return;
+ }
+
+ let head_dim = head_dim as usize;
+ let num_heads = num_heads as usize;
+ let num_groups = num_groups as usize;
+ let suffix_length = suffix_length as usize;
+ let max_sequence_length = max_sequence_length as usize;
+
+ let half_dim = head_dim / 2;
+ let total_heads = num_heads + 2 * num_groups;
+ let heads_per_group = num_heads / num_groups;
+
+ for head_index in 0..num_heads {
+ let group_index = head_index / heads_per_group;
+
+ for token_index in 0..suffix_length {
+ let raw_position = unsafe { *token_positions.add(token_index) } as usize;
+ let absolute_position = if raw_position >= max_sequence_length {
+ 0
+ } else {
+ raw_position
+ };
+
+ for dim_index in 0..head_dim {
+ let cos_val = unsafe { *cosines.add(absolute_position * head_dim + dim_index) };
+ let sin_val = unsafe { *sines.add(absolute_position * head_dim + dim_index) };
+
+ // Query rotation
+ let q_val =
+ unsafe { *qkv.add(token_index * total_heads * head_dim + head_index * head_dim + dim_index) };
+ let q_paired = get_paired_val(qkv, dim_index, half_dim, token_index, total_heads, head_dim, head_index);
+ let q_result = q_val * cos_val + q_paired * sin_val;
+ unsafe {
+ *rotated_queries.add(head_index * suffix_length * head_dim + token_index * head_dim + dim_index) =
+ q_result;
+ }
+
+ // Key rotation (only first head in each group)
+ if head_index == group_index * heads_per_group {
+ let key_head_index = num_heads + group_index;
+ let k_val = unsafe {
+ *qkv.add(token_index * total_heads * head_dim + key_head_index * head_dim + dim_index)
+ };
+ let k_paired =
+ get_paired_val(qkv, dim_index, half_dim, token_index, total_heads, head_dim, key_head_index);
+ let k_result = k_val * cos_val + k_paired * sin_val;
+ unsafe {
+ *rotated_keys
+ .add(group_index * suffix_length * head_dim + token_index * head_dim + dim_index) =
+ k_result;
+ }
+ }
+ }
+ }
+ }
}
diff --git a/crates/uzu/tests/kernel/mod.rs b/crates/uzu/tests/kernel/mod.rs
index fb030b664..455478a42 100644
--- a/crates/uzu/tests/kernel/mod.rs
+++ b/crates/uzu/tests/kernel/mod.rs
@@ -5,6 +5,7 @@ mod common;
mod full_precision_embedding_test;
mod qk_norm_test;
mod rms_norm_test;
+mod rope_test;
mod tensor_add_bias_test;
mod tensor_add_swap_test;
mod tensor_copy_test;
diff --git a/crates/uzu/tests/kernel/rope_test.rs b/crates/uzu/tests/kernel/rope_test.rs
new file mode 100644
index 000000000..be119b7bb
--- /dev/null
+++ b/crates/uzu/tests/kernel/rope_test.rs
@@ -0,0 +1,229 @@
+use std::{
+ fmt::{Debug, Display},
+ ops::{Deref, DerefMut},
+};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayElement, DataType,
+ array::ArrayContextExt,
+ backends::{
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels, kernel::RopeKernel,
+ },
+ cpu::Cpu,
+ },
+};
+
+use crate::common::assert::assert_eq_float;
+
+struct Input<T: ArrayElement + Float> {
+ qkv: Box<[T]>,
+ cosines: Box<[T]>,
+ sines: Box<[T]>,
+ token_positions: Box<[i32]>,
+ head_dim: u32,
+ num_heads: u32,
+ num_groups: u32,
+ suffix_length: u32,
+ max_sequence_length: u32,
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ num_heads: u32,
+ num_groups: u32,
+ head_dim: u32,
+ suffix_length: u32,
+ max_sequence_length: u32,
+) -> Input<T> {
+ let total_heads = (num_heads + 2 * num_groups) as usize;
+ let qkv_size = suffix_length as usize * total_heads * head_dim as usize;
+ let cos_sin_size = max_sequence_length as usize * head_dim as usize;
+
+ let mut qkv = vec![T::zero(); qkv_size];
+ for i in 0..qkv_size {
+ qkv[i] = T::from(((i as f32) * 0.1).sin() * 2.0).unwrap();
+ }
+
+ let mut cosines = vec![T::zero(); cos_sin_size];
+ let mut sines = vec![T::zero(); cos_sin_size];
+ for i in 0..cos_sin_size {
+ cosines[i] = T::from(((i as f32) * 0.05).cos()).unwrap();
+ sines[i] = T::from(((i as f32) * 0.05).sin()).unwrap();
+ }
+
+ let mut token_positions = vec![0i32; suffix_length as usize];
+ for i in 0..suffix_length as usize {
+ token_positions[i] = i as i32;
+ }
+
+ Input {
+ qkv: qkv.into_boxed_slice(),
+ cosines: cosines.into_boxed_slice(),
+ sines: sines.into_boxed_slice(),
+ token_positions: token_positions.into_boxed_slice(),
+ head_dim,
+ num_heads,
+ num_groups,
+ suffix_length,
+ max_sequence_length,
+ }
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>, Vec<T>) {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::RopeKernel::new(&context, T::data_type())
+ .expect("Failed to create RopeKernel");
+
+ let total_heads = (input.num_heads + 2 * input.num_groups) as usize;
+ let qkv_len = input.suffix_length as usize * total_heads * input.head_dim as usize;
+ let cos_sin_len = input.max_sequence_length as usize * input.head_dim as usize;
+ let queries_len = input.num_heads as usize * input.suffix_length as usize * input.head_dim as usize;
+ let keys_len = input.num_groups as usize * input.suffix_length as usize * input.head_dim as usize;
+
+ let qkv_array = context.create_array_from(&[qkv_len], &input.qkv, "");
+ let cosines_array = context.create_array_from(&[cos_sin_len], &input.cosines, "");
+ let sines_array = context.create_array_from(&[cos_sin_len], &input.sines, "");
+ let token_positions_array = context.create_array_from(&[input.suffix_length as usize], &input.token_positions, "");
+ let rotated_queries_array = context.create_array_uninitialized(&[queries_len], T::data_type(), "");
+ let rotated_keys_array = context.create_array_uninitialized(&[keys_len], T::data_type(), "");
+
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(
+ qkv_array.buffer().borrow().deref(),
+ cosines_array.buffer().borrow().deref(),
+ sines_array.buffer().borrow().deref(),
+ token_positions_array.buffer().borrow().deref(),
+ rotated_queries_array.buffer().borrow_mut().deref_mut(),
+ rotated_keys_array.buffer().borrow_mut().deref_mut(),
+ input.head_dim,
+ input.num_heads,
+ input.num_groups,
+ input.suffix_length,
+ input.max_sequence_length,
+ &mut command_buffer,
+ );
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+
+ (rotated_queries_array.as_slice().to_vec(), rotated_keys_array.as_slice().to_vec())
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(input: &Input<T>) {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.02f32
+ } else {
+ 1e-5
+ };
+
+ let (expected_queries, expected_keys) = get_output::<T, Cpu>(input);
+
+ for_each_backend!(|B| {
+ let (queries, keys) = get_output::<T, B>(input);
+ let msg = format!("Rope queries test failed with backend={}", std::any::type_name::<B>(),);
+ assert_eq_float::<T>(&expected_queries, &queries, eps, &msg);
+
+ let msg = format!("Rope keys test failed with backend={}", std::any::type_name::<B>(),);
+ assert_eq_float::<T>(&expected_keys, &keys, eps, &msg);
+ });
+}
+
+fn test_basic<T: ArrayElement + Float + Debug + Display>() {
+ // Typical GQA config: 32 query heads, 8 KV groups, head_dim=128, 4 tokens
+ let input = get_test_data::<T>(32, 8, 128, 4, 512);
+ test_internal(&input);
+}
+
+fn test_mha<T: ArrayElement + Float + Debug + Display>() {
+ // MHA: num_heads == num_groups
+ let input = get_test_data::<T>(8, 8, 64, 2, 256);
+ test_internal(&input);
+}
+
+fn test_single_token<T: ArrayElement + Float + Debug + Display>() {
+ // Single token (decode)
+ let input = get_test_data::<T>(16, 4, 64, 1, 1024);
+ test_internal(&input);
+}
+
+fn test_small<T: ArrayElement + Float + Debug + Display>() {
+ // Minimal config
+ let input = get_test_data::<T>(2, 1, 4, 1, 8);
+ test_internal(&input);
+}
+
+fn test_nonzero_positions<T: ArrayElement + Float + Debug + Display>() {
+ let mut input = get_test_data::<T>(4, 2, 8, 3, 64);
+ input.token_positions = vec![10, 11, 12].into_boxed_slice();
+ test_internal(&input);
+}
+
+// f32 tests
+#[test]
+fn test_basic_f32() {
+ test_basic::<f32>();
+}
+
+#[test]
+fn test_mha_f32() {
+ test_mha::<f32>();
+}
+
+#[test]
+fn test_single_token_f32() {
+ test_single_token::<f32>();
+}
+
+#[test]
+fn test_small_f32() {
+ test_small::<f32>();
+}
+
+#[test]
+fn test_nonzero_positions_f32() {
+ test_nonzero_positions::<f32>();
+}
+
+// f16 tests
+#[test]
+fn test_basic_f16() {
+ test_basic::<f16>();
+}
+
+#[test]
+fn test_mha_f16() {
+ test_mha::<f16>();
+}
+
+#[test]
+fn test_single_token_f16() {
+ test_single_token::<f16>();
+}
+
+#[test]
+fn test_small_f16() {
+ test_small::<f16>();
+}
+
+// bf16 tests
+#[test]
+fn test_basic_bf16() {
+ test_basic::<bf16>();
+}
+
+#[test]
+fn test_mha_bf16() {
+ test_mha::<bf16>();
+}
+
+#[test]
+fn test_single_token_bf16() {
+ test_single_token::<bf16>();
+}
+
+#[test]
+fn test_small_bf16() {
+ test_small::<bf16>();
+}