A high-performance inference engine for AI models
Add Hadamard output to CPU embedding and normalization (#866)
pawan690d958ec888
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/uzu-engine/src/backends/cpu/kernel/embedding/quant_embedding.rsindex 16a31203b..8a99ab825 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/embedding/quant_embedding.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/embedding/quant_embedding.rs@@ -4,7 +4,10 @@ use uzu_engine_macros::kernel;use crate::{array::ArrayElement,- backends::common::gpu_types::{QuantizationMethod, QuantizationMode},+ backends::{+ common::gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod, QuantizationMode},+ cpu::kernel::activation_transform::hadamard_transform,+ },data_type::DataType,};@@ -17,9 +20,7 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(#[optional(quantization_method == QuantizationMethod::ScaleZeroPoint)] zero_points: Option<*const u8>,#[optional(quantization_method == QuantizationMethod::ScaleBias)] biases: Option<*const T>,output: *mut T,- #[allow(unused)]- #[optional(use_hadamard)]- output_hadamard_factors: Option<*const i32>,+ #[optional(use_hadamard)] output_hadamard_factors: Option<*const i32>,batch_size: u32,vocab_size: u32,model_dim: u32,@@ -29,9 +30,8 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(#[specialize] quantization_method: QuantizationMethod,#[specialize] use_hadamard: bool,) {- if use_hadamard {- unimplemented!("not supported yet");- }+ assert_eq!(output_hadamard_factors.is_some(), use_hadamard);+ assert!(!use_hadamard || model_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));let packing_divisor = quantization_mode.packing_divisor();let weights_stride = model_dim / packing_divisor;@@ -112,6 +112,20 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(let out_f = out_f * input_scale;*output.add(out_idx) = T::from(out_f).unwrap();}++ // Output RHT like the Metal kernel: round to T, transform each block, then apply the factors+ if let Some(factors) = output_hadamard_factors {+ let row =+ std::slice::from_raw_parts_mut(output.add((batch_idx * model_dim) as usize), model_dim as usize);+ for block_start in (0..model_dim as usize).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE as usize) {+ let mut block = std::array::from_fn(|lane| row[block_start + lane].to_f32().unwrap());+ hadamard_transform(&mut block);+ for (lane, value) in block.into_iter().enumerate() {+ let factor = *factors.add(block_start + lane) as f32;+ row[block_start + lane] = T::from(value * factor).unwrap();+ }+ }+ }}}}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rsindex 0a7c8ffe3..c4956c05f 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs@@ -2,7 +2,12 @@ use half::{bf16, f16};use num_traits::Float;use uzu_engine_macros::kernel;-use crate::array::ArrayElement;+use crate::{+ array::ArrayElement,+ backends::{+ common::gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE, cpu::kernel::activation_transform::hadamard_transform,+ },+};#[kernel(Normalization)]#[variants(InputT, f32, f16, bf16)]@@ -42,10 +47,7 @@ pub fn normalization<assert_eq!(biases.is_some(), has_biases);assert_eq!(scales.is_some(), has_scales);assert!(copy_to_shortcut || !residual_add);-- if use_hadamard {- unimplemented!("not supported yet");- }+ assert!(!use_hadamard || element_count.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));let input = match in_place {true => output as *const InputT,@@ -56,6 +58,7 @@ pub fn normalization<let epsilon = AccumT::from(epsilon).unwrap();let scale_offset = AccumT::from(scale_offset).unwrap();let element_count_accum = AccumT::from(element_count).unwrap();+ let mut row = vec![OutputT::zero(); element_count];for batch in 0..(batch_size as usize) {let batch_offset = batch * element_count;@@ -117,6 +120,25 @@ pub fn normalization<let bias = unsafe { AccumT::from(*biases.unwrap().add(i)).unwrap() };result = OutputT::from(AccumT::from(result).unwrap() + bias).unwrap();}+ row[i] = result;+ }++ // Input RHT like the Metal kernel: apply the factors, then transform each block, before output scaling+ if let Some(factors) = hadamard_factors {+ for block_start in (0..element_count).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE as usize) {+ let mut block = std::array::from_fn(|lane| {+ let factor = unsafe { *factors.add(block_start + lane) } as f32;+ row[block_start + lane].to_f32().unwrap() * factor+ });+ hadamard_transform(&mut block);+ for (lane, value) in block.into_iter().enumerate() {+ row[block_start + lane] = OutputT::from(value).unwrap();+ }+ }+ }++ for (i, &value) in row.iter().enumerate() {+ let mut result = value;if scale_output {result = result * OutputT::from(post_layer_scalar).unwrap();}diff --git a/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rs b/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rsindex c67e6c598..f18c273e7 100644--- a/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rs@@ -8,16 +8,18 @@ use crate::{array::ArrayElement,backends::{common::{- Allocation, Backend, Context, Encoder, Kernels,+ Backend, Context, Encoder, Kernels,gpu_types::{QuantizationMethod, QuantizationMode},- kernel::QuantizedEmbeddingLookupKernel,+ kernel::{ActivationTransform, QuantizedEmbeddingLookupKernel},},cpu::Cpu,},data_type::DataType,tests::{assert::assert_eq_float,- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},+ helpers::{+ alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_backend, for_each_non_cpu_backend,+ },},};@@ -197,6 +199,13 @@ fn get_test_data_oob<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {}fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {+ get_output_with_hadamard::<T, B>(input, None)+}++fn get_output_with_hadamard<T: ArrayElement + Float, B: Backend>(+ input: &Input<T>,+ hadamard_factors: Option<&[i32]>,+) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(@@ -205,7 +214,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {input.group_size,input.quant_mode,input.quant_method,- false,+ hadamard_factors.is_some(),).expect("Failed to create QuantizedEmbeddingLookupKernel");@@ -215,6 +224,8 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let zero_points_allocation =input.zero_points.as_ref().map(|zero_points| alloc_allocation_with_data::<B, u8>(&context, zero_points));let biases_allocation = input.biases.as_ref().map(|biases| alloc_allocation_with_data::<B, T>(&context, biases));+ let hadamard_factors_allocation =+ hadamard_factors.map(|hadamard_factors| alloc_allocation_with_data::<B, i32>(&context, hadamard_factors));let mut output = alloc_allocation::<B, T>(&context, input.batch_size as usize * input.model_dim as usize);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");@@ -225,7 +236,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {zero_points_allocation.as_ref(),biases_allocation.as_ref(),&mut output,- None::<&Allocation<B>>,+ hadamard_factors_allocation.as_ref(),input.batch_size,input.vocab_size,input.model_dim,@@ -292,6 +303,56 @@ fn test_zero_point_group16_hadamard_constructor<T: ArrayElement + Float + Debug.expect("QuantizedEmbeddingLookupKernel group16 zero-point hadamard");}+// The fused transform has to match the plain lookup followed by the standalone output RHT+fn test_zero_point_group16_hadamard<T: ArrayElement + Float + Debug + Display>() {+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 0.5f32+ } else {+ 1e-4+ };+ let (input, plain) = get_test_data_zero_point_group16::<T>();+ let hadamard_factors: Vec<i32> = (0..input.model_dim)+ .map(|index| {+ if index % 3 == 0 {+ -1+ } else {+ 1+ }+ })+ .collect();++ let context = <Cpu as Backend>::Context::new().expect("Failed to create Context");+ let output_rht = ActivationTransform::<Cpu>::output_rht(context.as_ref(), T::data_type(), false)+ .expect("Failed to create ActivationTransform");+ let plain_allocation = alloc_allocation_with_data::<Cpu, T>(&context, &plain);+ let hadamard_factors_allocation = alloc_allocation_with_data::<Cpu, i32>(&context, &hadamard_factors);+ let mut expected_allocation = alloc_allocation::<Cpu, T>(&context, plain.len());+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ output_rht.encode_fp(+ &plain_allocation,+ &mut expected_allocation,+ &hadamard_factors_allocation,+ input.batch_size,+ input.model_dim,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ let expected = allocation_to_vec::<Cpu, T>(&expected_allocation);++ for_each_backend!(|B| {+ let output = get_output_with_hadamard::<T, B>(&input, Some(&hadamard_factors));+ assert_eq_float::<T>(+ &expected,+ &output,+ eps,+ &format!(+ "QuantizedEmbeddingLookup ScaleZeroPoint group16 hadamard test failed for backend {}",+ std::any::type_name::<B>()+ ),+ );+ });+}+fn test_quant_mode<T: ArrayElement + Float + Debug + Display>(quant_mode: QuantizationMode) {let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {0.5f32@@ -355,6 +416,16 @@ fn test_uint4_zero_point_group16_hadamard_bf16_constructor() {test_zero_point_group16_hadamard_constructor::<bf16>();}+#[uzu_test]+fn test_uint4_zero_point_group16_hadamard_f32() {+ test_zero_point_group16_hadamard::<f32>();+}++#[uzu_test]+fn test_uint4_zero_point_group16_hadamard_bf16() {+ test_zero_point_group16_hadamard::<bf16>();+}+// INT8 tests#[uzu_test]fn test_int8_f32() {diff --git a/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rs b/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rsindex 0924e8a94..8955d6185 100644--- a/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rs@@ -7,13 +7,18 @@ use uzu_engine_macros::uzu_test;use crate::{array::ArrayElement,backends::{- common::{Allocation, Backend, Context, Encoder, Kernels, kernel::NormalizationKernel},+ common::{+ Allocation, Backend, Context, Encoder, Kernels,+ kernel::{ActivationTransform, NormalizationKernel},+ },cpu::Cpu,},data_type::DataType,tests::{assert::assert_eq_float,- helpers::{alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},+ helpers::{+ alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_backend, for_each_non_cpu_backend,+ },},};@@ -29,6 +34,7 @@ fn get_output<element_count: u32,epsilon: f32,full_layer: bool,+ hadamard_factors: Option<&[i32]>,) -> Vec<OutputT> {let context = B::Context::new().expect("Failed to create Context");let kernel = <<B as Backend>::Kernels as Kernels>::NormalizationKernel::new(@@ -42,7 +48,7 @@ fn get_output<full_layer,false,false,- false,+ hadamard_factors.is_some(),false,false,false,@@ -52,6 +58,8 @@ fn get_output<let input_allocation = alloc_allocation_with_data::<B, InputT>(&context, input);let scales_allocation = scales.map(|scales| alloc_allocation_with_data::<B, AffineT>(&context, scales));+ let hadamard_factors_allocation =+ hadamard_factors.map(|hadamard_factors| alloc_allocation_with_data::<B, i32>(&context, hadamard_factors));let mut output_allocation = alloc_allocation_with_data::<B, OutputT>(&context, &vec![OutputT::zero(); input.len()]);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");@@ -61,7 +69,7 @@ fn get_output<None::<&Allocation<B>>,&mut output_allocation,None::<(&mut Allocation<B>, usize)>,- None::<&Allocation<B>>,+ hadamard_factors_allocation.as_ref(),batch_size,element_count,epsilon,@@ -99,6 +107,7 @@ fn test_internal<element_count,epsilon,full_layer,+ None,);let eps = if matches!(InputT::data_type(), DataType::F16 | DataType::BF16)@@ -118,6 +127,7 @@ fn test_internal<element_count,epsilon,full_layer,+ None,);let message = format!("Normalization kernel test failed with backend={}, has_scales={}, full_layer={}",@@ -141,6 +151,76 @@ fn test_normalization<}}+// The fused transform has to match plain normalization followed by the standalone input RHT+fn test_hadamard<T: ArrayElement + Float + Debug + Display>() {+ let batch_size = 2u32;+ let element_count = 64u32;+ let epsilon = 1e-6f32;++ let input: Vec<T> =+ (0..(batch_size * element_count)).map(|index| T::from(0.5f32 + (index as f32) * 0.01).unwrap()).collect();+ let scales: Vec<T> = (0..element_count).map(|index| T::from(1.0f32 + (index as f32) * 0.001).unwrap()).collect();+ let hadamard_factors: Vec<i32> = (0..element_count)+ .map(|index| {+ if index % 3 == 0 {+ -1+ } else {+ 1+ }+ })+ .collect();++ let plain = get_output::<Cpu, T, T, T>(&input, Some(&scales), batch_size, element_count, epsilon, true, None);++ let context = <Cpu as Backend>::Context::new().expect("Failed to create Context");+ let input_rht = ActivationTransform::<Cpu>::input_rht(context.as_ref(), T::data_type(), false)+ .expect("Failed to create ActivationTransform");+ let plain_allocation = alloc_allocation_with_data::<Cpu, T>(&context, &plain);+ let hadamard_factors_allocation = alloc_allocation_with_data::<Cpu, i32>(&context, &hadamard_factors);+ let mut expected_allocation = alloc_allocation::<Cpu, T>(&context, plain.len());+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ input_rht.encode_fp(+ &plain_allocation,+ &mut expected_allocation,+ &hadamard_factors_allocation,+ batch_size,+ element_count,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ let expected = allocation_to_vec::<Cpu, T>(&expected_allocation);++ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ 1e-2+ } else {+ 1e-5+ };++ for_each_backend!(|B| {+ let actual = get_output::<B, T, T, T>(+ &input,+ Some(&scales),+ batch_size,+ element_count,+ epsilon,+ true,+ Some(&hadamard_factors),+ );+ let message = format!("Normalization hadamard kernel test failed with backend={}", std::any::type_name::<B>());+ assert_eq_float::<T>(&expected, &actual, eps, &message);+ });+}++#[uzu_test]+fn test_normalization_hadamard_f32() {+ test_hadamard::<f32>();+}++#[uzu_test]+fn test_normalization_hadamard_bf16() {+ test_hadamard::<bf16>();+}+#[uzu_test]fn test_normalization_f32_f32_f32() {test_normalization::<f32, f32, f32>();