Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Add Hadamard output to CPU embedding and normalization (#866)

pawan690d958ec888
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/uzu-engine/src/backends/cpu/kernel/embedding/quant_embedding.rs
index 16a31203b..8a99ab825 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/embedding/quant_embedding.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/embedding/quant_embedding.rs
@@ -4,7 +4,10 @@ use uzu_engine_macros::kernel;
use crate::{
array::ArrayElement,
- backends::common::gpu_types::{QuantizationMethod, QuantizationMode},
+ backends::{
+ common::gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod, QuantizationMode},
+ cpu::kernel::activation_transform::hadamard_transform,
+ },
data_type::DataType,
};
@@ -17,9 +20,7 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
#[optional(quantization_method == QuantizationMethod::ScaleZeroPoint)] zero_points: Option<*const u8>,
#[optional(quantization_method == QuantizationMethod::ScaleBias)] biases: Option<*const T>,
output: *mut T,
- #[allow(unused)]
- #[optional(use_hadamard)]
- output_hadamard_factors: Option<*const i32>,
+ #[optional(use_hadamard)] output_hadamard_factors: Option<*const i32>,
batch_size: u32,
vocab_size: u32,
model_dim: u32,
@@ -29,9 +30,8 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
#[specialize] quantization_method: QuantizationMethod,
#[specialize] use_hadamard: bool,
) {
- if use_hadamard {
- unimplemented!("not supported yet");
- }
+ assert_eq!(output_hadamard_factors.is_some(), use_hadamard);
+ assert!(!use_hadamard || model_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));
let packing_divisor = quantization_mode.packing_divisor();
let weights_stride = model_dim / packing_divisor;
@@ -112,6 +112,20 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
let out_f = out_f * input_scale;
*output.add(out_idx) = T::from(out_f).unwrap();
}
+
+ // Output RHT like the Metal kernel: round to T, transform each block, then apply the factors
+ if let Some(factors) = output_hadamard_factors {
+ let row =
+ std::slice::from_raw_parts_mut(output.add((batch_idx * model_dim) as usize), model_dim as usize);
+ for block_start in (0..model_dim as usize).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE as usize) {
+ let mut block = std::array::from_fn(|lane| row[block_start + lane].to_f32().unwrap());
+ hadamard_transform(&mut block);
+ for (lane, value) in block.into_iter().enumerate() {
+ let factor = *factors.add(block_start + lane) as f32;
+ row[block_start + lane] = T::from(value * factor).unwrap();
+ }
+ }
+ }
}
}
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs
index 0a7c8ffe3..c4956c05f 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs
@@ -2,7 +2,12 @@ use half::{bf16, f16};
use num_traits::Float;
use uzu_engine_macros::kernel;
-use crate::array::ArrayElement;
+use crate::{
+ array::ArrayElement,
+ backends::{
+ common::gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE, cpu::kernel::activation_transform::hadamard_transform,
+ },
+};
#[kernel(Normalization)]
#[variants(InputT, f32, f16, bf16)]
@@ -42,10 +47,7 @@ pub fn normalization<
assert_eq!(biases.is_some(), has_biases);
assert_eq!(scales.is_some(), has_scales);
assert!(copy_to_shortcut || !residual_add);
-
- if use_hadamard {
- unimplemented!("not supported yet");
- }
+ assert!(!use_hadamard || element_count.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));
let input = match in_place {
true => output as *const InputT,
@@ -56,6 +58,7 @@ pub fn normalization<
let epsilon = AccumT::from(epsilon).unwrap();
let scale_offset = AccumT::from(scale_offset).unwrap();
let element_count_accum = AccumT::from(element_count).unwrap();
+ let mut row = vec![OutputT::zero(); element_count];
for batch in 0..(batch_size as usize) {
let batch_offset = batch * element_count;
@@ -117,6 +120,25 @@ pub fn normalization<
let bias = unsafe { AccumT::from(*biases.unwrap().add(i)).unwrap() };
result = OutputT::from(AccumT::from(result).unwrap() + bias).unwrap();
}
+ row[i] = result;
+ }
+
+ // Input RHT like the Metal kernel: apply the factors, then transform each block, before output scaling
+ if let Some(factors) = hadamard_factors {
+ for block_start in (0..element_count).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE as usize) {
+ let mut block = std::array::from_fn(|lane| {
+ let factor = unsafe { *factors.add(block_start + lane) } as f32;
+ row[block_start + lane].to_f32().unwrap() * factor
+ });
+ hadamard_transform(&mut block);
+ for (lane, value) in block.into_iter().enumerate() {
+ row[block_start + lane] = OutputT::from(value).unwrap();
+ }
+ }
+ }
+
+ for (i, &value) in row.iter().enumerate() {
+ let mut result = value;
if scale_output {
result = result * OutputT::from(post_layer_scalar).unwrap();
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rs b/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rs
index c67e6c598..f18c273e7 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/embedding/quant_embedding_test.rs
@@ -8,16 +8,18 @@ use crate::{
array::ArrayElement,
backends::{
common::{
- Allocation, Backend, Context, Encoder, Kernels,
+ Backend, Context, Encoder, Kernels,
gpu_types::{QuantizationMethod, QuantizationMode},
- kernel::QuantizedEmbeddingLookupKernel,
+ kernel::{ActivationTransform, QuantizedEmbeddingLookupKernel},
},
cpu::Cpu,
},
data_type::DataType,
tests::{
assert::assert_eq_float,
- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},
+ helpers::{
+ alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_backend, for_each_non_cpu_backend,
+ },
},
};
@@ -197,6 +199,13 @@ fn get_test_data_oob<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
}
fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
+ get_output_with_hadamard::<T, B>(input, None)
+}
+
+fn get_output_with_hadamard<T: ArrayElement + Float, B: Backend>(
+ input: &Input<T>,
+ hadamard_factors: Option<&[i32]>,
+) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
@@ -205,7 +214,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
input.group_size,
input.quant_mode,
input.quant_method,
- false,
+ hadamard_factors.is_some(),
)
.expect("Failed to create QuantizedEmbeddingLookupKernel");
@@ -215,6 +224,8 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let zero_points_allocation =
input.zero_points.as_ref().map(|zero_points| alloc_allocation_with_data::<B, u8>(&context, zero_points));
let biases_allocation = input.biases.as_ref().map(|biases| alloc_allocation_with_data::<B, T>(&context, biases));
+ let hadamard_factors_allocation =
+ hadamard_factors.map(|hadamard_factors| alloc_allocation_with_data::<B, i32>(&context, hadamard_factors));
let mut output = alloc_allocation::<B, T>(&context, input.batch_size as usize * input.model_dim as usize);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
@@ -225,7 +236,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
zero_points_allocation.as_ref(),
biases_allocation.as_ref(),
&mut output,
- None::<&Allocation<B>>,
+ hadamard_factors_allocation.as_ref(),
input.batch_size,
input.vocab_size,
input.model_dim,
@@ -292,6 +303,56 @@ fn test_zero_point_group16_hadamard_constructor<T: ArrayElement + Float + Debug
.expect("QuantizedEmbeddingLookupKernel group16 zero-point hadamard");
}
+// The fused transform has to match the plain lookup followed by the standalone output RHT
+fn test_zero_point_group16_hadamard<T: ArrayElement + Float + Debug + Display>() {
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 0.5f32
+ } else {
+ 1e-4
+ };
+ let (input, plain) = get_test_data_zero_point_group16::<T>();
+ let hadamard_factors: Vec<i32> = (0..input.model_dim)
+ .map(|index| {
+ if index % 3 == 0 {
+ -1
+ } else {
+ 1
+ }
+ })
+ .collect();
+
+ let context = <Cpu as Backend>::Context::new().expect("Failed to create Context");
+ let output_rht = ActivationTransform::<Cpu>::output_rht(context.as_ref(), T::data_type(), false)
+ .expect("Failed to create ActivationTransform");
+ let plain_allocation = alloc_allocation_with_data::<Cpu, T>(&context, &plain);
+ let hadamard_factors_allocation = alloc_allocation_with_data::<Cpu, i32>(&context, &hadamard_factors);
+ let mut expected_allocation = alloc_allocation::<Cpu, T>(&context, plain.len());
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ output_rht.encode_fp(
+ &plain_allocation,
+ &mut expected_allocation,
+ &hadamard_factors_allocation,
+ input.batch_size,
+ input.model_dim,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let expected = allocation_to_vec::<Cpu, T>(&expected_allocation);
+
+ for_each_backend!(|B| {
+ let output = get_output_with_hadamard::<T, B>(&input, Some(&hadamard_factors));
+ assert_eq_float::<T>(
+ &expected,
+ &output,
+ eps,
+ &format!(
+ "QuantizedEmbeddingLookup ScaleZeroPoint group16 hadamard test failed for backend {}",
+ std::any::type_name::<B>()
+ ),
+ );
+ });
+}
+
fn test_quant_mode<T: ArrayElement + Float + Debug + Display>(quant_mode: QuantizationMode) {
let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
0.5f32
@@ -355,6 +416,16 @@ fn test_uint4_zero_point_group16_hadamard_bf16_constructor() {
test_zero_point_group16_hadamard_constructor::<bf16>();
}
+#[uzu_test]
+fn test_uint4_zero_point_group16_hadamard_f32() {
+ test_zero_point_group16_hadamard::<f32>();
+}
+
+#[uzu_test]
+fn test_uint4_zero_point_group16_hadamard_bf16() {
+ test_zero_point_group16_hadamard::<bf16>();
+}
+
// INT8 tests
#[uzu_test]
fn test_int8_f32() {
diff --git a/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rs b/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rs
index 0924e8a94..8955d6185 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/normalization_test.rs
@@ -7,13 +7,18 @@ use uzu_engine_macros::uzu_test;
use crate::{
array::ArrayElement,
backends::{
- common::{Allocation, Backend, Context, Encoder, Kernels, kernel::NormalizationKernel},
+ common::{
+ Allocation, Backend, Context, Encoder, Kernels,
+ kernel::{ActivationTransform, NormalizationKernel},
+ },
cpu::Cpu,
},
data_type::DataType,
tests::{
assert::assert_eq_float,
- helpers::{alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},
+ helpers::{
+ alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_backend, for_each_non_cpu_backend,
+ },
},
};
@@ -29,6 +34,7 @@ fn get_output<
element_count: u32,
epsilon: f32,
full_layer: bool,
+ hadamard_factors: Option<&[i32]>,
) -> Vec<OutputT> {
let context = B::Context::new().expect("Failed to create Context");
let kernel = <<B as Backend>::Kernels as Kernels>::NormalizationKernel::new(
@@ -42,7 +48,7 @@ fn get_output<
full_layer,
false,
false,
- false,
+ hadamard_factors.is_some(),
false,
false,
false,
@@ -52,6 +58,8 @@ fn get_output<
let input_allocation = alloc_allocation_with_data::<B, InputT>(&context, input);
let scales_allocation = scales.map(|scales| alloc_allocation_with_data::<B, AffineT>(&context, scales));
+ let hadamard_factors_allocation =
+ hadamard_factors.map(|hadamard_factors| alloc_allocation_with_data::<B, i32>(&context, hadamard_factors));
let mut output_allocation = alloc_allocation_with_data::<B, OutputT>(&context, &vec![OutputT::zero(); input.len()]);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
@@ -61,7 +69,7 @@ fn get_output<
None::<&Allocation<B>>,
&mut output_allocation,
None::<(&mut Allocation<B>, usize)>,
- None::<&Allocation<B>>,
+ hadamard_factors_allocation.as_ref(),
batch_size,
element_count,
epsilon,
@@ -99,6 +107,7 @@ fn test_internal<
element_count,
epsilon,
full_layer,
+ None,
);
let eps = if matches!(InputT::data_type(), DataType::F16 | DataType::BF16)
@@ -118,6 +127,7 @@ fn test_internal<
element_count,
epsilon,
full_layer,
+ None,
);
let message = format!(
"Normalization kernel test failed with backend={}, has_scales={}, full_layer={}",
@@ -141,6 +151,76 @@ fn test_normalization<
}
}
+// The fused transform has to match plain normalization followed by the standalone input RHT
+fn test_hadamard<T: ArrayElement + Float + Debug + Display>() {
+ let batch_size = 2u32;
+ let element_count = 64u32;
+ let epsilon = 1e-6f32;
+
+ let input: Vec<T> =
+ (0..(batch_size * element_count)).map(|index| T::from(0.5f32 + (index as f32) * 0.01).unwrap()).collect();
+ let scales: Vec<T> = (0..element_count).map(|index| T::from(1.0f32 + (index as f32) * 0.001).unwrap()).collect();
+ let hadamard_factors: Vec<i32> = (0..element_count)
+ .map(|index| {
+ if index % 3 == 0 {
+ -1
+ } else {
+ 1
+ }
+ })
+ .collect();
+
+ let plain = get_output::<Cpu, T, T, T>(&input, Some(&scales), batch_size, element_count, epsilon, true, None);
+
+ let context = <Cpu as Backend>::Context::new().expect("Failed to create Context");
+ let input_rht = ActivationTransform::<Cpu>::input_rht(context.as_ref(), T::data_type(), false)
+ .expect("Failed to create ActivationTransform");
+ let plain_allocation = alloc_allocation_with_data::<Cpu, T>(&context, &plain);
+ let hadamard_factors_allocation = alloc_allocation_with_data::<Cpu, i32>(&context, &hadamard_factors);
+ let mut expected_allocation = alloc_allocation::<Cpu, T>(&context, plain.len());
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ input_rht.encode_fp(
+ &plain_allocation,
+ &mut expected_allocation,
+ &hadamard_factors_allocation,
+ batch_size,
+ element_count,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ let expected = allocation_to_vec::<Cpu, T>(&expected_allocation);
+
+ let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ 1e-2
+ } else {
+ 1e-5
+ };
+
+ for_each_backend!(|B| {
+ let actual = get_output::<B, T, T, T>(
+ &input,
+ Some(&scales),
+ batch_size,
+ element_count,
+ epsilon,
+ true,
+ Some(&hadamard_factors),
+ );
+ let message = format!("Normalization hadamard kernel test failed with backend={}", std::any::type_name::<B>());
+ assert_eq_float::<T>(&expected, &actual, eps, &message);
+ });
+}
+
+#[uzu_test]
+fn test_normalization_hadamard_f32() {
+ test_hadamard::<f32>();
+}
+
+#[uzu_test]
+fn test_normalization_hadamard_bf16() {
+ test_hadamard::<bf16>();
+}
+
#[uzu_test]
fn test_normalization_f32_f32_f32() {
test_normalization::<f32, f32, f32>();