A high-performance inference engine for AI models
use half::{bf16, f16};
use num_traits::Float;
use uzu_engine_macros::kernel;
use crate::{
backends::{
common::gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
cpu::kernel::activation_transform::hadamard_transform,
},
data_type::DataTypeRepr,
};
#[kernel(Normalization)]
#[variants(InputT, f32, f16, bf16)]
#[variants(AffineT, f32, f16, bf16)]
#[variants(OutputT, f32, f16, bf16)]
#[variants(AccumT, f32)]
pub fn normalization<
InputT: DataTypeRepr + Float,
AffineT: DataTypeRepr + Float,
OutputT: DataTypeRepr + Float,
AccumT: DataTypeRepr + Float,
>(
#[optional(!in_place)] input: Option<*const InputT>,
#[optional(has_scales)] scales: Option<*const AffineT>,
#[optional(has_biases)] biases: Option<*const AffineT>,
output: *mut OutputT,
#[optional(copy_to_shortcut)] shortcut: Option<*mut InputT>,
#[optional(use_hadamard)] hadamard_factors: Option<*const i32>,
batch_size: u32,
element_count: u32,
epsilon: f32,
scale_offset: f32,
post_layer_scalar: f32,
#[specialize] in_place: bool,
#[specialize] subtract_mean: bool,
#[specialize] full_layer: bool,
#[specialize] copy_to_shortcut: bool,
#[specialize] residual_add: bool,
#[specialize] use_hadamard: bool,
#[specialize] scale_residual_sum: bool,
#[specialize] scale_output: bool,
#[specialize] has_biases: bool,
#[specialize] has_scales: bool,
) {
assert_eq!(shortcut.is_some(), copy_to_shortcut);
assert_eq!(hadamard_factors.is_some(), use_hadamard);
assert_eq!(biases.is_some(), has_biases);
assert_eq!(scales.is_some(), has_scales);
assert!(copy_to_shortcut || !residual_add);
assert!(!use_hadamard || element_count.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));
let input = match in_place {
true => output as *const InputT,
false => input.unwrap(),
};
let element_count = element_count as usize;
let epsilon = AccumT::from(epsilon).unwrap();
let scale_offset = AccumT::from(scale_offset).unwrap();
let element_count_accum = AccumT::from(element_count).unwrap();
let mut row = vec![OutputT::zero(); element_count];
for batch in 0..(batch_size as usize) {
let batch_offset = batch * element_count;
// Compute mean / rms_inv (+ residual add or copy to shortcut)
let mut sum = AccumT::zero();
let mut sum_sq = AccumT::zero();
for i in 0..element_count {
let input_val = unsafe { *input.add(batch_offset + i) };
let mut val = input_val;
if copy_to_shortcut {
let skip_ptr = unsafe { shortcut.unwrap().add(batch_offset + i) };
if residual_add {
val = val + unsafe { *skip_ptr };
if scale_residual_sum {
val = InputT::from(val.to_f32().unwrap() * post_layer_scalar).unwrap();
}
}
unsafe { *skip_ptr = val };
}
let accum_val = AccumT::from(val).unwrap();
if subtract_mean {
sum = sum + accum_val;
}
sum_sq = sum_sq + accum_val * accum_val;
}
let mean = if subtract_mean {
sum / element_count_accum
} else {
AccumT::zero()
};
let variance = sum_sq / element_count_accum - mean * mean;
let rms_inv = AccumT::from((variance + epsilon).to_f32().unwrap().sqrt().recip()).unwrap();
// Normalization and scaling
for i in 0..element_count {
let input_val = if residual_add {
unsafe { AccumT::from(*shortcut.unwrap().add(batch_offset + i)).unwrap() }
} else {
unsafe { AccumT::from(*input.add(batch_offset + i)).unwrap() }
};
let normalized: AccumT = (input_val - mean) * rms_inv;
let mut result: OutputT = if has_scales {
let scale_val = unsafe { AccumT::from(*scales.unwrap().add(i)).unwrap() };
if full_layer {
// Full-layer: keep everything in accumulation precision
let scale_with_offset: AccumT = scale_val + scale_offset;
OutputT::from(normalized * scale_with_offset).unwrap()
} else {
// Only-normalization: cast down to output precision for the scale multiply
let normalized_out = OutputT::from(normalized).unwrap();
let scale_with_offset_out = OutputT::from(scale_val + scale_offset).unwrap();
normalized_out * scale_with_offset_out
}
} else {
OutputT::from(normalized).unwrap()
};
if has_biases {
let bias = unsafe { AccumT::from(*biases.unwrap().add(i)).unwrap() };
result = OutputT::from(AccumT::from(result).unwrap() + bias).unwrap();
}
row[i] = result;
}
// Input RHT like the Metal kernel: apply the factors, then transform each block, before output scaling
if let Some(factors) = hadamard_factors {
for block_start in (0..element_count).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE as usize) {
let mut block: [f32; HADAMARD_TRANSFORM_BLOCK_SIZE as usize] = std::array::from_fn(|lane| {
let factor = unsafe { *factors.add(block_start + lane) } as f32;
row[block_start + lane].to_f32().unwrap() * factor
});
hadamard_transform(&mut block);
for (lane, value) in block.into_iter().enumerate() {
row[block_start + lane] = OutputT::from(value).unwrap();
}
}
}
for (i, &value) in row.iter().enumerate() {
let mut result = value;
if scale_output {
result = result * OutputT::from(post_layer_scalar).unwrap();
}
unsafe { *output.add(batch_offset + i) = result };
}
}
}