A high-performance inference engine for AI models
rmsnorm simplification (also makes it faster) (#327)
uuuvnc6d1c8232a47
diff --git a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rsindex 352a6ed2c..05d147995 100644--- a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs+++ b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs@@ -18,13 +18,13 @@ pub fn rms_norm<#[optional(!in_place)] input: Option<*const InputT>,scales: *const ScaleT,output: *mut OutputT,- #[optional(copy_to_shortcut)] shortcut_buffer: Option<*mut InputT>,+ #[optional(copy_to_shortcut)] shortcut: Option<*mut InputT>,batch_size: u32,element_count: u32,epsilon: f32,scale_offset: f32,- full_layer: bool,#[specialize] in_place: bool,+ #[specialize] full_layer: bool,#[specialize] copy_to_shortcut: bool,#[specialize] residual_add: bool,) {@@ -49,7 +49,7 @@ pub fn rms_norm<let input_val = unsafe { *input.add(batch_offset + i) };let mut val = input_val;if copy_to_shortcut {- let skip_ptr = unsafe { shortcut_buffer.unwrap().add(batch_offset + i) };+ let skip_ptr = unsafe { shortcut.unwrap().add(batch_offset + i) };if residual_add {val = val + unsafe { *skip_ptr };}@@ -64,7 +64,7 @@ pub fn rms_norm<// Normalization and scalingfor i in 0..element_count {let input_val = if residual_add {- unsafe { AccumT::from(*shortcut_buffer.unwrap().add(batch_offset + i)).unwrap() }+ unsafe { AccumT::from(*shortcut.unwrap().add(batch_offset + i)).unwrap() }} else {unsafe { AccumT::from(*input.add(batch_offset + i)).unwrap() }};diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex 1ebabc916..553f0c52c 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -7,7 +7,6 @@using namespace metal;#define BLOCK_SIZE 1024-#define GRAIN_SIZE 4template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>VARIANTS(InputT, float, half, bfloat)@@ -18,13 +17,13 @@ PUBLIC KERNEL(RMSNorm)(const device InputT* input OPTIONAL(!in_place),const device ScaleT* scales,device OutputT* output,- device InputT* shortcut_buffer OPTIONAL(copy_to_shortcut),+ device InputT* shortcut OPTIONAL(copy_to_shortcut),constant uint& batch_size,constant uint& element_count,constant float& epsilon,constant float& scale_offset,- constant bool& full_layer,const bool in_place SPECIALIZE,+ const bool full_layer SPECIALIZE,const bool copy_to_shortcut SPECIALIZE,const bool residual_add SPECIALIZE,threadgroup AccumT shared_sum[METAL_SIMD_SIZE],@@ -36,104 +35,67 @@ PUBLIC KERNEL(RMSNorm)(input = reinterpret_cast<const device InputT*>(output);}- const uint input_offset = batch_idx * element_count;- const device InputT* input_data = input + input_offset;- const device ScaleT* scales_data = scales;- device OutputT* output_data = output + input_offset;-- AccumT partial_sum = static_cast<AccumT>(0.0f);+ const uint batch_offset = batch_idx * element_count;+ input += batch_offset;+ output += batch_offset;+ if (copy_to_shortcut) {+ shortcut += batch_offset;+ }- // Compute thread local partial sum (+ residual add or copy to shortcut)- for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;- base_i += BLOCK_SIZE * GRAIN_SIZE) {- AccumT vals[GRAIN_SIZE];+ // Step 1 - threads read from global and accumulate sum of squares+ AccumT thread_sum_of_squares = static_cast<AccumT>(0.0f);- for (uint j = 0; j < GRAIN_SIZE; ++j) {- uint i = base_i + j;- if (i < element_count) {- InputT val = input_data[i];- if (copy_to_shortcut) {- if (residual_add) {- val = val + shortcut_buffer[input_offset + i];- }- shortcut_buffer[input_offset + i] = val;- }- vals[j] = static_cast<AccumT>(val);- } else {- vals[j] = 0.0f;+ for (uint i = thread_in_row; i < element_count; i += BLOCK_SIZE) {+ InputT val = input[i];+ // We can also fuse:+ // - TensorCopy (copy_to_shortcut)+ // - TensorAddSwap (copy_to_shortcut + residual_add)+ // RMSNorm in TensorAddSwap fusion mode operates on input + shortcut+ if (copy_to_shortcut) {+ if (residual_add) {+ val += shortcut[i];}+ shortcut[i] = val;}-- for (uint j = 0; j < GRAIN_SIZE; ++j) {- partial_sum += vals[j] * vals[j];- }+ AccumT val_accum_t = static_cast<AccumT>(val);+ thread_sum_of_squares += val_accum_t * val_accum_t;}- // Compute total sum across threadgroup- AccumT total_sum =+ // Step 2 - threads reduce their partial sums of squares+ AccumT total_sum_of_squares =threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(- partial_sum,+ thread_sum_of_squares,shared_sum,thread_context);- // Compute RMS norm factor- AccumT mean_square =- static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);- AccumT rms_norm = rsqrt(mean_square + static_cast<AccumT>(epsilon));-- // Apply normalization and scaling using the same vectorized pattern- for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;- base_i += BLOCK_SIZE * GRAIN_SIZE) {- AccumT vals[GRAIN_SIZE];- AccumT scaled_vals[GRAIN_SIZE];-- // Load GRAIN_SIZE input elements (from shortcut if residual_add, since it- // has the sum)- for (uint j = 0; j < GRAIN_SIZE; ++j) {- uint i = base_i + j;- if (residual_add) {- vals[j] = (i < element_count)- ? static_cast<AccumT>(shortcut_buffer[input_offset + i])- : 0.0f;- } else {- vals[j] =- (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;- }+ // And pre-calculate rms_inv+ AccumT rms_inv = rsqrt(+ total_sum_of_squares / static_cast<AccumT>(element_count) ++ static_cast<AccumT>(epsilon)+ );++ // Step 3 - elementwise normalization+ for (uint i = thread_in_row; i < element_count; i += BLOCK_SIZE) {+ AccumT x;+ // If we fuse TensorAddSwap, read shortcut (that now has input + shortcut)+ // No need for memory barrier because each thread only reads what it wrote+ if (residual_add) {+ x = static_cast<AccumT>(shortcut[i]);+ } else {+ x = static_cast<AccumT>(input[i]);}- // Process GRAIN_SIZE elements: normalize and scale- for (uint j = 0; j < GRAIN_SIZE; ++j) {- uint i = base_i + j;- if (i >= element_count) {- continue;- }-- AccumT normalized_high = vals[j] * rms_norm;+ AccumT scale =+ static_cast<AccumT>(scales[i]) + static_cast<AccumT>(scale_offset);- if (full_layer) {- // Full-layer: keep everything in accumulation precision- AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) +- static_cast<AccumT>(scale_offset);- scaled_vals[j] = normalized_high * scale_value_high;- } else {- // Only-normalization: cast down for the scale multiply- OutputT normalized_low = static_cast<OutputT>(normalized_high);- OutputT scale_value_low = static_cast<OutputT>(- static_cast<AccumT>(scales_data[i]) +- static_cast<AccumT>(scale_offset)- );- OutputT product_low = normalized_low * scale_value_low;- scaled_vals[j] = static_cast<AccumT>(product_low);- }- }-- // Store GRAIN_SIZE output elements- for (uint j = 0; j < GRAIN_SIZE; ++j) {- uint i = base_i + j;- if (i < element_count) {- output_data[i] = static_cast<OutputT>(scaled_vals[j]);- }+ // If full_layer, normalize and scale in AccumT, cast to OutputT at the end+ // If not, cast to OutputT after normalize, scale in OutputT+ if (full_layer) {+ output[i] = static_cast<OutputT>(x * rms_inv * scale);+ } else {+ output[i] =+ static_cast<OutputT>(x * rms_inv) * static_cast<OutputT>(scale);}}}diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rsindex 16cc25b9f..7e06ae5de 100644--- a/crates/uzu/src/encodable_block/rms_norm.rs+++ b/crates/uzu/src/encodable_block/rms_norm.rs@@ -65,6 +65,7 @@ impl<B: Backend> RMSNorm<B> {output_type,accumulation_data_type,input_array_id == output_array_id,+ config.upcast_mode == UpcastMode::FullLayer,shortcut_array_id.is_some(),residual_add,)@@ -134,7 +135,6 @@ impl<B: Backend> RMSNorm<B> {element_count as u32,self.config.epsilon,self.config.scale_offset.unwrap_or(0.0),- self.config.upcast_mode == UpcastMode::FullLayer,encoder,);Ok(())diff --git a/crates/uzu/tests/unit/kernel/rms_norm_test.rs b/crates/uzu/tests/unit/kernel/rms_norm_test.rsindex 05bf22344..11e06e464 100644--- a/crates/uzu/tests/unit/kernel/rms_norm_test.rs+++ b/crates/uzu/tests/unit/kernel/rms_norm_test.rs@@ -4,19 +4,23 @@ use std::{time::{Duration, Instant},};+use criterion::{BenchmarkId, Criterion, Throughput};use half::{bf16, f16};+use itertools::iproduct;use num_traits::Float;-#[cfg(metal_backend)]-use rand::{RngExt, SeedableRng, prelude::StdRng};+use rand::{RngExt, SeedableRng, rngs::SmallRng};use uzu::{ArrayContextExt, ArrayElement, DataType,backends::{- common::{Backend, Context, Encoder, Kernels, kernel::RMSNormKernel},+ common::{Backend, Buffer, Context, Encoder, Kernels, kernel::RMSNormKernel},cpu::Cpu,},};-use crate::{common::assert::assert_eq_float, uzu_test};+use crate::{+ common::{assert::assert_eq_float, type_short_name},+ uzu_bench, uzu_test,+};static BOOL_ALL: &[bool] = &[true, false];static BOOL_FALSE: &[bool] = &[false];@@ -126,6 +130,7 @@ fn get_output<OutputT::data_type(),AccumT::data_type(),input.in_place,+ input.full_layer,false,false,)@@ -154,7 +159,6 @@ fn get_output<input.element_count,input.epsilon,input.scale_offset,- input.full_layer,&mut encoder,);@@ -235,75 +239,22 @@ fn test_edge<}}-#[cfg(metal_backend)]-fn test_performance<B: Backend>(+fn get_rms_norm_data(+ seed: u64,batch_size: usize,model_dim: usize,-) {- let epsilon = 1e-6f32;- let scale_offset = 0.0f32;- let mut rng = StdRng::seed_from_u64(42);-+) -> (Box<[f32]>, Box<[f32]>) {+ let mut rng = SmallRng::seed_from_u64(seed);let input_size = batch_size * model_dim;- let mut input_data = vec![0.0f32; batch_size * model_dim];- let mut output_data = vec![0.0f32; batch_size * model_dim];- for i in 0..input_size {- input_data[i] = rng.random_range(-2.0f32..2.0f32);- output_data[i] = input_data[i]+ let mut input_data = vec![0.0f32; input_size];+ for x in input_data.iter_mut() {+ *x = rng.random_range(-2.0f32..2.0f32);}-let mut scale_data = vec![0.0f32; model_dim];for x in scale_data.iter_mut() {*x = rng.random_range(0.1f32..3.0f32);}-- let input = Input::<f32, f32, f32> {- input: input_data.into_boxed_slice(),- scales: scale_data.into_boxed_slice(),- output: output_data.into_boxed_slice(),- batch_size: batch_size as u32,- element_count: model_dim as u32,- epsilon,- scale_offset,- full_layer: false,- in_place: false,- };- let (output_data, host_elapsed_ms, gpu_time) = get_output::<B, f32, f32, f32, f32>(&input);- match gpu_time {- Some(gpu_time) => {- println!(- "RMS norm perf (backend={}, batch={}, model_dim={}): GPU={:.2} ms, Host-side={:.2} ms",- std::any::type_name::<B>(),- batch_size,- model_dim,- gpu_time.as_secs_f64() * 1e3,- host_elapsed_ms- );- },- None => {- println!(- "RMS norm perf (backend={}, batch={}, model_dim={}): Host-side={:.2} ms (GPU timing unavailable)",- std::any::type_name::<B>(),- batch_size,- model_dim,- host_elapsed_ms- );- },- }-- // Sample check for large outputs- let sample_size = std::cmp::min(1000, output_data.len());- for i in (0..output_data.len()).step_by(std::cmp::max(1, output_data.len() / sample_size)) {- assert!(output_data[i].is_finite(), "Output at index {} is not finite: {}", i, output_data[i]);- }-- // Check that normalization is working- let mean_abs = output_data.iter().take(sample_size).map(|x| x.abs()).sum::<f32>() / sample_size as f32;- assert!(- mean_abs > 0.01 && mean_abs < 100.0,- "Mean absolute value {} seems unreasonable - normalization may not be working",- mean_abs- );+ (input_data.into_boxed_slice(), scale_data.into_boxed_slice())}// AccumT f32@@ -436,17 +387,83 @@ fn test_edge_f16_f16_f16_f16() {test_edge::<f16, f16, f16, f16>();}-// performance tests-#[cfg(metal_backend)]-#[uzu_test]-fn test_perf_8k() {- use uzu::backends::metal::Metal;- test_performance::<Metal>(8, 8192); // Large model like LLaMA-70B-}+// benchmarks+#[uzu_bench]+fn bench_rms_norm(c: &mut Criterion) {+ type T = f32;+ let epsilon = 1e-6f32;+ let scale_offset = 0.0f32;-#[cfg(metal_backend)]-#[uzu_test]-fn test_perf_16k() {- use uzu::backends::metal::Metal;- test_performance::<Metal>(16, 16384); // Huge model dimension+ for_each_backend!(|B| {+ let context = <B as Backend>::Context::new().unwrap();++ let kernel = <<B as Backend>::Kernels as Kernels>::RMSNormKernel::new(+ &context,+ T::data_type(),+ T::data_type(),+ T::data_type(),+ T::data_type(),+ false,+ false,+ false,+ false,+ )+ .unwrap();++ let mut group = c.benchmark_group(format!("{}/Kernel/RMSNorm", type_short_name::<B>()));++ for (batch_size, model_dim) in iproduct!(+ [1, 4, 32, 128, 1024], // Batch sizes+ [+ 1024, // LFM2-350M+ 1152, // Gemma-3-1b+ 2048, // Llama-3.2-1B+ 2560, // Qwen3-4B+ 3072, // Llama-3.2-3B+ 4096, // Qwen3-8B+ 5120, // Qwen2.5-32B+ ]+ ) {+ let (input_data, scale_data) = get_rms_norm_data(1337, batch_size, model_dim);+ let input_size = batch_size * model_dim;++ let input_buffer = context.create_buffer(input_size * std::mem::size_of::<T>()).unwrap();+ unsafe {+ std::slice::from_raw_parts_mut::<T>(input_buffer.cpu_ptr().as_ptr() as *mut T, input_size)+ .copy_from_slice(input_data.as_ref());+ }++ let scales_buffer = context.create_buffer(model_dim * std::mem::size_of::<T>()).unwrap();+ unsafe {+ std::slice::from_raw_parts_mut::<T>(scales_buffer.cpu_ptr().as_ptr() as *mut T, model_dim)+ .copy_from_slice(scale_data.as_ref());+ }++ let mut output_buffer = context.create_buffer(input_size * std::mem::size_of::<T>()).unwrap();++ group.throughput(Throughput::Elements((batch_size * model_dim) as u64));++ group.bench_function(BenchmarkId::from_parameter(format!("Batch[{batch_size}]Dim[{model_dim}]")), |b| {+ b.iter_custom(|n_iters| {+ let mut encoder = Encoder::<B>::new(&context).unwrap();++ for _ in 0..n_iters {+ kernel.encode(+ Some(&input_buffer),+ &scales_buffer,+ &mut output_buffer,+ None::<&mut <B as Backend>::Buffer>,+ batch_size as u32,+ model_dim as u32,+ epsilon,+ scale_offset,+ &mut encoder,+ );+ }++ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time().unwrap()+ })+ });+ }+ });}