Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

rmsnorm simplification (also makes it faster) (#327)

uuuvnc6d1c8232a47
diff --git a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
index 352a6ed2c..05d147995 100644
--- a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
+++ b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
@@ -18,13 +18,13 @@ pub fn rms_norm<
#[optional(!in_place)] input: Option<*const InputT>,
scales: *const ScaleT,
output: *mut OutputT,
- #[optional(copy_to_shortcut)] shortcut_buffer: Option<*mut InputT>,
+ #[optional(copy_to_shortcut)] shortcut: Option<*mut InputT>,
batch_size: u32,
element_count: u32,
epsilon: f32,
scale_offset: f32,
- full_layer: bool,
#[specialize] in_place: bool,
+ #[specialize] full_layer: bool,
#[specialize] copy_to_shortcut: bool,
#[specialize] residual_add: bool,
) {
@@ -49,7 +49,7 @@ pub fn rms_norm<
let input_val = unsafe { *input.add(batch_offset + i) };
let mut val = input_val;
if copy_to_shortcut {
- let skip_ptr = unsafe { shortcut_buffer.unwrap().add(batch_offset + i) };
+ let skip_ptr = unsafe { shortcut.unwrap().add(batch_offset + i) };
if residual_add {
val = val + unsafe { *skip_ptr };
}
@@ -64,7 +64,7 @@ pub fn rms_norm<
// Normalization and scaling
for i in 0..element_count {
let input_val = if residual_add {
- unsafe { AccumT::from(*shortcut_buffer.unwrap().add(batch_offset + i)).unwrap() }
+ unsafe { AccumT::from(*shortcut.unwrap().add(batch_offset + i)).unwrap() }
} else {
unsafe { AccumT::from(*input.add(batch_offset + i)).unwrap() }
};
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index 1ebabc916..553f0c52c 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -7,7 +7,6 @@
using namespace metal;
#define BLOCK_SIZE 1024
-#define GRAIN_SIZE 4
template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>
VARIANTS(InputT, float, half, bfloat)
@@ -18,13 +17,13 @@ PUBLIC KERNEL(RMSNorm)(
const device InputT* input OPTIONAL(!in_place),
const device ScaleT* scales,
device OutputT* output,
- device InputT* shortcut_buffer OPTIONAL(copy_to_shortcut),
+ device InputT* shortcut OPTIONAL(copy_to_shortcut),
constant uint& batch_size,
constant uint& element_count,
constant float& epsilon,
constant float& scale_offset,
- constant bool& full_layer,
const bool in_place SPECIALIZE,
+ const bool full_layer SPECIALIZE,
const bool copy_to_shortcut SPECIALIZE,
const bool residual_add SPECIALIZE,
threadgroup AccumT shared_sum[METAL_SIMD_SIZE],
@@ -36,104 +35,67 @@ PUBLIC KERNEL(RMSNorm)(
input = reinterpret_cast<const device InputT*>(output);
}
- const uint input_offset = batch_idx * element_count;
- const device InputT* input_data = input + input_offset;
- const device ScaleT* scales_data = scales;
- device OutputT* output_data = output + input_offset;
-
- AccumT partial_sum = static_cast<AccumT>(0.0f);
+ const uint batch_offset = batch_idx * element_count;
+ input += batch_offset;
+ output += batch_offset;
+ if (copy_to_shortcut) {
+ shortcut += batch_offset;
+ }
- // Compute thread local partial sum (+ residual add or copy to shortcut)
- for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;
- base_i += BLOCK_SIZE * GRAIN_SIZE) {
- AccumT vals[GRAIN_SIZE];
+ // Step 1 - threads read from global and accumulate sum of squares
+ AccumT thread_sum_of_squares = static_cast<AccumT>(0.0f);
- for (uint j = 0; j < GRAIN_SIZE; ++j) {
- uint i = base_i + j;
- if (i < element_count) {
- InputT val = input_data[i];
- if (copy_to_shortcut) {
- if (residual_add) {
- val = val + shortcut_buffer[input_offset + i];
- }
- shortcut_buffer[input_offset + i] = val;
- }
- vals[j] = static_cast<AccumT>(val);
- } else {
- vals[j] = 0.0f;
+ for (uint i = thread_in_row; i < element_count; i += BLOCK_SIZE) {
+ InputT val = input[i];
+ // We can also fuse:
+ // - TensorCopy (copy_to_shortcut)
+ // - TensorAddSwap (copy_to_shortcut + residual_add)
+ // RMSNorm in TensorAddSwap fusion mode operates on input + shortcut
+ if (copy_to_shortcut) {
+ if (residual_add) {
+ val += shortcut[i];
}
+ shortcut[i] = val;
}
-
- for (uint j = 0; j < GRAIN_SIZE; ++j) {
- partial_sum += vals[j] * vals[j];
- }
+ AccumT val_accum_t = static_cast<AccumT>(val);
+ thread_sum_of_squares += val_accum_t * val_accum_t;
}
- // Compute total sum across threadgroup
- AccumT total_sum =
+ // Step 2 - threads reduce their partial sums of squares
+ AccumT total_sum_of_squares =
threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(
- partial_sum,
+ thread_sum_of_squares,
shared_sum,
thread_context
);
- // Compute RMS norm factor
- AccumT mean_square =
- static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);
- AccumT rms_norm = rsqrt(mean_square + static_cast<AccumT>(epsilon));
-
- // Apply normalization and scaling using the same vectorized pattern
- for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;
- base_i += BLOCK_SIZE * GRAIN_SIZE) {
- AccumT vals[GRAIN_SIZE];
- AccumT scaled_vals[GRAIN_SIZE];
-
- // Load GRAIN_SIZE input elements (from shortcut if residual_add, since it
- // has the sum)
- for (uint j = 0; j < GRAIN_SIZE; ++j) {
- uint i = base_i + j;
- if (residual_add) {
- vals[j] = (i < element_count)
- ? static_cast<AccumT>(shortcut_buffer[input_offset + i])
- : 0.0f;
- } else {
- vals[j] =
- (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
- }
+ // And pre-calculate rms_inv
+ AccumT rms_inv = rsqrt(
+ total_sum_of_squares / static_cast<AccumT>(element_count) +
+ static_cast<AccumT>(epsilon)
+ );
+
+ // Step 3 - elementwise normalization
+ for (uint i = thread_in_row; i < element_count; i += BLOCK_SIZE) {
+ AccumT x;
+ // If we fuse TensorAddSwap, read shortcut (that now has input + shortcut)
+ // No need for memory barrier because each thread only reads what it wrote
+ if (residual_add) {
+ x = static_cast<AccumT>(shortcut[i]);
+ } else {
+ x = static_cast<AccumT>(input[i]);
}
- // Process GRAIN_SIZE elements: normalize and scale
- for (uint j = 0; j < GRAIN_SIZE; ++j) {
- uint i = base_i + j;
- if (i >= element_count) {
- continue;
- }
-
- AccumT normalized_high = vals[j] * rms_norm;
+ AccumT scale =
+ static_cast<AccumT>(scales[i]) + static_cast<AccumT>(scale_offset);
- if (full_layer) {
- // Full-layer: keep everything in accumulation precision
- AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) +
- static_cast<AccumT>(scale_offset);
- scaled_vals[j] = normalized_high * scale_value_high;
- } else {
- // Only-normalization: cast down for the scale multiply
- OutputT normalized_low = static_cast<OutputT>(normalized_high);
- OutputT scale_value_low = static_cast<OutputT>(
- static_cast<AccumT>(scales_data[i]) +
- static_cast<AccumT>(scale_offset)
- );
- OutputT product_low = normalized_low * scale_value_low;
- scaled_vals[j] = static_cast<AccumT>(product_low);
- }
- }
-
- // Store GRAIN_SIZE output elements
- for (uint j = 0; j < GRAIN_SIZE; ++j) {
- uint i = base_i + j;
- if (i < element_count) {
- output_data[i] = static_cast<OutputT>(scaled_vals[j]);
- }
+ // If full_layer, normalize and scale in AccumT, cast to OutputT at the end
+ // If not, cast to OutputT after normalize, scale in OutputT
+ if (full_layer) {
+ output[i] = static_cast<OutputT>(x * rms_inv * scale);
+ } else {
+ output[i] =
+ static_cast<OutputT>(x * rms_inv) * static_cast<OutputT>(scale);
}
}
}
diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rs
index 16cc25b9f..7e06ae5de 100644
--- a/crates/uzu/src/encodable_block/rms_norm.rs
+++ b/crates/uzu/src/encodable_block/rms_norm.rs
@@ -65,6 +65,7 @@ impl<B: Backend> RMSNorm<B> {
output_type,
accumulation_data_type,
input_array_id == output_array_id,
+ config.upcast_mode == UpcastMode::FullLayer,
shortcut_array_id.is_some(),
residual_add,
)
@@ -134,7 +135,6 @@ impl<B: Backend> RMSNorm<B> {
element_count as u32,
self.config.epsilon,
self.config.scale_offset.unwrap_or(0.0),
- self.config.upcast_mode == UpcastMode::FullLayer,
encoder,
);
Ok(())
diff --git a/crates/uzu/tests/unit/kernel/rms_norm_test.rs b/crates/uzu/tests/unit/kernel/rms_norm_test.rs
index 05bf22344..11e06e464 100644
--- a/crates/uzu/tests/unit/kernel/rms_norm_test.rs
+++ b/crates/uzu/tests/unit/kernel/rms_norm_test.rs
@@ -4,19 +4,23 @@ use std::{
time::{Duration, Instant},
};
+use criterion::{BenchmarkId, Criterion, Throughput};
use half::{bf16, f16};
+use itertools::iproduct;
use num_traits::Float;
-#[cfg(metal_backend)]
-use rand::{RngExt, SeedableRng, prelude::StdRng};
+use rand::{RngExt, SeedableRng, rngs::SmallRng};
use uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
- common::{Backend, Context, Encoder, Kernels, kernel::RMSNormKernel},
+ common::{Backend, Buffer, Context, Encoder, Kernels, kernel::RMSNormKernel},
cpu::Cpu,
},
};
-use crate::{common::assert::assert_eq_float, uzu_test};
+use crate::{
+ common::{assert::assert_eq_float, type_short_name},
+ uzu_bench, uzu_test,
+};
static BOOL_ALL: &[bool] = &[true, false];
static BOOL_FALSE: &[bool] = &[false];
@@ -126,6 +130,7 @@ fn get_output<
OutputT::data_type(),
AccumT::data_type(),
input.in_place,
+ input.full_layer,
false,
false,
)
@@ -154,7 +159,6 @@ fn get_output<
input.element_count,
input.epsilon,
input.scale_offset,
- input.full_layer,
&mut encoder,
);
@@ -235,75 +239,22 @@ fn test_edge<
}
}
-#[cfg(metal_backend)]
-fn test_performance<B: Backend>(
+fn get_rms_norm_data(
+ seed: u64,
batch_size: usize,
model_dim: usize,
-) {
- let epsilon = 1e-6f32;
- let scale_offset = 0.0f32;
- let mut rng = StdRng::seed_from_u64(42);
-
+) -> (Box<[f32]>, Box<[f32]>) {
+ let mut rng = SmallRng::seed_from_u64(seed);
let input_size = batch_size * model_dim;
- let mut input_data = vec![0.0f32; batch_size * model_dim];
- let mut output_data = vec![0.0f32; batch_size * model_dim];
- for i in 0..input_size {
- input_data[i] = rng.random_range(-2.0f32..2.0f32);
- output_data[i] = input_data[i]
+ let mut input_data = vec![0.0f32; input_size];
+ for x in input_data.iter_mut() {
+ *x = rng.random_range(-2.0f32..2.0f32);
}
-
let mut scale_data = vec![0.0f32; model_dim];
for x in scale_data.iter_mut() {
*x = rng.random_range(0.1f32..3.0f32);
}
-
- let input = Input::<f32, f32, f32> {
- input: input_data.into_boxed_slice(),
- scales: scale_data.into_boxed_slice(),
- output: output_data.into_boxed_slice(),
- batch_size: batch_size as u32,
- element_count: model_dim as u32,
- epsilon,
- scale_offset,
- full_layer: false,
- in_place: false,
- };
- let (output_data, host_elapsed_ms, gpu_time) = get_output::<B, f32, f32, f32, f32>(&input);
- match gpu_time {
- Some(gpu_time) => {
- println!(
- "RMS norm perf (backend={}, batch={}, model_dim={}): GPU={:.2} ms, Host-side={:.2} ms",
- std::any::type_name::<B>(),
- batch_size,
- model_dim,
- gpu_time.as_secs_f64() * 1e3,
- host_elapsed_ms
- );
- },
- None => {
- println!(
- "RMS norm perf (backend={}, batch={}, model_dim={}): Host-side={:.2} ms (GPU timing unavailable)",
- std::any::type_name::<B>(),
- batch_size,
- model_dim,
- host_elapsed_ms
- );
- },
- }
-
- // Sample check for large outputs
- let sample_size = std::cmp::min(1000, output_data.len());
- for i in (0..output_data.len()).step_by(std::cmp::max(1, output_data.len() / sample_size)) {
- assert!(output_data[i].is_finite(), "Output at index {} is not finite: {}", i, output_data[i]);
- }
-
- // Check that normalization is working
- let mean_abs = output_data.iter().take(sample_size).map(|x| x.abs()).sum::<f32>() / sample_size as f32;
- assert!(
- mean_abs > 0.01 && mean_abs < 100.0,
- "Mean absolute value {} seems unreasonable - normalization may not be working",
- mean_abs
- );
+ (input_data.into_boxed_slice(), scale_data.into_boxed_slice())
}
// AccumT f32
@@ -436,17 +387,83 @@ fn test_edge_f16_f16_f16_f16() {
test_edge::<f16, f16, f16, f16>();
}
-// performance tests
-#[cfg(metal_backend)]
-#[uzu_test]
-fn test_perf_8k() {
- use uzu::backends::metal::Metal;
- test_performance::<Metal>(8, 8192); // Large model like LLaMA-70B
-}
+// benchmarks
+#[uzu_bench]
+fn bench_rms_norm(c: &mut Criterion) {
+ type T = f32;
+ let epsilon = 1e-6f32;
+ let scale_offset = 0.0f32;
-#[cfg(metal_backend)]
-#[uzu_test]
-fn test_perf_16k() {
- use uzu::backends::metal::Metal;
- test_performance::<Metal>(16, 16384); // Huge model dimension
+ for_each_backend!(|B| {
+ let context = <B as Backend>::Context::new().unwrap();
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::RMSNormKernel::new(
+ &context,
+ T::data_type(),
+ T::data_type(),
+ T::data_type(),
+ T::data_type(),
+ false,
+ false,
+ false,
+ false,
+ )
+ .unwrap();
+
+ let mut group = c.benchmark_group(format!("{}/Kernel/RMSNorm", type_short_name::<B>()));
+
+ for (batch_size, model_dim) in iproduct!(
+ [1, 4, 32, 128, 1024], // Batch sizes
+ [
+ 1024, // LFM2-350M
+ 1152, // Gemma-3-1b
+ 2048, // Llama-3.2-1B
+ 2560, // Qwen3-4B
+ 3072, // Llama-3.2-3B
+ 4096, // Qwen3-8B
+ 5120, // Qwen2.5-32B
+ ]
+ ) {
+ let (input_data, scale_data) = get_rms_norm_data(1337, batch_size, model_dim);
+ let input_size = batch_size * model_dim;
+
+ let input_buffer = context.create_buffer(input_size * std::mem::size_of::<T>()).unwrap();
+ unsafe {
+ std::slice::from_raw_parts_mut::<T>(input_buffer.cpu_ptr().as_ptr() as *mut T, input_size)
+ .copy_from_slice(input_data.as_ref());
+ }
+
+ let scales_buffer = context.create_buffer(model_dim * std::mem::size_of::<T>()).unwrap();
+ unsafe {
+ std::slice::from_raw_parts_mut::<T>(scales_buffer.cpu_ptr().as_ptr() as *mut T, model_dim)
+ .copy_from_slice(scale_data.as_ref());
+ }
+
+ let mut output_buffer = context.create_buffer(input_size * std::mem::size_of::<T>()).unwrap();
+
+ group.throughput(Throughput::Elements((batch_size * model_dim) as u64));
+
+ group.bench_function(BenchmarkId::from_parameter(format!("Batch[{batch_size}]Dim[{model_dim}]")), |b| {
+ b.iter_custom(|n_iters| {
+ let mut encoder = Encoder::<B>::new(&context).unwrap();
+
+ for _ in 0..n_iters {
+ kernel.encode(
+ Some(&input_buffer),
+ &scales_buffer,
+ &mut output_buffer,
+ None::<&mut <B as Backend>::Buffer>,
+ batch_size as u32,
+ model_dim as u32,
+ epsilon,
+ scale_offset,
+ &mut encoder,
+ );
+ }
+
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time().unwrap()
+ })
+ });
+ }
+ });
}