A high-performance inference engine for AI models
Fused tensorcopy+rmsnorm (#319)
uuuvne6f9e3c48289
diff --git a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rsindex 8099202cb..6a1a73a3d 100644--- a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs+++ b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs@@ -18,12 +18,14 @@ pub fn rms_norm<#[optional(!in_place)] input: Option<*const InputT>,scales: *const ScaleT,output: *mut OutputT,+ #[optional(copy_to_shortcut)] shortcut_buffer: Option<*mut InputT>,batch_size: u32,element_count: u32,epsilon: f32,scale_offset: f32,full_layer: bool,#[specialize] in_place: bool,+ #[specialize] copy_to_shortcut: bool,) {let input = match in_place {true => output as *const InputT,@@ -43,8 +45,12 @@ pub fn rms_norm<// Compute rms_invlet mut sum_sq = AccumT::zero();for i in 0..element_count {- let input_val = unsafe { AccumT::from(*input.add(batch_offset + i)).unwrap() };- sum_sq = sum_sq + input_val * input_val;+ let input_val = unsafe { *input.add(batch_offset + i) };+ let accum_val = AccumT::from(input_val).unwrap();+ sum_sq = sum_sq + accum_val * accum_val;+ if copy_to_shortcut {+ unsafe { *shortcut_buffer.unwrap().add(batch_offset + i) = input_val };+ }}let mean_sq: AccumT = sum_sq / element_count_accum;let rms_inv = AccumT::from((mean_sq + epsilon).to_f32().unwrap().sqrt().recip()).unwrap();diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex 79e1e57b9..7615d5690 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -18,12 +18,14 @@ PUBLIC KERNEL(RMSNorm)(const device InputT* input OPTIONAL(!in_place),const device ScaleT* scales,device OutputT* output,+ device InputT* shortcut_buffer OPTIONAL(copy_to_shortcut),constant uint& batch_size,constant uint& element_count,constant float& epsilon,constant float& scale_offset,constant bool& full_layer,const bool in_place SPECIALIZE,+ const bool copy_to_shortcut SPECIALIZE,threadgroup AccumT shared_sum[METAL_SIMD_SIZE],const ThreadContext thread_context,const uint batch_idx GROUPS(batch_size),@@ -40,14 +42,22 @@ PUBLIC KERNEL(RMSNorm)(AccumT partial_sum = static_cast<AccumT>(0.0f);- // Compute thread local partial sum+ // Compute thread local partial sum (+ copy to shortcut if enabled)for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;base_i += BLOCK_SIZE * GRAIN_SIZE) {AccumT vals[GRAIN_SIZE];for (uint j = 0; j < GRAIN_SIZE; ++j) {uint i = base_i + j;- vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;+ if (i < element_count) {+ InputT val = input_data[i];+ vals[j] = static_cast<AccumT>(val);+ if (copy_to_shortcut) {+ shortcut_buffer[input_offset + i] = val;+ }+ } else {+ vals[j] = 0.0f;+ }}for (uint j = 0; j < GRAIN_SIZE; ++j) {diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rsindex 4fa6ac27b..fa64cadee 100644--- a/crates/uzu/src/encodable_block/decoder.rs+++ b/crates/uzu/src/encodable_block/decoder.rs@@ -187,6 +187,7 @@ impl<B: Backend> Decoder<B> {ArrayId::Main,ArrayId::Main,&decoder_weight_loader.subtree("output_norm").unwrap(),+ None,).map(RMSNorm::with_sampling_range).expect("Failed to create output RMS norm kernel");diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex e9ca29161..7056a0ef4 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -9,7 +9,7 @@ use crate::{config::{DecoderLayerConfig, DecoderLayerType, MixerConfig},encodable_block::{Attention, DeltaNetMixer, EncodingParameters, Linear, MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvMixer,- TensorAddSwap, TensorCopy,+ TensorAddSwap,},forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,@@ -19,7 +19,6 @@ use crate::{pub struct LayerExecutables<B: Backend> {#[cfg(feature = "tracing")]pub layer_index: usize,- pub copy_main_to_shortcut: TensorCopy<B>,pub pre_attention_norm: RMSNorm<B>,pub(crate) mixer: MixerExecutables<B>,pub post_attention_norm: Option<RMSNorm<B>>,@@ -51,9 +50,11 @@ impl<B: Backend> LayerExecutables<B> {MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),MixerConfig::DeltaNet(config) => config.in_proj_config.activation_precision().into(),};- let copy_main_to_shortcut =- TensorCopy::<B>::new(context, intermediate_data_type, ArrayId::Main, ArrayId::Shortcut).unwrap();-+ let shortcut_copy = if layer_index == 0 {+ Some(ArrayId::Shortcut)+ } else {+ None+ };let pre_attention_norm = RMSNorm::new(context,intermediate_data_type,@@ -61,6 +62,7 @@ impl<B: Backend> LayerExecutables<B> {ArrayId::Main,ArrayId::Main,&decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),+ shortcut_copy,).expect("Failed to create RMS norm kernel");@@ -212,6 +214,7 @@ impl<B: Backend> LayerExecutables<B> {ArrayId::Main,ArrayId::Main,&decoder_layer_loader.subtree("post_mixer_norm").unwrap(),+ None,).expect("Failed to create RMS norm kernel"),)@@ -229,6 +232,7 @@ impl<B: Backend> LayerExecutables<B> {ArrayId::Main,ArrayId::Main,&decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),+ None,).expect("Failed to create RMS norm kernel");@@ -250,6 +254,7 @@ impl<B: Backend> LayerExecutables<B> {ArrayId::Main,ArrayId::Main,&decoder_layer_loader.subtree("post_mlp_norm").unwrap(),+ None,).expect("Failed to create RMS norm kernel"),)@@ -260,7 +265,6 @@ impl<B: Backend> LayerExecutables<B> {Self {#[cfg(feature = "tracing")]layer_index,- copy_main_to_shortcut,pre_attention_norm,mixer,post_attention_norm,@@ -285,9 +289,6 @@ impl<B: Backend> LayerExecutables<B> {state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().inputs.clone());}- self.copy_main_to_shortcut.encode(state, encoder)?;- // shortcut = input-self.pre_attention_norm.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {diff --git a/crates/uzu/src/encodable_block/normalization.rs b/crates/uzu/src/encodable_block/normalization.rsindex 7c99d0873..bf0490b84 100644--- a/crates/uzu/src/encodable_block/normalization.rs+++ b/crates/uzu/src/encodable_block/normalization.rs@@ -53,6 +53,7 @@ impl<B: Backend> Normalization<B> {input_array_id,output_array_id,parameter_tree,+ None,)?))}}diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rsindex 64514c5fa..c82db8e58 100644--- a/crates/uzu/src/encodable_block/rms_norm.rs+++ b/crates/uzu/src/encodable_block/rms_norm.rs@@ -32,6 +32,7 @@ pub struct RMSNorm<B: Backend> {config: NormalizationConfig,input_array_id: ArrayId,output_array_id: ArrayId,+ shortcut_array_id: Option<ArrayId>,scales_buffer: Rc<RefCell<B::Buffer>>,use_sampling_range: bool,}@@ -44,6 +45,7 @@ impl<B: Backend> RMSNorm<B> {input_array_id: ArrayId,output_array_id: ArrayId,parameter_tree: &ParameterTree<B::Context>,+ shortcut_array_id: Option<ArrayId>,) -> Result<Self, RMSNormError<B>> {let scales = parameter_tree.leaf_array("scales").map_err(RMSNormError::ParameterError)?;@@ -62,6 +64,7 @@ impl<B: Backend> RMSNorm<B> {output_type,accumulation_data_type,input_array_id == output_array_id,+ shortcut_array_id.is_some(),).map_err(RMSNormError::BackendError)?;@@ -70,6 +73,7 @@ impl<B: Backend> RMSNorm<B> {config,input_array_id,output_array_id,+ shortcut_array_id,scales_buffer: scales.buffer(),use_sampling_range: false,})@@ -116,10 +120,14 @@ impl<B: Backend> RMSNorm<B> {let input_buffer = (self.input_array_id != self.output_array_id).then(|| input_array.buffer());let input_buffer_borrow = input_buffer.as_ref().map(|b| b.borrow());+ let shortcut_rc = self.shortcut_array_id.map(|id| state.array(id).buffer());+ let mut shortcut_borrow = shortcut_rc.as_ref().map(|rc| rc.borrow_mut());+self.kernel.encode(input_buffer_borrow.as_deref().map(|b| (b, input_offset)),self.scales_buffer.borrow().deref(),(output_array.buffer().borrow_mut().deref_mut(), output_offset),+ shortcut_borrow.as_deref_mut(),batch_len as u32,element_count as u32,self.config.epsilon,diff --git a/crates/uzu/tests/unit/kernel/rms_norm_test.rs b/crates/uzu/tests/unit/kernel/rms_norm_test.rsindex 2a99e3083..ee403016e 100644--- a/crates/uzu/tests/unit/kernel/rms_norm_test.rs+++ b/crates/uzu/tests/unit/kernel/rms_norm_test.rs@@ -126,6 +126,7 @@ fn get_output<OutputT::data_type(),AccumT::data_type(),input.in_place,+ false,).expect("Failed to create RMSNormKernel");@@ -147,6 +148,7 @@ fn get_output<input_buffer,scales_array.buffer().borrow().deref(),output_array.buffer().borrow_mut().deref_mut(),+ None::<&mut B::Buffer>,input.batch_size,input.element_count,input.epsilon,@@ -177,7 +179,7 @@ fn test_internal<|| matches!(OutputT::data_type(), DataType::F16 | DataType::BF16)|| matches!(AccumT::data_type(), DataType::F16 | DataType::BF16){- 0.012f32+ 0.016f32} else {1e-5};