A high-performance inference engine for AI models
Fused tensoraddswap+rmsnorm (#320)
uuuvn57f1cdeb3988
diff --git a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rsindex 6a1a73a3d..352a6ed2c 100644--- a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs+++ b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs@@ -26,6 +26,7 @@ pub fn rms_norm<full_layer: bool,#[specialize] in_place: bool,#[specialize] copy_to_shortcut: bool,+ #[specialize] residual_add: bool,) {let input = match in_place {true => output as *const InputT,@@ -42,22 +43,31 @@ pub fn rms_norm<for batch in 0..(batch_size as usize) {let batch_offset = batch * element_count;- // Compute rms_inv+ // Compute rms_inv (+ residual add or copy to shortcut)let mut sum_sq = AccumT::zero();for i in 0..element_count {let input_val = unsafe { *input.add(batch_offset + i) };- let accum_val = AccumT::from(input_val).unwrap();- sum_sq = sum_sq + accum_val * accum_val;+ let mut val = input_val;if copy_to_shortcut {- unsafe { *shortcut_buffer.unwrap().add(batch_offset + i) = input_val };+ let skip_ptr = unsafe { shortcut_buffer.unwrap().add(batch_offset + i) };+ if residual_add {+ val = val + unsafe { *skip_ptr };+ }+ unsafe { *skip_ptr = val };}+ let accum_val = AccumT::from(val).unwrap();+ sum_sq = sum_sq + accum_val * accum_val;}let mean_sq: AccumT = sum_sq / element_count_accum;let rms_inv = AccumT::from((mean_sq + epsilon).to_f32().unwrap().sqrt().recip()).unwrap();// Normalization and scalingfor i in 0..element_count {- let input_val = unsafe { AccumT::from(*input.add(batch_offset + i)).unwrap() };+ let input_val = if residual_add {+ unsafe { AccumT::from(*shortcut_buffer.unwrap().add(batch_offset + i)).unwrap() }+ } else {+ unsafe { AccumT::from(*input.add(batch_offset + i)).unwrap() }+ };let scale_val = unsafe { AccumT::from(*scales.add(i)).unwrap() };let normalized: AccumT = input_val * rms_inv;let result: OutputT = if full_layer {diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex 7615d5690..1ebabc916 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -26,6 +26,7 @@ PUBLIC KERNEL(RMSNorm)(constant bool& full_layer,const bool in_place SPECIALIZE,const bool copy_to_shortcut SPECIALIZE,+ const bool residual_add SPECIALIZE,threadgroup AccumT shared_sum[METAL_SIMD_SIZE],const ThreadContext thread_context,const uint batch_idx GROUPS(batch_size),@@ -42,7 +43,7 @@ PUBLIC KERNEL(RMSNorm)(AccumT partial_sum = static_cast<AccumT>(0.0f);- // Compute thread local partial sum (+ copy to shortcut if enabled)+ // Compute thread local partial sum (+ residual add or copy to shortcut)for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;base_i += BLOCK_SIZE * GRAIN_SIZE) {AccumT vals[GRAIN_SIZE];@@ -51,10 +52,13 @@ PUBLIC KERNEL(RMSNorm)(uint i = base_i + j;if (i < element_count) {InputT val = input_data[i];- vals[j] = static_cast<AccumT>(val);if (copy_to_shortcut) {+ if (residual_add) {+ val = val + shortcut_buffer[input_offset + i];+ }shortcut_buffer[input_offset + i] = val;}+ vals[j] = static_cast<AccumT>(val);} else {vals[j] = 0.0f;}@@ -84,10 +88,18 @@ PUBLIC KERNEL(RMSNorm)(AccumT vals[GRAIN_SIZE];AccumT scaled_vals[GRAIN_SIZE];- // Load GRAIN_SIZE input elements+ // Load GRAIN_SIZE input elements (from shortcut if residual_add, since it+ // has the sum)for (uint j = 0; j < GRAIN_SIZE; ++j) {uint i = base_i + j;- vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;+ if (residual_add) {+ vals[j] = (i < element_count)+ ? static_cast<AccumT>(shortcut_buffer[input_offset + i])+ : 0.0f;+ } else {+ vals[j] =+ (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;+ }}// Process GRAIN_SIZE elements: normalize and scalediff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rsindex fa64cadee..8dd52858f 100644--- a/crates/uzu/src/encodable_block/decoder.rs+++ b/crates/uzu/src/encodable_block/decoder.rs@@ -187,7 +187,8 @@ impl<B: Backend> Decoder<B> {ArrayId::Main,ArrayId::Main,&decoder_weight_loader.subtree("output_norm").unwrap(),- None,+ Some(ArrayId::Shortcut),+ true,).map(RMSNorm::with_sampling_range).expect("Failed to create output RMS norm kernel");diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex 7056a0ef4..8f58dda5c 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -1,15 +1,18 @@//! Layer executables - a single decoder layer with mixer, norms, and MLP.+#[cfg(feature = "tracing")]+use std::ops::{Deref, DerefMut};use std::rc::Rc;use super::MixerExecutables;+#[cfg(feature = "tracing")]+use crate::backends::common::{Kernels, kernel::TensorAddBiasKernel};use crate::{DataType,backends::common::{Backend, Encoder},config::{DecoderLayerConfig, DecoderLayerType, MixerConfig},encodable_block::{Attention, DeltaNetMixer, EncodingParameters, Linear, MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvMixer,- TensorAddSwap,},forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,@@ -19,10 +22,11 @@ use crate::{pub struct LayerExecutables<B: Backend> {#[cfg(feature = "tracing")]pub layer_index: usize,+ #[cfg(feature = "tracing")]+ pub tensor_add: <B::Kernels as Kernels>::TensorAddBiasKernel,pub pre_attention_norm: RMSNorm<B>,pub(crate) mixer: MixerExecutables<B>,pub post_attention_norm: Option<RMSNorm<B>>,- pub main_shortcut_add_swap: TensorAddSwap<B>,pub pre_mlp_norm: RMSNorm<B>,pub mlp: Box<dyn Mlp<B>>,pub post_mlp_norm: Option<RMSNorm<B>>,@@ -50,11 +54,11 @@ impl<B: Backend> LayerExecutables<B> {MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),MixerConfig::DeltaNet(config) => config.in_proj_config.activation_precision().into(),};- let shortcut_copy = if layer_index == 0 {- Some(ArrayId::Shortcut)- } else {- None- };++ #[cfg(feature = "tracing")]+ let tensor_add = TensorAddBiasKernel::new(context, intermediate_data_type, false)+ .expect("Failed to create TensorAddBiasKernel kernel"); // TODO: this function return Result+let pre_attention_norm = RMSNorm::new(context,intermediate_data_type,@@ -62,7 +66,8 @@ impl<B: Backend> LayerExecutables<B> {ArrayId::Main,ArrayId::Main,&decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),- shortcut_copy,+ Some(ArrayId::Shortcut),+ layer_index > 0,).expect("Failed to create RMS norm kernel");@@ -215,6 +220,7 @@ impl<B: Backend> LayerExecutables<B> {ArrayId::Main,&decoder_layer_loader.subtree("post_mixer_norm").unwrap(),None,+ false,).expect("Failed to create RMS norm kernel"),)@@ -222,9 +228,6 @@ impl<B: Backend> LayerExecutables<B> {None};- let main_shortcut_add_swap =- TensorAddSwap::<B>::new(context, intermediate_data_type, ArrayId::Shortcut, ArrayId::Main).unwrap();-let pre_mlp_norm = RMSNorm::new(context,intermediate_data_type,@@ -232,7 +235,8 @@ impl<B: Backend> LayerExecutables<B> {ArrayId::Main,ArrayId::Main,&decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),- None,+ Some(ArrayId::Shortcut),+ true,).expect("Failed to create RMS norm kernel");@@ -255,6 +259,7 @@ impl<B: Backend> LayerExecutables<B> {ArrayId::Main,&decoder_layer_loader.subtree("post_mlp_norm").unwrap(),None,+ false,).expect("Failed to create RMS norm kernel"),)@@ -265,10 +270,11 @@ impl<B: Backend> LayerExecutables<B> {Self {#[cfg(feature = "tracing")]layer_index,+ #[cfg(feature = "tracing")]+ tensor_add,pre_attention_norm,mixer,post_attention_norm,- main_shortcut_add_swap,pre_mlp_norm,mlp,post_mlp_norm,@@ -284,14 +290,10 @@ impl<B: Backend> LayerExecutables<B> {#[cfg(feature = "tracing")]let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();- #[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().inputs.clone());- }-self.pre_attention_norm.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(encoder, ArrayId::Shortcut, layer_traces.borrow().inputs.clone());state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());}@@ -355,19 +357,13 @@ impl<B: Backend> LayerExecutables<B> {state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_attention_norm.clone());}}- //main = attention_result-- self.main_shortcut_add_swap.encode(state, encoder)?;- // shortcut = input + attention_result- // main = input + attention_result- #[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());- }+ // main = attention_resultself.pre_mlp_norm.encode(state, encoder)?;+ // shortcut = attention_result + shortcut; main = normalized(shortcut)#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(encoder, ArrayId::Shortcut, layer_traces.borrow().mlp_inputs.clone());state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());}@@ -385,13 +381,22 @@ impl<B: Backend> LayerExecutables<B> {}}// main = mlp_result+ // next layer's pre_attention_norm (or output_norm) fuses the residual add- self.main_shortcut_add_swap.encode(state, encoder)?;- // shortcut = input + attention_result + mlp_result- // main = input + attention_result + mlp_result#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().outputs.clone());+ let size = state.array(ArrayId::Main).shape().into_iter().copied().product::<usize>() as u32;+ let input_a = state.array(ArrayId::Main).buffer();+ let input_b = state.array(ArrayId::Shortcut).buffer();+ let output = layer_traces.borrow().outputs.buffer();+ self.tensor_add.encode(+ Some(input_a.borrow().deref()),+ input_b.borrow().deref(),+ output.borrow_mut().deref_mut(),+ size,+ size,+ encoder,+ );}Ok(())diff --git a/crates/uzu/src/encodable_block/normalization.rs b/crates/uzu/src/encodable_block/normalization.rsindex bf0490b84..1c6fcf34b 100644--- a/crates/uzu/src/encodable_block/normalization.rs+++ b/crates/uzu/src/encodable_block/normalization.rs@@ -54,6 +54,7 @@ impl<B: Backend> Normalization<B> {output_array_id,parameter_tree,None,+ false,)?))}}diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rsindex c82db8e58..16cc25b9f 100644--- a/crates/uzu/src/encodable_block/rms_norm.rs+++ b/crates/uzu/src/encodable_block/rms_norm.rs@@ -46,6 +46,7 @@ impl<B: Backend> RMSNorm<B> {output_array_id: ArrayId,parameter_tree: &ParameterTree<B::Context>,shortcut_array_id: Option<ArrayId>,+ residual_add: bool,) -> Result<Self, RMSNormError<B>> {let scales = parameter_tree.leaf_array("scales").map_err(RMSNormError::ParameterError)?;@@ -65,6 +66,7 @@ impl<B: Backend> RMSNorm<B> {accumulation_data_type,input_array_id == output_array_id,shortcut_array_id.is_some(),+ residual_add,).map_err(RMSNormError::BackendError)?;@@ -127,7 +129,7 @@ impl<B: Backend> RMSNorm<B> {input_buffer_borrow.as_deref().map(|b| (b, input_offset)),self.scales_buffer.borrow().deref(),(output_array.buffer().borrow_mut().deref_mut(), output_offset),- shortcut_borrow.as_deref_mut(),+ shortcut_borrow.as_deref_mut().map(|b| (b, input_offset)),batch_len as u32,element_count as u32,self.config.epsilon,diff --git a/crates/uzu/src/session/tts_session/decoder_runtime.rs b/crates/uzu/src/session/tts_session/decoder_runtime.rsindex 9e0d49386..b988e2c96 100644--- a/crates/uzu/src/session/tts_session/decoder_runtime.rs+++ b/crates/uzu/src/session/tts_session/decoder_runtime.rs@@ -743,10 +743,11 @@ impl<B: Backend> TokenDecoderRunner<B> {.encode(&mut state, &encoding_parameters, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;}+ self.ctx.executables.norm.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;if capture_hidden {+ // After output norm with residual_add, Shortcut holds the un-normalized full residualself.encode_capture_last_hidden_into_single_buffer_on(encoder, &state, token_count)?;}- self.ctx.executables.norm.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;self.ctx.executables.embed@@ -923,8 +924,8 @@ impl<B: Backend> TokenDecoderRunner<B> {let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {model_dim,})?;- let main = state.array(ArrayId::Main);- let bytes_per_element = main.data_type().size_in_bytes();+ let shortcut = state.array(ArrayId::Shortcut);+ let bytes_per_element = shortcut.data_type().size_in_bytes();let row_offset = (token_count - 1).checked_mul(model_dim).and_then(|value| value.checked_mul(bytes_per_element))@@ -933,23 +934,23 @@ impl<B: Backend> TokenDecoderRunner<B> {model_dim,})?;let src_offset =- main.offset().checked_add(row_offset).ok_or(TtsModelConfigError::HiddenCaptureSourceOffsetOverflow)?;+ shortcut.offset().checked_add(row_offset).ok_or(TtsModelConfigError::HiddenCaptureSourceOffsetOverflow)?;let capture = &self.single_hidden_capture;- if capture.shape() != [1, model_dim] || capture.data_type() != main.data_type() {+ if capture.shape() != [1, model_dim] || capture.data_type() != shortcut.data_type() {return Err(TtsModelConfigError::HiddenCaptureTensorMismatch {expected_shape: [1, model_dim].into(),- expected_data_type: main.data_type(),+ expected_data_type: shortcut.data_type(),actual_shape: capture.shape().into(),actual_data_type: capture.data_type(),}.into());}- let main_buffer = main.buffer();- let main_buffer = main_buffer.borrow();+ let shortcut_buffer = shortcut.buffer();+ let shortcut_buffer = shortcut_buffer.borrow();let capture_buffer = capture.buffer();let mut capture_buffer = capture_buffer.borrow_mut();- self.tensor_copy.encode((&*main_buffer, src_offset), &mut *capture_buffer, model_dim_u32, encoder);+ self.tensor_copy.encode((&*shortcut_buffer, src_offset), &mut *capture_buffer, model_dim_u32, encoder);Ok(())}diff --git a/crates/uzu/tests/unit/kernel/rms_norm_test.rs b/crates/uzu/tests/unit/kernel/rms_norm_test.rsindex ee403016e..fb779c3cc 100644--- a/crates/uzu/tests/unit/kernel/rms_norm_test.rs+++ b/crates/uzu/tests/unit/kernel/rms_norm_test.rs@@ -127,6 +127,7 @@ fn get_output<AccumT::data_type(),input.in_place,false,+ false,).expect("Failed to create RMSNormKernel");