Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Fused tensoraddswap+rmsnorm (#320)

uuuvn57f1cdeb3988
diff --git a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
index 6a1a73a3d..352a6ed2c 100644
--- a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
+++ b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
@@ -26,6 +26,7 @@ pub fn rms_norm<
full_layer: bool,
#[specialize] in_place: bool,
#[specialize] copy_to_shortcut: bool,
+ #[specialize] residual_add: bool,
) {
let input = match in_place {
true => output as *const InputT,
@@ -42,22 +43,31 @@ pub fn rms_norm<
for batch in 0..(batch_size as usize) {
let batch_offset = batch * element_count;
- // Compute rms_inv
+ // Compute rms_inv (+ residual add or copy to shortcut)
let mut sum_sq = AccumT::zero();
for i in 0..element_count {
let input_val = unsafe { *input.add(batch_offset + i) };
- let accum_val = AccumT::from(input_val).unwrap();
- sum_sq = sum_sq + accum_val * accum_val;
+ let mut val = input_val;
if copy_to_shortcut {
- unsafe { *shortcut_buffer.unwrap().add(batch_offset + i) = input_val };
+ let skip_ptr = unsafe { shortcut_buffer.unwrap().add(batch_offset + i) };
+ if residual_add {
+ val = val + unsafe { *skip_ptr };
+ }
+ unsafe { *skip_ptr = val };
}
+ let accum_val = AccumT::from(val).unwrap();
+ sum_sq = sum_sq + accum_val * accum_val;
}
let mean_sq: AccumT = sum_sq / element_count_accum;
let rms_inv = AccumT::from((mean_sq + epsilon).to_f32().unwrap().sqrt().recip()).unwrap();
// Normalization and scaling
for i in 0..element_count {
- let input_val = unsafe { AccumT::from(*input.add(batch_offset + i)).unwrap() };
+ let input_val = if residual_add {
+ unsafe { AccumT::from(*shortcut_buffer.unwrap().add(batch_offset + i)).unwrap() }
+ } else {
+ unsafe { AccumT::from(*input.add(batch_offset + i)).unwrap() }
+ };
let scale_val = unsafe { AccumT::from(*scales.add(i)).unwrap() };
let normalized: AccumT = input_val * rms_inv;
let result: OutputT = if full_layer {
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index 7615d5690..1ebabc916 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -26,6 +26,7 @@ PUBLIC KERNEL(RMSNorm)(
constant bool& full_layer,
const bool in_place SPECIALIZE,
const bool copy_to_shortcut SPECIALIZE,
+ const bool residual_add SPECIALIZE,
threadgroup AccumT shared_sum[METAL_SIMD_SIZE],
const ThreadContext thread_context,
const uint batch_idx GROUPS(batch_size),
@@ -42,7 +43,7 @@ PUBLIC KERNEL(RMSNorm)(
AccumT partial_sum = static_cast<AccumT>(0.0f);
- // Compute thread local partial sum (+ copy to shortcut if enabled)
+ // Compute thread local partial sum (+ residual add or copy to shortcut)
for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;
base_i += BLOCK_SIZE * GRAIN_SIZE) {
AccumT vals[GRAIN_SIZE];
@@ -51,10 +52,13 @@ PUBLIC KERNEL(RMSNorm)(
uint i = base_i + j;
if (i < element_count) {
InputT val = input_data[i];
- vals[j] = static_cast<AccumT>(val);
if (copy_to_shortcut) {
+ if (residual_add) {
+ val = val + shortcut_buffer[input_offset + i];
+ }
shortcut_buffer[input_offset + i] = val;
}
+ vals[j] = static_cast<AccumT>(val);
} else {
vals[j] = 0.0f;
}
@@ -84,10 +88,18 @@ PUBLIC KERNEL(RMSNorm)(
AccumT vals[GRAIN_SIZE];
AccumT scaled_vals[GRAIN_SIZE];
- // Load GRAIN_SIZE input elements
+ // Load GRAIN_SIZE input elements (from shortcut if residual_add, since it
+ // has the sum)
for (uint j = 0; j < GRAIN_SIZE; ++j) {
uint i = base_i + j;
- vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
+ if (residual_add) {
+ vals[j] = (i < element_count)
+ ? static_cast<AccumT>(shortcut_buffer[input_offset + i])
+ : 0.0f;
+ } else {
+ vals[j] =
+ (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
+ }
}
// Process GRAIN_SIZE elements: normalize and scale
diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rs
index fa64cadee..8dd52858f 100644
--- a/crates/uzu/src/encodable_block/decoder.rs
+++ b/crates/uzu/src/encodable_block/decoder.rs
@@ -187,7 +187,8 @@ impl<B: Backend> Decoder<B> {
ArrayId::Main,
ArrayId::Main,
&decoder_weight_loader.subtree("output_norm").unwrap(),
- None,
+ Some(ArrayId::Shortcut),
+ true,
)
.map(RMSNorm::with_sampling_range)
.expect("Failed to create output RMS norm kernel");
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index 7056a0ef4..8f58dda5c 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -1,15 +1,18 @@
//! Layer executables - a single decoder layer with mixer, norms, and MLP.
+#[cfg(feature = "tracing")]
+use std::ops::{Deref, DerefMut};
use std::rc::Rc;
use super::MixerExecutables;
+#[cfg(feature = "tracing")]
+use crate::backends::common::{Kernels, kernel::TensorAddBiasKernel};
use crate::{
DataType,
backends::common::{Backend, Encoder},
config::{DecoderLayerConfig, DecoderLayerType, MixerConfig},
encodable_block::{
Attention, DeltaNetMixer, EncodingParameters, Linear, MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvMixer,
- TensorAddSwap,
},
forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
@@ -19,10 +22,11 @@ use crate::{
pub struct LayerExecutables<B: Backend> {
#[cfg(feature = "tracing")]
pub layer_index: usize,
+ #[cfg(feature = "tracing")]
+ pub tensor_add: <B::Kernels as Kernels>::TensorAddBiasKernel,
pub pre_attention_norm: RMSNorm<B>,
pub(crate) mixer: MixerExecutables<B>,
pub post_attention_norm: Option<RMSNorm<B>>,
- pub main_shortcut_add_swap: TensorAddSwap<B>,
pub pre_mlp_norm: RMSNorm<B>,
pub mlp: Box<dyn Mlp<B>>,
pub post_mlp_norm: Option<RMSNorm<B>>,
@@ -50,11 +54,11 @@ impl<B: Backend> LayerExecutables<B> {
MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),
MixerConfig::DeltaNet(config) => config.in_proj_config.activation_precision().into(),
};
- let shortcut_copy = if layer_index == 0 {
- Some(ArrayId::Shortcut)
- } else {
- None
- };
+
+ #[cfg(feature = "tracing")]
+ let tensor_add = TensorAddBiasKernel::new(context, intermediate_data_type, false)
+ .expect("Failed to create TensorAddBiasKernel kernel"); // TODO: this function return Result
+
let pre_attention_norm = RMSNorm::new(
context,
intermediate_data_type,
@@ -62,7 +66,8 @@ impl<B: Backend> LayerExecutables<B> {
ArrayId::Main,
ArrayId::Main,
&decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),
- shortcut_copy,
+ Some(ArrayId::Shortcut),
+ layer_index > 0,
)
.expect("Failed to create RMS norm kernel");
@@ -215,6 +220,7 @@ impl<B: Backend> LayerExecutables<B> {
ArrayId::Main,
&decoder_layer_loader.subtree("post_mixer_norm").unwrap(),
None,
+ false,
)
.expect("Failed to create RMS norm kernel"),
)
@@ -222,9 +228,6 @@ impl<B: Backend> LayerExecutables<B> {
None
};
- let main_shortcut_add_swap =
- TensorAddSwap::<B>::new(context, intermediate_data_type, ArrayId::Shortcut, ArrayId::Main).unwrap();
-
let pre_mlp_norm = RMSNorm::new(
context,
intermediate_data_type,
@@ -232,7 +235,8 @@ impl<B: Backend> LayerExecutables<B> {
ArrayId::Main,
ArrayId::Main,
&decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),
- None,
+ Some(ArrayId::Shortcut),
+ true,
)
.expect("Failed to create RMS norm kernel");
@@ -255,6 +259,7 @@ impl<B: Backend> LayerExecutables<B> {
ArrayId::Main,
&decoder_layer_loader.subtree("post_mlp_norm").unwrap(),
None,
+ false,
)
.expect("Failed to create RMS norm kernel"),
)
@@ -265,10 +270,11 @@ impl<B: Backend> LayerExecutables<B> {
Self {
#[cfg(feature = "tracing")]
layer_index,
+ #[cfg(feature = "tracing")]
+ tensor_add,
pre_attention_norm,
mixer,
post_attention_norm,
- main_shortcut_add_swap,
pre_mlp_norm,
mlp,
post_mlp_norm,
@@ -284,14 +290,10 @@ impl<B: Backend> LayerExecutables<B> {
#[cfg(feature = "tracing")]
let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();
- #[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().inputs.clone());
- }
-
self.pre_attention_norm.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(encoder, ArrayId::Shortcut, layer_traces.borrow().inputs.clone());
state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());
}
@@ -355,19 +357,13 @@ impl<B: Backend> LayerExecutables<B> {
state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_attention_norm.clone());
}
}
- //main = attention_result
-
- self.main_shortcut_add_swap.encode(state, encoder)?;
- // shortcut = input + attention_result
- // main = input + attention_result
- #[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());
- }
+ // main = attention_result
self.pre_mlp_norm.encode(state, encoder)?;
+ // shortcut = attention_result + shortcut; main = normalized(shortcut)
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(encoder, ArrayId::Shortcut, layer_traces.borrow().mlp_inputs.clone());
state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());
}
@@ -385,13 +381,22 @@ impl<B: Backend> LayerExecutables<B> {
}
}
// main = mlp_result
+ // next layer's pre_attention_norm (or output_norm) fuses the residual add
- self.main_shortcut_add_swap.encode(state, encoder)?;
- // shortcut = input + attention_result + mlp_result
- // main = input + attention_result + mlp_result
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().outputs.clone());
+ let size = state.array(ArrayId::Main).shape().into_iter().copied().product::<usize>() as u32;
+ let input_a = state.array(ArrayId::Main).buffer();
+ let input_b = state.array(ArrayId::Shortcut).buffer();
+ let output = layer_traces.borrow().outputs.buffer();
+ self.tensor_add.encode(
+ Some(input_a.borrow().deref()),
+ input_b.borrow().deref(),
+ output.borrow_mut().deref_mut(),
+ size,
+ size,
+ encoder,
+ );
}
Ok(())
diff --git a/crates/uzu/src/encodable_block/normalization.rs b/crates/uzu/src/encodable_block/normalization.rs
index bf0490b84..1c6fcf34b 100644
--- a/crates/uzu/src/encodable_block/normalization.rs
+++ b/crates/uzu/src/encodable_block/normalization.rs
@@ -54,6 +54,7 @@ impl<B: Backend> Normalization<B> {
output_array_id,
parameter_tree,
None,
+ false,
)?))
}
}
diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rs
index c82db8e58..16cc25b9f 100644
--- a/crates/uzu/src/encodable_block/rms_norm.rs
+++ b/crates/uzu/src/encodable_block/rms_norm.rs
@@ -46,6 +46,7 @@ impl<B: Backend> RMSNorm<B> {
output_array_id: ArrayId,
parameter_tree: &ParameterTree<B::Context>,
shortcut_array_id: Option<ArrayId>,
+ residual_add: bool,
) -> Result<Self, RMSNormError<B>> {
let scales = parameter_tree.leaf_array("scales").map_err(RMSNormError::ParameterError)?;
@@ -65,6 +66,7 @@ impl<B: Backend> RMSNorm<B> {
accumulation_data_type,
input_array_id == output_array_id,
shortcut_array_id.is_some(),
+ residual_add,
)
.map_err(RMSNormError::BackendError)?;
@@ -127,7 +129,7 @@ impl<B: Backend> RMSNorm<B> {
input_buffer_borrow.as_deref().map(|b| (b, input_offset)),
self.scales_buffer.borrow().deref(),
(output_array.buffer().borrow_mut().deref_mut(), output_offset),
- shortcut_borrow.as_deref_mut(),
+ shortcut_borrow.as_deref_mut().map(|b| (b, input_offset)),
batch_len as u32,
element_count as u32,
self.config.epsilon,
diff --git a/crates/uzu/src/session/tts_session/decoder_runtime.rs b/crates/uzu/src/session/tts_session/decoder_runtime.rs
index 9e0d49386..b988e2c96 100644
--- a/crates/uzu/src/session/tts_session/decoder_runtime.rs
+++ b/crates/uzu/src/session/tts_session/decoder_runtime.rs
@@ -743,10 +743,11 @@ impl<B: Backend> TokenDecoderRunner<B> {
.encode(&mut state, &encoding_parameters, encoder)
.map_err(|err| Error::EncodeFailed(Box::new(err)))?;
}
+ self.ctx.executables.norm.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;
if capture_hidden {
+ // After output norm with residual_add, Shortcut holds the un-normalized full residual
self.encode_capture_last_hidden_into_single_buffer_on(encoder, &state, token_count)?;
}
- self.ctx.executables.norm.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;
self.ctx
.executables
.embed
@@ -923,8 +924,8 @@ impl<B: Backend> TokenDecoderRunner<B> {
let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {
model_dim,
})?;
- let main = state.array(ArrayId::Main);
- let bytes_per_element = main.data_type().size_in_bytes();
+ let shortcut = state.array(ArrayId::Shortcut);
+ let bytes_per_element = shortcut.data_type().size_in_bytes();
let row_offset = (token_count - 1)
.checked_mul(model_dim)
.and_then(|value| value.checked_mul(bytes_per_element))
@@ -933,23 +934,23 @@ impl<B: Backend> TokenDecoderRunner<B> {
model_dim,
})?;
let src_offset =
- main.offset().checked_add(row_offset).ok_or(TtsModelConfigError::HiddenCaptureSourceOffsetOverflow)?;
+ shortcut.offset().checked_add(row_offset).ok_or(TtsModelConfigError::HiddenCaptureSourceOffsetOverflow)?;
let capture = &self.single_hidden_capture;
- if capture.shape() != [1, model_dim] || capture.data_type() != main.data_type() {
+ if capture.shape() != [1, model_dim] || capture.data_type() != shortcut.data_type() {
return Err(TtsModelConfigError::HiddenCaptureTensorMismatch {
expected_shape: [1, model_dim].into(),
- expected_data_type: main.data_type(),
+ expected_data_type: shortcut.data_type(),
actual_shape: capture.shape().into(),
actual_data_type: capture.data_type(),
}
.into());
}
- let main_buffer = main.buffer();
- let main_buffer = main_buffer.borrow();
+ let shortcut_buffer = shortcut.buffer();
+ let shortcut_buffer = shortcut_buffer.borrow();
let capture_buffer = capture.buffer();
let mut capture_buffer = capture_buffer.borrow_mut();
- self.tensor_copy.encode((&*main_buffer, src_offset), &mut *capture_buffer, model_dim_u32, encoder);
+ self.tensor_copy.encode((&*shortcut_buffer, src_offset), &mut *capture_buffer, model_dim_u32, encoder);
Ok(())
}
diff --git a/crates/uzu/tests/unit/kernel/rms_norm_test.rs b/crates/uzu/tests/unit/kernel/rms_norm_test.rs
index ee403016e..fb779c3cc 100644
--- a/crates/uzu/tests/unit/kernel/rms_norm_test.rs
+++ b/crates/uzu/tests/unit/kernel/rms_norm_test.rs
@@ -127,6 +127,7 @@ fn get_output<
AccumT::data_type(),
input.in_place,
false,
+ false,
)
.expect("Failed to create RMSNormKernel");