Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Fused tensorcopy+rmsnorm (#319)

uuuvne6f9e3c48289
diff --git a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
index 8099202cb..6a1a73a3d 100644
--- a/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
+++ b/crates/uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
@@ -18,12 +18,14 @@ pub fn rms_norm<
#[optional(!in_place)] input: Option<*const InputT>,
scales: *const ScaleT,
output: *mut OutputT,
+ #[optional(copy_to_shortcut)] shortcut_buffer: Option<*mut InputT>,
batch_size: u32,
element_count: u32,
epsilon: f32,
scale_offset: f32,
full_layer: bool,
#[specialize] in_place: bool,
+ #[specialize] copy_to_shortcut: bool,
) {
let input = match in_place {
true => output as *const InputT,
@@ -43,8 +45,12 @@ pub fn rms_norm<
// Compute rms_inv
let mut sum_sq = AccumT::zero();
for i in 0..element_count {
- let input_val = unsafe { AccumT::from(*input.add(batch_offset + i)).unwrap() };
- sum_sq = sum_sq + input_val * input_val;
+ let input_val = unsafe { *input.add(batch_offset + i) };
+ let accum_val = AccumT::from(input_val).unwrap();
+ sum_sq = sum_sq + accum_val * accum_val;
+ if copy_to_shortcut {
+ unsafe { *shortcut_buffer.unwrap().add(batch_offset + i) = input_val };
+ }
}
let mean_sq: AccumT = sum_sq / element_count_accum;
let rms_inv = AccumT::from((mean_sq + epsilon).to_f32().unwrap().sqrt().recip()).unwrap();
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index 79e1e57b9..7615d5690 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -18,12 +18,14 @@ PUBLIC KERNEL(RMSNorm)(
const device InputT* input OPTIONAL(!in_place),
const device ScaleT* scales,
device OutputT* output,
+ device InputT* shortcut_buffer OPTIONAL(copy_to_shortcut),
constant uint& batch_size,
constant uint& element_count,
constant float& epsilon,
constant float& scale_offset,
constant bool& full_layer,
const bool in_place SPECIALIZE,
+ const bool copy_to_shortcut SPECIALIZE,
threadgroup AccumT shared_sum[METAL_SIMD_SIZE],
const ThreadContext thread_context,
const uint batch_idx GROUPS(batch_size),
@@ -40,14 +42,22 @@ PUBLIC KERNEL(RMSNorm)(
AccumT partial_sum = static_cast<AccumT>(0.0f);
- // Compute thread local partial sum
+ // Compute thread local partial sum (+ copy to shortcut if enabled)
for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;
base_i += BLOCK_SIZE * GRAIN_SIZE) {
AccumT vals[GRAIN_SIZE];
for (uint j = 0; j < GRAIN_SIZE; ++j) {
uint i = base_i + j;
- vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
+ if (i < element_count) {
+ InputT val = input_data[i];
+ vals[j] = static_cast<AccumT>(val);
+ if (copy_to_shortcut) {
+ shortcut_buffer[input_offset + i] = val;
+ }
+ } else {
+ vals[j] = 0.0f;
+ }
}
for (uint j = 0; j < GRAIN_SIZE; ++j) {
diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rs
index 4fa6ac27b..fa64cadee 100644
--- a/crates/uzu/src/encodable_block/decoder.rs
+++ b/crates/uzu/src/encodable_block/decoder.rs
@@ -187,6 +187,7 @@ impl<B: Backend> Decoder<B> {
ArrayId::Main,
ArrayId::Main,
&decoder_weight_loader.subtree("output_norm").unwrap(),
+ None,
)
.map(RMSNorm::with_sampling_range)
.expect("Failed to create output RMS norm kernel");
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index e9ca29161..7056a0ef4 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -9,7 +9,7 @@ use crate::{
config::{DecoderLayerConfig, DecoderLayerType, MixerConfig},
encodable_block::{
Attention, DeltaNetMixer, EncodingParameters, Linear, MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvMixer,
- TensorAddSwap, TensorCopy,
+ TensorAddSwap,
},
forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
@@ -19,7 +19,6 @@ use crate::{
pub struct LayerExecutables<B: Backend> {
#[cfg(feature = "tracing")]
pub layer_index: usize,
- pub copy_main_to_shortcut: TensorCopy<B>,
pub pre_attention_norm: RMSNorm<B>,
pub(crate) mixer: MixerExecutables<B>,
pub post_attention_norm: Option<RMSNorm<B>>,
@@ -51,9 +50,11 @@ impl<B: Backend> LayerExecutables<B> {
MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),
MixerConfig::DeltaNet(config) => config.in_proj_config.activation_precision().into(),
};
- let copy_main_to_shortcut =
- TensorCopy::<B>::new(context, intermediate_data_type, ArrayId::Main, ArrayId::Shortcut).unwrap();
-
+ let shortcut_copy = if layer_index == 0 {
+ Some(ArrayId::Shortcut)
+ } else {
+ None
+ };
let pre_attention_norm = RMSNorm::new(
context,
intermediate_data_type,
@@ -61,6 +62,7 @@ impl<B: Backend> LayerExecutables<B> {
ArrayId::Main,
ArrayId::Main,
&decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),
+ shortcut_copy,
)
.expect("Failed to create RMS norm kernel");
@@ -212,6 +214,7 @@ impl<B: Backend> LayerExecutables<B> {
ArrayId::Main,
ArrayId::Main,
&decoder_layer_loader.subtree("post_mixer_norm").unwrap(),
+ None,
)
.expect("Failed to create RMS norm kernel"),
)
@@ -229,6 +232,7 @@ impl<B: Backend> LayerExecutables<B> {
ArrayId::Main,
ArrayId::Main,
&decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),
+ None,
)
.expect("Failed to create RMS norm kernel");
@@ -250,6 +254,7 @@ impl<B: Backend> LayerExecutables<B> {
ArrayId::Main,
ArrayId::Main,
&decoder_layer_loader.subtree("post_mlp_norm").unwrap(),
+ None,
)
.expect("Failed to create RMS norm kernel"),
)
@@ -260,7 +265,6 @@ impl<B: Backend> LayerExecutables<B> {
Self {
#[cfg(feature = "tracing")]
layer_index,
- copy_main_to_shortcut,
pre_attention_norm,
mixer,
post_attention_norm,
@@ -285,9 +289,6 @@ impl<B: Backend> LayerExecutables<B> {
state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().inputs.clone());
}
- self.copy_main_to_shortcut.encode(state, encoder)?;
- // shortcut = input
-
self.pre_attention_norm.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
diff --git a/crates/uzu/src/encodable_block/normalization.rs b/crates/uzu/src/encodable_block/normalization.rs
index 7c99d0873..bf0490b84 100644
--- a/crates/uzu/src/encodable_block/normalization.rs
+++ b/crates/uzu/src/encodable_block/normalization.rs
@@ -53,6 +53,7 @@ impl<B: Backend> Normalization<B> {
input_array_id,
output_array_id,
parameter_tree,
+ None,
)?))
}
}
diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rs
index 64514c5fa..c82db8e58 100644
--- a/crates/uzu/src/encodable_block/rms_norm.rs
+++ b/crates/uzu/src/encodable_block/rms_norm.rs
@@ -32,6 +32,7 @@ pub struct RMSNorm<B: Backend> {
config: NormalizationConfig,
input_array_id: ArrayId,
output_array_id: ArrayId,
+ shortcut_array_id: Option<ArrayId>,
scales_buffer: Rc<RefCell<B::Buffer>>,
use_sampling_range: bool,
}
@@ -44,6 +45,7 @@ impl<B: Backend> RMSNorm<B> {
input_array_id: ArrayId,
output_array_id: ArrayId,
parameter_tree: &ParameterTree<B::Context>,
+ shortcut_array_id: Option<ArrayId>,
) -> Result<Self, RMSNormError<B>> {
let scales = parameter_tree.leaf_array("scales").map_err(RMSNormError::ParameterError)?;
@@ -62,6 +64,7 @@ impl<B: Backend> RMSNorm<B> {
output_type,
accumulation_data_type,
input_array_id == output_array_id,
+ shortcut_array_id.is_some(),
)
.map_err(RMSNormError::BackendError)?;
@@ -70,6 +73,7 @@ impl<B: Backend> RMSNorm<B> {
config,
input_array_id,
output_array_id,
+ shortcut_array_id,
scales_buffer: scales.buffer(),
use_sampling_range: false,
})
@@ -116,10 +120,14 @@ impl<B: Backend> RMSNorm<B> {
let input_buffer = (self.input_array_id != self.output_array_id).then(|| input_array.buffer());
let input_buffer_borrow = input_buffer.as_ref().map(|b| b.borrow());
+ let shortcut_rc = self.shortcut_array_id.map(|id| state.array(id).buffer());
+ let mut shortcut_borrow = shortcut_rc.as_ref().map(|rc| rc.borrow_mut());
+
self.kernel.encode(
input_buffer_borrow.as_deref().map(|b| (b, input_offset)),
self.scales_buffer.borrow().deref(),
(output_array.buffer().borrow_mut().deref_mut(), output_offset),
+ shortcut_borrow.as_deref_mut(),
batch_len as u32,
element_count as u32,
self.config.epsilon,
diff --git a/crates/uzu/tests/unit/kernel/rms_norm_test.rs b/crates/uzu/tests/unit/kernel/rms_norm_test.rs
index 2a99e3083..ee403016e 100644
--- a/crates/uzu/tests/unit/kernel/rms_norm_test.rs
+++ b/crates/uzu/tests/unit/kernel/rms_norm_test.rs
@@ -126,6 +126,7 @@ fn get_output<
OutputT::data_type(),
AccumT::data_type(),
input.in_place,
+ false,
)
.expect("Failed to create RMSNormKernel");
@@ -147,6 +148,7 @@ fn get_output<
input_buffer,
scales_array.buffer().borrow().deref(),
output_array.buffer().borrow_mut().deref_mut(),
+ None::<&mut B::Buffer>,
input.batch_size,
input.element_count,
input.epsilon,
@@ -177,7 +179,7 @@ fn test_internal<
|| matches!(OutputT::data_type(), DataType::F16 | DataType::BF16)
|| matches!(AccumT::data_type(), DataType::F16 | DataType::BF16)
{
- 0.012f32
+ 0.016f32
} else {
1e-5
};