Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Support post_layer_scalar (#419)

CCYeha78f652e1aa0
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs b/crates/backend-uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
index c60ed2065..0aee4549c 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs
@@ -24,11 +24,14 @@ pub fn rms_norm<
element_count: u32,
epsilon: f32,
scale_offset: f32,
+ post_layer_scalar: f32,
#[specialize] in_place: bool,
#[specialize] full_layer: bool,
#[specialize] copy_to_shortcut: bool,
#[specialize] residual_add: bool,
#[specialize] use_hadamard: bool,
+ #[specialize] scale_residual_sum: bool,
+ #[specialize] scale_output: bool,
) {
if use_hadamard {
unimplemented!("not supported yet");
@@ -58,6 +61,9 @@ pub fn rms_norm<
let skip_ptr = unsafe { shortcut.unwrap().add(batch_offset + i) };
if residual_add {
val = val + unsafe { *skip_ptr };
+ if scale_residual_sum {
+ val = val * InputT::from(post_layer_scalar).unwrap();
+ }
}
unsafe { *skip_ptr = val };
}
@@ -86,6 +92,11 @@ pub fn rms_norm<
let scale_with_offset_out = OutputT::from(scale_val + scale_offset).unwrap();
normalized_out * scale_with_offset_out
};
+ let result = if scale_output {
+ result * OutputT::from(post_layer_scalar).unwrap()
+ } else {
+ result
+ };
unsafe { *output.add(batch_offset + i) = result };
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index 61b9757b2..bc8afaa78 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -24,11 +24,14 @@ PUBLIC KERNEL(RMSNorm)(
constant uint& element_count,
constant float& epsilon,
constant float& scale_offset,
+ constant float& post_layer_scalar,
const bool in_place SPECIALIZE,
const bool full_layer SPECIALIZE,
const bool copy_to_shortcut SPECIALIZE,
const bool residual_add SPECIALIZE,
const bool use_hadamard SPECIALIZE,
+ const bool scale_residual_sum SPECIALIZE,
+ const bool scale_output SPECIALIZE,
threadgroup AccumT shared_sum[METAL_SIMD_SIZE],
const ThreadContext thread_context,
const uint batch_idx GROUPS(batch_size),
@@ -57,6 +60,10 @@ PUBLIC KERNEL(RMSNorm)(
if (copy_to_shortcut) {
if (residual_add) {
val += shortcut[i];
+ if (scale_residual_sum) {
+ val =
+ static_cast<InputT>(static_cast<float>(val) * post_layer_scalar);
+ }
}
shortcut[i] = val;
}
@@ -109,6 +116,10 @@ PUBLIC KERNEL(RMSNorm)(
));
}
+ if (scale_output) {
+ val = static_cast<OutputT>(static_cast<float>(val) * post_layer_scalar);
+ }
+
output[i] = val;
}
}
diff --git a/crates/backend-uzu/src/config/transformer_layer.rs b/crates/backend-uzu/src/config/transformer_layer.rs
index 908c5fcd0..b6510e6ab 100644
--- a/crates/backend-uzu/src/config/transformer_layer.rs
+++ b/crates/backend-uzu/src/config/transformer_layer.rs
@@ -1,4 +1,3 @@
-use monostate::MustBe;
use proc_macros::uzu_config;
use super::{MLPConfig, MixerConfig, NormalizationConfig, RoPEConfig};
@@ -15,6 +14,6 @@ pub struct TransformerLayerConfig {
pub rope_config: Option<RoPEConfig>,
pub hidden_dim: Option<usize>,
pub ple_config: Option<Unsupported>,
- pub has_post_layer_scalar: MustBe!(false),
+ pub has_post_layer_scalar: bool,
pub kv_source_layer: Option<Unsupported>,
}
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rs
index 0baa60bb9..474236279 100644
--- a/crates/backend-uzu/src/encodable_block/decoder.rs
+++ b/crates/backend-uzu/src/encodable_block/decoder.rs
@@ -11,7 +11,8 @@ use crate::{
backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},
config::DecoderConfig,
encodable_block::{
- Embedding, LayerArguments, LayerExecutables, QkUnpack, RMSNorm, Rope, embedding::EmbeddingError,
+ Embedding, LayerArguments, LayerExecutables, PostLayerScalar, QkUnpack, RMSNorm, Rope,
+ embedding::EmbeddingError,
},
forward_pass::{cache_layers::CacheLayers, state::SharedBuffers},
parameters::ParameterTree,
@@ -145,6 +146,7 @@ impl<B: Backend> Decoder<B> {
None,
true,
true,
+ PostLayerScalar::None,
)
.expect("Failed to create output RMS norm kernel");
diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rs
index ba7a2207b..e9ed4f513 100644
--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs
@@ -2,6 +2,8 @@
use std::rc::Rc;
+use half::{bf16, f16};
+
use super::MixerExecutables;
#[cfg(feature = "tracing")]
use crate::backends::common::{Kernels, kernel::TensorAddBiasKernel};
@@ -13,7 +15,7 @@ use crate::{
config::{MixerConfig, TransformerConfig, TransformerLayerConfig},
encodable_block::{
Attention, AttentionArguments, DeltaNetArguments, DeltaNetMixer, Linear, MambaArguments, MambaMixer, Mlp,
- QKNorm, QkUnpack, RMSNorm, Rope, ShortConvArguments, ShortConvMixer,
+ PostLayerScalar, QKNorm, QkUnpack, RMSNorm, Rope, ShortConvArguments, ShortConvMixer,
},
forward_pass::{cache_layers::CacheLayer, state::RopeBuffers},
parameters::ParameterTree,
@@ -47,6 +49,25 @@ impl<B: Backend> LayerExecutables<B> {
) -> Self {
let intermediate_data_type: DataType = layer_config.mixer_config.activation_precision().into();
+ let (residual_sum_scalar, output_scalar) = if layer_config.has_post_layer_scalar {
+ assert!(
+ layer_config.post_mlp_norm_config.is_some(),
+ "layer {layer_index} sets post_layer_scalar but has no post_mlp_norm"
+ );
+ let leaf = decoder_layer_loader.leaf("post_layer_scalar").expect("Failed to read post_layer_scalar weight");
+ let scalar = match leaf.data_type() {
+ DataType::BF16 => {
+ leaf.read_slice::<bf16>().expect("Failed to read post_layer_scalar weight")[0].to_f32()
+ },
+ DataType::F16 => leaf.read_slice::<f16>().expect("Failed to read post_layer_scalar weight")[0].to_f32(),
+ DataType::F32 => leaf.read_slice::<f32>().expect("Failed to read post_layer_scalar weight")[0],
+ other => panic!("post_layer_scalar must be a float dtype, got {other:?}"),
+ };
+ (PostLayerScalar::ScaleResidualSum(scalar), PostLayerScalar::ScaleOutput(scalar))
+ } else {
+ (PostLayerScalar::None, PostLayerScalar::None)
+ };
+
#[cfg(feature = "tracing")]
let tensor_add = TensorAddBiasKernel::new(context, intermediate_data_type, false)
.expect("Failed to create TensorAddBiasKernel kernel"); // TODO: this function return Result
@@ -211,6 +232,7 @@ impl<B: Backend> LayerExecutables<B> {
mixer_hadamard_factors,
true,
layer_index > 0,
+ PostLayerScalar::None,
)
.expect("Failed to create RMS norm kernel");
@@ -224,6 +246,7 @@ impl<B: Backend> LayerExecutables<B> {
None,
false,
false,
+ PostLayerScalar::None,
)
.expect("Failed to create RMS norm kernel"),
)
@@ -248,6 +271,7 @@ impl<B: Backend> LayerExecutables<B> {
mlp_input_hadamard_factors,
true,
true,
+ residual_sum_scalar,
)
.expect("Failed to create RMS norm kernel");
@@ -261,6 +285,7 @@ impl<B: Backend> LayerExecutables<B> {
None,
false,
false,
+ output_scalar,
)
.expect("Failed to create RMS norm kernel"),
)
diff --git a/crates/backend-uzu/src/encodable_block/mod.rs b/crates/backend-uzu/src/encodable_block/mod.rs
index b315743b3..afa7bfeaa 100644
--- a/crates/backend-uzu/src/encodable_block/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mod.rs
@@ -33,7 +33,7 @@ pub use pooling::Pooling;
pub use prediction_head::ClassifierPredictionHead;
pub use qk_norm::QKNorm;
pub use qk_unpack::QkUnpack;
-pub use rms_norm::{RMSNorm, RMSNormError};
+pub use rms_norm::{PostLayerScalar, RMSNorm, RMSNormError};
pub use rope::Rope;
pub(crate) use sampling::SamplingInputs;
pub use sampling::{Sampling, SamplingArguments};
diff --git a/crates/backend-uzu/src/encodable_block/normalization.rs b/crates/backend-uzu/src/encodable_block/normalization.rs
index ecc943705..829cbadf5 100644
--- a/crates/backend-uzu/src/encodable_block/normalization.rs
+++ b/crates/backend-uzu/src/encodable_block/normalization.rs
@@ -2,7 +2,7 @@
use thiserror::Error;
-use super::{LayerNorm, LayerNormError, RMSNorm, RMSNormError};
+use super::{LayerNorm, LayerNormError, PostLayerScalar, RMSNorm, RMSNormError};
use crate::{
DataType,
backends::common::{Allocation, Backend, Encoder},
@@ -44,6 +44,7 @@ impl<B: Backend> Normalization<B> {
None,
false,
false,
+ PostLayerScalar::None,
)?))
}
}
diff --git a/crates/backend-uzu/src/encodable_block/rms_norm.rs b/crates/backend-uzu/src/encodable_block/rms_norm.rs
index 67b3047e8..4b1d83134 100644
--- a/crates/backend-uzu/src/encodable_block/rms_norm.rs
+++ b/crates/backend-uzu/src/encodable_block/rms_norm.rs
@@ -13,6 +13,13 @@ use crate::{
parameters::{ParameterLoaderError, ParameterTree},
};
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum PostLayerScalar {
+ None,
+ ScaleResidualSum(f32),
+ ScaleOutput(f32),
+}
+
#[derive(Debug, Error)]
pub enum RMSNormError<B: Backend> {
#[error("Backend error: {0}")]
@@ -29,6 +36,7 @@ pub struct RMSNorm<B: Backend> {
input_data_type: DataType,
output_data_type: DataType,
hadamard_factors: Option<Allocation<B>>,
+ post_layer_scalar_value: f32,
}
impl<B: Backend> RMSNorm<B> {
@@ -40,6 +48,7 @@ impl<B: Backend> RMSNorm<B> {
hadamard_factors: Option<Allocation<B>>,
use_shortcut: bool,
residual_add: bool,
+ post_layer_scalar: PostLayerScalar,
) -> Result<Self, RMSNormError<B>> {
let scales_leaf = parameter_tree.leaf("scales").map_err(RMSNormError::ParameterError)?;
let element_count = scales_leaf.shape()[0];
@@ -48,6 +57,12 @@ impl<B: Backend> RMSNorm<B> {
let accumulation_data_type: DataType = config.accumulation_precision.into();
let scale_data_type: DataType = config.scale_precision.into();
+ let (scale_residual_sum, scale_output, post_layer_scalar_value) = match post_layer_scalar {
+ PostLayerScalar::None => (false, false, 1.0),
+ PostLayerScalar::ScaleResidualSum(value) => (true, false, value),
+ PostLayerScalar::ScaleOutput(value) => (false, true, value),
+ };
+
let kernel = <B::Kernels as Kernels>::RMSNormKernel::new(
context,
intermediate_data_type,
@@ -59,6 +74,8 @@ impl<B: Backend> RMSNorm<B> {
use_shortcut,
residual_add,
hadamard_factors.is_some(),
+ scale_residual_sum,
+ scale_output,
)
.map_err(RMSNormError::BackendError)?;
@@ -70,6 +87,7 @@ impl<B: Backend> RMSNorm<B> {
input_data_type: intermediate_data_type,
output_data_type: scale_data_type,
hadamard_factors,
+ post_layer_scalar_value,
})
}
@@ -96,6 +114,7 @@ impl<B: Backend> RMSNorm<B> {
self.element_count as u32,
self.config.epsilon,
self.config.scale_offset.unwrap_or(0.0),
+ self.post_layer_scalar_value,
encoder,
);
Ok(output)
diff --git a/crates/backend-uzu/src/parameters/loader.rs b/crates/backend-uzu/src/parameters/loader.rs
index 79965ab7b..d79c7ebd2 100644
--- a/crates/backend-uzu/src/parameters/loader.rs
+++ b/crates/backend-uzu/src/parameters/loader.rs
@@ -7,7 +7,7 @@ use thiserror::Error;
use super::safetensors_metadata::{HashMetadata as STMetadata, HeaderLoadingError, read_metadata as read_st_metadata};
use crate::{
- DataType,
+ ArrayElement, DataType,
array::{Array, ArrayContextExt},
backends::common::{Allocation, AllocationType, AsBufferRangeRef, Backend, Context, DenseBuffer},
utils::fs::file_read_exact_at,
@@ -189,6 +189,13 @@ impl<'file, 'context, 'leaf, C: Context> ParameterLeaf<'file, 'context, 'leaf, C
Ok(())
}
+ pub fn read_slice<T: ArrayElement>(&self) -> Result<Box<[T]>, ParameterLoaderError<C::Backend>> {
+ let element_count = self.metadata.size / std::mem::size_of::<T>();
+ let mut data = vec![T::zeroed(); element_count];
+ file_read_exact_at(self.loader.file, bytemuck::cast_slice_mut(&mut data), self.metadata.offset as u64)?;
+ Ok(data.into_boxed_slice())
+ }
+
pub fn read_allocation(&self) -> Result<Allocation<C::Backend>, ParameterLoaderError<C::Backend>> {
let allocation = self
.loader
diff --git a/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rs b/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rs
index 63bb22b79..816d88b50 100644
--- a/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rs
@@ -133,6 +133,8 @@ fn get_output<
false,
false,
false,
+ false,
+ false,
)
.expect("Failed to create RMSNormKernel");
@@ -158,6 +160,7 @@ fn get_output<
input.element_count,
input.epsilon,
input.scale_offset,
+ 1.0,
&mut encoder,
);
@@ -390,6 +393,69 @@ fn test_edge_f16_f16_f16_f16() {
test_edge::<f16, f16, f16, f16>();
}
+fn run_scaled_rms<B: Backend>(with_shortcut: bool, scale_residual_sum: bool, scale_output: bool) -> (Vec<f32>, Vec<f32>) {
+ let (batch, dim) = (2usize, 64usize);
+ let total = batch * dim;
+ let mut rng = SmallRng::seed_from_u64(99);
+ let input: Vec<f32> = (0..total).map(|_| rng.random_range(-2.0f32..2.0f32)).collect();
+ let scales: Vec<f32> = (0..dim).map(|_| rng.random_range(0.1f32..3.0f32)).collect();
+ let shortcut: Vec<f32> = (0..total).map(|_| rng.random_range(-1.0f32..1.0f32)).collect();
+
+ let context = B::Context::new().expect("Failed to create Context");
+ let kernel = <<B as Backend>::Kernels as Kernels>::RMSNormKernel::new(
+ &context,
+ f32::data_type(),
+ f32::data_type(),
+ f32::data_type(),
+ f32::data_type(),
+ false,
+ false,
+ with_shortcut,
+ with_shortcut,
+ false,
+ scale_residual_sum,
+ scale_output,
+ )
+ .expect("Failed to create RMSNormKernel");
+
+ let input_buffer = context.create_array_from(&[total], &input).into_allocation();
+ let scales_buffer = context.create_array_from(&[dim], &scales).into_allocation();
+ let mut output = context.create_array_uninitialized(&[total], f32::data_type()).into_allocation();
+ let mut shortcut_buffer = with_shortcut.then(|| context.create_array_from(&[total], &shortcut).into_allocation());
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ Some(&input_buffer),
+ &scales_buffer,
+ &mut output,
+ shortcut_buffer.as_mut(),
+ None::<&Allocation<B>>,
+ batch as u32,
+ dim as u32,
+ 1e-6,
+ 0.0,
+ 0.7,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ let output_values = crate::common::helpers::allocation_to_vec(&output);
+ let shortcut_values = shortcut_buffer.as_ref().map(crate::common::helpers::allocation_to_vec).unwrap_or_default();
+ (output_values, shortcut_values)
+}
+
+#[uzu_test]
+fn test_post_layer_scalar_metal_matches_cpu() {
+ for &(shortcut, residual_sum, output) in &[(false, false, true), (true, true, false)] {
+ let (cpu_out, cpu_shortcut) = run_scaled_rms::<Cpu>(shortcut, residual_sum, output);
+ for_each_non_cpu_backend!(|B| {
+ let (out, sc) = run_scaled_rms::<B>(shortcut, residual_sum, output);
+ assert_eq_float::<f32>(&cpu_out, &out, 1e-5, "post_layer_scalar output: metal != cpu");
+ assert_eq_float::<f32>(&cpu_shortcut, &sc, 1e-5, "post_layer_scalar shortcut: metal != cpu");
+ });
+ }
+}
+
// benchmarks
#[uzu_bench]
fn bench_rms_norm(c: &mut Criterion) {
@@ -411,6 +477,8 @@ fn bench_rms_norm(c: &mut Criterion) {
false,
false,
false,
+ false,
+ false,
)
.unwrap();
@@ -452,6 +520,7 @@ fn bench_rms_norm(c: &mut Criterion) {
model_dim as u32,
epsilon,
scale_offset,
+ 1.0,
&mut encoder,
);
}