A high-performance inference engine for AI models
Support post_layer_scalar (#419)
CCYeha78f652e1aa0
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs b/crates/backend-uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rsindex c60ed2065..0aee4549c 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/rms_norm/rms_norm.rs@@ -24,11 +24,14 @@ pub fn rms_norm<element_count: u32,epsilon: f32,scale_offset: f32,+ post_layer_scalar: f32,#[specialize] in_place: bool,#[specialize] full_layer: bool,#[specialize] copy_to_shortcut: bool,#[specialize] residual_add: bool,#[specialize] use_hadamard: bool,+ #[specialize] scale_residual_sum: bool,+ #[specialize] scale_output: bool,) {if use_hadamard {unimplemented!("not supported yet");@@ -58,6 +61,9 @@ pub fn rms_norm<let skip_ptr = unsafe { shortcut.unwrap().add(batch_offset + i) };if residual_add {val = val + unsafe { *skip_ptr };+ if scale_residual_sum {+ val = val * InputT::from(post_layer_scalar).unwrap();+ }}unsafe { *skip_ptr = val };}@@ -86,6 +92,11 @@ pub fn rms_norm<let scale_with_offset_out = OutputT::from(scale_val + scale_offset).unwrap();normalized_out * scale_with_offset_out};+ let result = if scale_output {+ result * OutputT::from(post_layer_scalar).unwrap()+ } else {+ result+ };unsafe { *output.add(batch_offset + i) = result };}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex 61b9757b2..bc8afaa78 100644--- a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -24,11 +24,14 @@ PUBLIC KERNEL(RMSNorm)(constant uint& element_count,constant float& epsilon,constant float& scale_offset,+ constant float& post_layer_scalar,const bool in_place SPECIALIZE,const bool full_layer SPECIALIZE,const bool copy_to_shortcut SPECIALIZE,const bool residual_add SPECIALIZE,const bool use_hadamard SPECIALIZE,+ const bool scale_residual_sum SPECIALIZE,+ const bool scale_output SPECIALIZE,threadgroup AccumT shared_sum[METAL_SIMD_SIZE],const ThreadContext thread_context,const uint batch_idx GROUPS(batch_size),@@ -57,6 +60,10 @@ PUBLIC KERNEL(RMSNorm)(if (copy_to_shortcut) {if (residual_add) {val += shortcut[i];+ if (scale_residual_sum) {+ val =+ static_cast<InputT>(static_cast<float>(val) * post_layer_scalar);+ }}shortcut[i] = val;}@@ -109,6 +116,10 @@ PUBLIC KERNEL(RMSNorm)());}+ if (scale_output) {+ val = static_cast<OutputT>(static_cast<float>(val) * post_layer_scalar);+ }+output[i] = val;}}diff --git a/crates/backend-uzu/src/config/transformer_layer.rs b/crates/backend-uzu/src/config/transformer_layer.rsindex 908c5fcd0..b6510e6ab 100644--- a/crates/backend-uzu/src/config/transformer_layer.rs+++ b/crates/backend-uzu/src/config/transformer_layer.rs@@ -1,4 +1,3 @@-use monostate::MustBe;use proc_macros::uzu_config;use super::{MLPConfig, MixerConfig, NormalizationConfig, RoPEConfig};@@ -15,6 +14,6 @@ pub struct TransformerLayerConfig {pub rope_config: Option<RoPEConfig>,pub hidden_dim: Option<usize>,pub ple_config: Option<Unsupported>,- pub has_post_layer_scalar: MustBe!(false),+ pub has_post_layer_scalar: bool,pub kv_source_layer: Option<Unsupported>,}diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rsindex 0baa60bb9..474236279 100644--- a/crates/backend-uzu/src/encodable_block/decoder.rs+++ b/crates/backend-uzu/src/encodable_block/decoder.rs@@ -11,7 +11,8 @@ use crate::{backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},config::DecoderConfig,encodable_block::{- Embedding, LayerArguments, LayerExecutables, QkUnpack, RMSNorm, Rope, embedding::EmbeddingError,+ Embedding, LayerArguments, LayerExecutables, PostLayerScalar, QkUnpack, RMSNorm, Rope,+ embedding::EmbeddingError,},forward_pass::{cache_layers::CacheLayers, state::SharedBuffers},parameters::ParameterTree,@@ -145,6 +146,7 @@ impl<B: Backend> Decoder<B> {None,true,true,+ PostLayerScalar::None,).expect("Failed to create output RMS norm kernel");diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rsindex ba7a2207b..e9ed4f513 100644--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs@@ -2,6 +2,8 @@use std::rc::Rc;+use half::{bf16, f16};+use super::MixerExecutables;#[cfg(feature = "tracing")]use crate::backends::common::{Kernels, kernel::TensorAddBiasKernel};@@ -13,7 +15,7 @@ use crate::{config::{MixerConfig, TransformerConfig, TransformerLayerConfig},encodable_block::{Attention, AttentionArguments, DeltaNetArguments, DeltaNetMixer, Linear, MambaArguments, MambaMixer, Mlp,- QKNorm, QkUnpack, RMSNorm, Rope, ShortConvArguments, ShortConvMixer,+ PostLayerScalar, QKNorm, QkUnpack, RMSNorm, Rope, ShortConvArguments, ShortConvMixer,},forward_pass::{cache_layers::CacheLayer, state::RopeBuffers},parameters::ParameterTree,@@ -47,6 +49,25 @@ impl<B: Backend> LayerExecutables<B> {) -> Self {let intermediate_data_type: DataType = layer_config.mixer_config.activation_precision().into();+ let (residual_sum_scalar, output_scalar) = if layer_config.has_post_layer_scalar {+ assert!(+ layer_config.post_mlp_norm_config.is_some(),+ "layer {layer_index} sets post_layer_scalar but has no post_mlp_norm"+ );+ let leaf = decoder_layer_loader.leaf("post_layer_scalar").expect("Failed to read post_layer_scalar weight");+ let scalar = match leaf.data_type() {+ DataType::BF16 => {+ leaf.read_slice::<bf16>().expect("Failed to read post_layer_scalar weight")[0].to_f32()+ },+ DataType::F16 => leaf.read_slice::<f16>().expect("Failed to read post_layer_scalar weight")[0].to_f32(),+ DataType::F32 => leaf.read_slice::<f32>().expect("Failed to read post_layer_scalar weight")[0],+ other => panic!("post_layer_scalar must be a float dtype, got {other:?}"),+ };+ (PostLayerScalar::ScaleResidualSum(scalar), PostLayerScalar::ScaleOutput(scalar))+ } else {+ (PostLayerScalar::None, PostLayerScalar::None)+ };+#[cfg(feature = "tracing")]let tensor_add = TensorAddBiasKernel::new(context, intermediate_data_type, false).expect("Failed to create TensorAddBiasKernel kernel"); // TODO: this function return Result@@ -211,6 +232,7 @@ impl<B: Backend> LayerExecutables<B> {mixer_hadamard_factors,true,layer_index > 0,+ PostLayerScalar::None,).expect("Failed to create RMS norm kernel");@@ -224,6 +246,7 @@ impl<B: Backend> LayerExecutables<B> {None,false,false,+ PostLayerScalar::None,).expect("Failed to create RMS norm kernel"),)@@ -248,6 +271,7 @@ impl<B: Backend> LayerExecutables<B> {mlp_input_hadamard_factors,true,true,+ residual_sum_scalar,).expect("Failed to create RMS norm kernel");@@ -261,6 +285,7 @@ impl<B: Backend> LayerExecutables<B> {None,false,false,+ output_scalar,).expect("Failed to create RMS norm kernel"),)diff --git a/crates/backend-uzu/src/encodable_block/mod.rs b/crates/backend-uzu/src/encodable_block/mod.rsindex b315743b3..afa7bfeaa 100644--- a/crates/backend-uzu/src/encodable_block/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mod.rs@@ -33,7 +33,7 @@ pub use pooling::Pooling;pub use prediction_head::ClassifierPredictionHead;pub use qk_norm::QKNorm;pub use qk_unpack::QkUnpack;-pub use rms_norm::{RMSNorm, RMSNormError};+pub use rms_norm::{PostLayerScalar, RMSNorm, RMSNormError};pub use rope::Rope;pub(crate) use sampling::SamplingInputs;pub use sampling::{Sampling, SamplingArguments};diff --git a/crates/backend-uzu/src/encodable_block/normalization.rs b/crates/backend-uzu/src/encodable_block/normalization.rsindex ecc943705..829cbadf5 100644--- a/crates/backend-uzu/src/encodable_block/normalization.rs+++ b/crates/backend-uzu/src/encodable_block/normalization.rs@@ -2,7 +2,7 @@use thiserror::Error;-use super::{LayerNorm, LayerNormError, RMSNorm, RMSNormError};+use super::{LayerNorm, LayerNormError, PostLayerScalar, RMSNorm, RMSNormError};use crate::{DataType,backends::common::{Allocation, Backend, Encoder},@@ -44,6 +44,7 @@ impl<B: Backend> Normalization<B> {None,false,false,+ PostLayerScalar::None,)?))}}diff --git a/crates/backend-uzu/src/encodable_block/rms_norm.rs b/crates/backend-uzu/src/encodable_block/rms_norm.rsindex 67b3047e8..4b1d83134 100644--- a/crates/backend-uzu/src/encodable_block/rms_norm.rs+++ b/crates/backend-uzu/src/encodable_block/rms_norm.rs@@ -13,6 +13,13 @@ use crate::{parameters::{ParameterLoaderError, ParameterTree},};+#[derive(Debug, Clone, Copy, PartialEq)]+pub enum PostLayerScalar {+ None,+ ScaleResidualSum(f32),+ ScaleOutput(f32),+}+#[derive(Debug, Error)]pub enum RMSNormError<B: Backend> {#[error("Backend error: {0}")]@@ -29,6 +36,7 @@ pub struct RMSNorm<B: Backend> {input_data_type: DataType,output_data_type: DataType,hadamard_factors: Option<Allocation<B>>,+ post_layer_scalar_value: f32,}impl<B: Backend> RMSNorm<B> {@@ -40,6 +48,7 @@ impl<B: Backend> RMSNorm<B> {hadamard_factors: Option<Allocation<B>>,use_shortcut: bool,residual_add: bool,+ post_layer_scalar: PostLayerScalar,) -> Result<Self, RMSNormError<B>> {let scales_leaf = parameter_tree.leaf("scales").map_err(RMSNormError::ParameterError)?;let element_count = scales_leaf.shape()[0];@@ -48,6 +57,12 @@ impl<B: Backend> RMSNorm<B> {let accumulation_data_type: DataType = config.accumulation_precision.into();let scale_data_type: DataType = config.scale_precision.into();+ let (scale_residual_sum, scale_output, post_layer_scalar_value) = match post_layer_scalar {+ PostLayerScalar::None => (false, false, 1.0),+ PostLayerScalar::ScaleResidualSum(value) => (true, false, value),+ PostLayerScalar::ScaleOutput(value) => (false, true, value),+ };+let kernel = <B::Kernels as Kernels>::RMSNormKernel::new(context,intermediate_data_type,@@ -59,6 +74,8 @@ impl<B: Backend> RMSNorm<B> {use_shortcut,residual_add,hadamard_factors.is_some(),+ scale_residual_sum,+ scale_output,).map_err(RMSNormError::BackendError)?;@@ -70,6 +87,7 @@ impl<B: Backend> RMSNorm<B> {input_data_type: intermediate_data_type,output_data_type: scale_data_type,hadamard_factors,+ post_layer_scalar_value,})}@@ -96,6 +114,7 @@ impl<B: Backend> RMSNorm<B> {self.element_count as u32,self.config.epsilon,self.config.scale_offset.unwrap_or(0.0),+ self.post_layer_scalar_value,encoder,);Ok(output)diff --git a/crates/backend-uzu/src/parameters/loader.rs b/crates/backend-uzu/src/parameters/loader.rsindex 79965ab7b..d79c7ebd2 100644--- a/crates/backend-uzu/src/parameters/loader.rs+++ b/crates/backend-uzu/src/parameters/loader.rs@@ -7,7 +7,7 @@ use thiserror::Error;use super::safetensors_metadata::{HashMetadata as STMetadata, HeaderLoadingError, read_metadata as read_st_metadata};use crate::{- DataType,+ ArrayElement, DataType,array::{Array, ArrayContextExt},backends::common::{Allocation, AllocationType, AsBufferRangeRef, Backend, Context, DenseBuffer},utils::fs::file_read_exact_at,@@ -189,6 +189,13 @@ impl<'file, 'context, 'leaf, C: Context> ParameterLeaf<'file, 'context, 'leaf, COk(())}+ pub fn read_slice<T: ArrayElement>(&self) -> Result<Box<[T]>, ParameterLoaderError<C::Backend>> {+ let element_count = self.metadata.size / std::mem::size_of::<T>();+ let mut data = vec![T::zeroed(); element_count];+ file_read_exact_at(self.loader.file, bytemuck::cast_slice_mut(&mut data), self.metadata.offset as u64)?;+ Ok(data.into_boxed_slice())+ }+pub fn read_allocation(&self) -> Result<Allocation<C::Backend>, ParameterLoaderError<C::Backend>> {let allocation = self.loaderdiff --git a/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rs b/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rsindex 63bb22b79..816d88b50 100644--- a/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rs@@ -133,6 +133,8 @@ fn get_output<false,false,false,+ false,+ false,).expect("Failed to create RMSNormKernel");@@ -158,6 +160,7 @@ fn get_output<input.element_count,input.epsilon,input.scale_offset,+ 1.0,&mut encoder,);@@ -390,6 +393,69 @@ fn test_edge_f16_f16_f16_f16() {test_edge::<f16, f16, f16, f16>();}+fn run_scaled_rms<B: Backend>(with_shortcut: bool, scale_residual_sum: bool, scale_output: bool) -> (Vec<f32>, Vec<f32>) {+ let (batch, dim) = (2usize, 64usize);+ let total = batch * dim;+ let mut rng = SmallRng::seed_from_u64(99);+ let input: Vec<f32> = (0..total).map(|_| rng.random_range(-2.0f32..2.0f32)).collect();+ let scales: Vec<f32> = (0..dim).map(|_| rng.random_range(0.1f32..3.0f32)).collect();+ let shortcut: Vec<f32> = (0..total).map(|_| rng.random_range(-1.0f32..1.0f32)).collect();++ let context = B::Context::new().expect("Failed to create Context");+ let kernel = <<B as Backend>::Kernels as Kernels>::RMSNormKernel::new(+ &context,+ f32::data_type(),+ f32::data_type(),+ f32::data_type(),+ f32::data_type(),+ false,+ false,+ with_shortcut,+ with_shortcut,+ false,+ scale_residual_sum,+ scale_output,+ )+ .expect("Failed to create RMSNormKernel");++ let input_buffer = context.create_array_from(&[total], &input).into_allocation();+ let scales_buffer = context.create_array_from(&[dim], &scales).into_allocation();+ let mut output = context.create_array_uninitialized(&[total], f32::data_type()).into_allocation();+ let mut shortcut_buffer = with_shortcut.then(|| context.create_array_from(&[total], &shortcut).into_allocation());++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ Some(&input_buffer),+ &scales_buffer,+ &mut output,+ shortcut_buffer.as_mut(),+ None::<&Allocation<B>>,+ batch as u32,+ dim as u32,+ 1e-6,+ 0.0,+ 0.7,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ let output_values = crate::common::helpers::allocation_to_vec(&output);+ let shortcut_values = shortcut_buffer.as_ref().map(crate::common::helpers::allocation_to_vec).unwrap_or_default();+ (output_values, shortcut_values)+}++#[uzu_test]+fn test_post_layer_scalar_metal_matches_cpu() {+ for &(shortcut, residual_sum, output) in &[(false, false, true), (true, true, false)] {+ let (cpu_out, cpu_shortcut) = run_scaled_rms::<Cpu>(shortcut, residual_sum, output);+ for_each_non_cpu_backend!(|B| {+ let (out, sc) = run_scaled_rms::<B>(shortcut, residual_sum, output);+ assert_eq_float::<f32>(&cpu_out, &out, 1e-5, "post_layer_scalar output: metal != cpu");+ assert_eq_float::<f32>(&cpu_shortcut, &sc, 1e-5, "post_layer_scalar shortcut: metal != cpu");+ });+ }+}+// benchmarks#[uzu_bench]fn bench_rms_norm(c: &mut Criterion) {@@ -411,6 +477,8 @@ fn bench_rms_norm(c: &mut Criterion) {false,false,false,+ false,+ false,).unwrap();@@ -452,6 +520,7 @@ fn bench_rms_norm(c: &mut Criterion) {model_dim as u32,epsilon,scale_offset,+ 1.0,&mut encoder,);}