A high-performance inference engine for AI models
dslify-rms-norm (#166)
Aleksandr Golokoz616dcb1605cd
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsindex 798c31d93..f0b8bf99d 100644--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs@@ -41,7 +41,7 @@ pub use linear::{FullPrecisionLinear, QuantizedLinear};pub(crate) use mamba_mixer::MambaMixer;pub use mlp::{MlpBlock, MlpFusedBlock, MlpFusedUpKernel};pub use moe_block::{MoeBlock, SharedMoeWeights};-pub use normalization::{LayerNorm, Normalization, NormalizationError, QKNorm, RMSNorm};+pub use normalization::{LayerNorm, Normalization, QKNorm, RMSNorm};pub use pooling::Pooling;pub use prediction_head::ClassifierPredictionHead;pub use rope::Rope;diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rsindex 8c281fb0d..8a0e233fb 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs@@ -11,25 +11,13 @@ pub use rms_norm::RMSNorm;use super::{EncodableBlock, Metal};use crate::{DataType,- backends::metal::{- MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject, Retained,- kernel::RMSNormError,- },+ backends::metal::{MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject, Retained},config::NormalizationConfig,encodable_block::EncodingParameters,forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};-/// Unified error type for normalization operations.-#[derive(Debug, thiserror::Error)]-pub enum NormalizationError {- #[error("LayerNorm error: {0}")]- LayerNorm(#[from] MTLError),- #[error("RMSNorm error: {0}")]- RMSNorm(#[from] RMSNormError),-}-/// Unified normalization encodable that can be either LayerNorm or RMSNorm.pub enum Normalization {LayerNorm(LayerNorm),@@ -46,7 +34,7 @@ impl Normalization {input_array_id: ArrayId,output_array_id: ArrayId,parameter_tree: &ParameterTree<MTLContext>,- ) -> Result<Self, NormalizationError> {+ ) -> Result<Self, MTLError> {if config.subtract_mean {// Use LayerNorm (subtract mean before normalization)Ok(Normalization::LayerNorm(LayerNorm::new(diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rsindex b429d68d0..2f1277456 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs@@ -3,10 +3,12 @@use super::super::{EncodableBlock, Metal};use crate::{DataType,- backends::metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,- MTLResourceOptions, ProtocolObject, Retained,- kernel::rms_norm::{QKNormArguments, QKNormTarget, RMSNormError, RMSNormKernel, RMSNormKernelType},+ backends::{+ common::kernel::QKNormKernel,+ metal::{+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt,+ MTLError, MTLResourceOptions, ProtocolObject, Retained, kernel::dsl::QKNormMetalKernel,+ },},config::{NormalizationConfig, UpcastMode},encodable_block::EncodingParameters,@@ -15,8 +17,8 @@ use crate::{};pub struct QKNorm {- query_kernel: Option<RMSNormKernel>,- key_kernel: Option<RMSNormKernel>,+ query_kernel: Option<QKNormMetalKernel>,+ key_kernel: Option<QKNormMetalKernel>,query_config: Option<NormalizationConfig>,key_config: Option<NormalizationConfig>,qkv_array_id: ArrayId,@@ -38,7 +40,7 @@ impl QKNorm {num_q_heads: usize,num_kv_heads: usize,head_dim: usize,- ) -> Result<Self, RMSNormError> {+ ) -> Result<Self, MTLError> {let mut query_kernel = None;let mut key_kernel = None;let mut query_scales_buffer = None;@@ -47,8 +49,9 @@ impl QKNorm {// Setup query normalization if configuredif let Some(ref q_config) = query_config {let scales_param = parameter_tree.leaf("query_norm.scales").map_err(|e| {- RMSNormError::MetalError(MTLError::Library(crate::backends::metal::error::LibraryError::Custom(- format!("Failed to load query scales: {:?}", e),+ MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(+ "Failed to load query scales: {:?}",+ e)))})?;@@ -64,15 +67,7 @@ impl QKNorm {UpcastMode::FullLayer => (intermediate_data_type, scale_data_type, scale_data_type),};- let kernel = RMSNormKernel::new_with_mode(- context,- input_type,- scales_type,- output_type,- accumulation_data_type,- RMSNormKernelType::QueryKey,- q_config.upcast_mode == UpcastMode::FullLayer,- )?;+ let kernel = QKNormMetalKernel::new(context, input_type, scales_type, output_type, accumulation_data_type)?;query_kernel = Some(kernel);query_scales_buffer = Some(scales_buffer);@@ -81,8 +76,9 @@ impl QKNorm {// Setup key normalization if configuredif let Some(ref k_config) = key_config {let scales_param = parameter_tree.leaf("key_norm.scales").map_err(|e| {- RMSNormError::MetalError(MTLError::Library(crate::backends::metal::error::LibraryError::Custom(- format!("Failed to load key scales: {:?}", e),+ MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(+ "Failed to load key scales: {:?}",+ e)))})?;@@ -98,15 +94,7 @@ impl QKNorm {UpcastMode::FullLayer => (intermediate_data_type, scale_data_type, scale_data_type),};- let kernel = RMSNormKernel::new_with_mode(- context,- input_type,- scales_type,- output_type,- accumulation_data_type,- RMSNormKernelType::QueryKey,- k_config.upcast_mode == UpcastMode::FullLayer,- )?;+ let kernel = QKNormMetalKernel::new(context, input_type, scales_type, output_type, accumulation_data_type)?;key_kernel = Some(kernel);key_scales_buffer = Some(scales_buffer);@@ -171,48 +159,44 @@ impl EncodableBlock<Metal> for QKNorm {if let (Some(query_kernel), Some(query_scales_buffer), Some(query_config)) =(&self.query_kernel, &self.query_scales_buffer, &self.query_config){- if let Err(e) = query_kernel.encode_qk_norm(+ let query_head_count = self.num_q_heads as u32;+ query_kernel.encode(+ qkv_buffer,+ query_scales_buffer,+ qkv_buffer,+ batch_size as u32,+ self.num_q_heads as u32,+ self.num_kv_heads as u32,+ head_dim as u32,+ query_config.epsilon,+ query_config.scale_offset.unwrap_or(0.0),+ 0,+ query_head_count,+ query_config.upcast_mode == UpcastMode::FullLayer,compute_encoder,- QKNormArguments {- qkv_input_buffer: &qkv_buffer,- scales_buffer: query_scales_buffer,- qkv_output_buffer: &qkv_buffer,- batch_size,- // Always pass actual head counts (needed for correct buffer addressing)- num_q_heads: self.num_q_heads as i32,- num_kv_heads: self.num_kv_heads as i32,- head_dim,- epsilon: query_config.epsilon,- scale_offset: query_config.scale_offset.unwrap_or(0.0),- target: QKNormTarget::QueryHeads,- },- ) {- eprintln!("Failed to encode query normalization kernel: {:?}", e);- }+ );}// Process key normalization if configuredif let (Some(key_kernel), Some(key_scales_buffer), Some(key_config)) =(&self.key_kernel, &self.key_scales_buffer, &self.key_config){- if let Err(e) = key_kernel.encode_qk_norm(+ let key_head_count = self.num_kv_heads as u32;+ key_kernel.encode(+ qkv_buffer,+ key_scales_buffer,+ qkv_buffer,+ batch_size as u32,+ self.num_q_heads as u32,+ self.num_kv_heads as u32,+ head_dim as u32,+ key_config.epsilon,+ key_config.scale_offset.unwrap_or(0.0),+ self.num_q_heads as u32,+ key_head_count,+ key_config.upcast_mode == UpcastMode::FullLayer,compute_encoder,- QKNormArguments {- qkv_input_buffer: &qkv_buffer,- scales_buffer: key_scales_buffer,- qkv_output_buffer: &qkv_buffer,- batch_size,- // Always pass actual head counts (needed for correct buffer addressing)- num_q_heads: self.num_q_heads as i32,- num_kv_heads: self.num_kv_heads as i32,- head_dim,- epsilon: key_config.epsilon,- scale_offset: key_config.scale_offset.unwrap_or(0.0),- target: QKNormTarget::KeyHeads,- },- ) {- eprintln!("Failed to encode key normalization kernel: {:?}", e);- }+ );}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rsindex fb54be5be..558b7f041 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs@@ -1,12 +1,13 @@//! RMS Normalization encodable.-use super::super::{EncodableBlock, Metal};use crate::{DataType,- backends::metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,- MTLResourceOptions, ProtocolObject, Retained,- kernel::rms_norm::{RMSNormArguments, RMSNormError, RMSNormKernel, RMSNormKernelType},+ backends::{+ common::kernel::RMSNormKernel,+ metal::{+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt,+ MTLError, MTLResourceOptions, ProtocolObject, Retained, kernel::dsl::RMSNormMetalKernel,+ },},config::{NormalizationConfig, UpcastMode},encodable_block::EncodingParameters,@@ -15,7 +16,7 @@ use crate::{};pub struct RMSNorm {- kernel: RMSNormKernel,+ kernel: RMSNormMetalKernel,config: NormalizationConfig,input_array_id: ArrayId,output_array_id: ArrayId,@@ -31,13 +32,13 @@ impl RMSNorm {input_array_id: ArrayId,output_array_id: ArrayId,parameter_tree: &ParameterTree<MTLContext>,- ) -> Result<Self, RMSNormError> {+ ) -> Result<Self, MTLError> {// Load scales from parameter treelet scales_param = parameter_tree.leaf("scales").map_err(|e| {- RMSNormError::MetalError(MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(+ MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!("Failed to load scales: {:?}",e- ))))+ )))})?;// TODO: Don't create buffers dynamically, we need to use forward pass storage for thing like this@@ -61,16 +62,7 @@ impl RMSNorm {},};- let kernel = RMSNormKernel::new_with_mode(- context,- input_type,- scales_type,- output_type,- accumulation_data_type,- RMSNormKernelType::Standard,- config.upcast_mode == UpcastMode::FullLayer,- )?;-+ let kernel = RMSNormMetalKernel::new(context, input_type, scales_type, output_type, accumulation_data_type)?;Ok(Self {kernel,config,@@ -145,29 +137,24 @@ impl EncodableBlock<Metal> for RMSNorm {}let row_size_in_bytes = input_shape[1] * input_elem_size;- let input_offset = (batch_start * row_size_in_bytes) as u64;+ let input_offset = batch_start * row_size_in_bytes;let output_row_size_in_bytes = input_shape[1] * output_elem_size;- let output_offset = (batch_start * output_row_size_in_bytes) as u64;+ let output_offset = batch_start * output_row_size_in_bytes;let batch_size = batch_len as i32;let model_dim = input_shape[1] as i32;- if let Err(e) = self.kernel.encode(+ self.kernel.encode(+ (input_buffer, input_offset),+ &self.scales_buffer,+ (output_buffer, output_offset),+ batch_size as u32,+ model_dim as u32,+ self.config.epsilon,+ self.config.scale_offset.unwrap_or(0.0),+ self.config.upcast_mode == UpcastMode::FullLayer,compute_encoder,- RMSNormArguments {- input_buffer: &input_buffer,- input_offset,- scales_buffer: &self.scales_buffer,- output_buffer: &output_buffer,- output_offset,- batch_size,- model_dim,- epsilon: self.config.epsilon,- scale_offset: self.config.scale_offset.unwrap_or(0.0),- },- ) {- eprintln!("Failed to encode RMS norm kernel: {:?}", e);- }+ )}}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex c9bd68bed..9cb03bdbd 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -15,7 +15,6 @@ pub use mlp::{pub mod mlp_fused;pub mod moe;pub mod quant_matmul;-pub mod rms_norm;pub mod rope;pub mod short_conv;pub mod ssm;@@ -33,7 +32,6 @@ pub use moe::{MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError, MoeScatterKernels,MoeScatterWithMapArguments,};-pub use rms_norm::{QKNormArguments, QKNormTarget, RMSNormArguments, RMSNormError, RMSNormKernel, RMSNormKernelType};pub use rope::{RopeError, RopeKernel, RopeKernelArguments};pub use short_conv::{ShortConvDecodeArguments, ShortConvKernel, ShortConvKernelError, ShortConvPrefillArguments};pub use ssm::{diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rsdeleted file mode 100644index facef02b7..000000000--- a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs+++ /dev/null@@ -1,280 +0,0 @@-use crate::{- DataType,- backends::metal::{- ComputeEncoderSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,- MTLSize, ProtocolObject, Retained,- },-};--#[derive(Debug, Clone, Copy, PartialEq)]-pub enum RMSNormKernelType {- Standard,- QueryKey,-}--#[derive(Debug, Clone, Copy, PartialEq)]-pub enum QKNormTarget {- QueryHeads,- KeyHeads,- Both,-}--pub struct RMSNormKernel {- pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- kernel_type: RMSNormKernelType,-}--#[derive(Debug)]-pub struct RMSNormArguments<'a> {- pub input_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub input_offset: u64,- pub scales_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub output_offset: u64,- pub batch_size: i32,- pub model_dim: i32,- pub epsilon: f32,- pub scale_offset: f32,-}--#[derive(Debug)]-pub struct QKNormArguments<'a> {- pub qkv_input_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub scales_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub qkv_output_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub batch_size: i32,- pub num_q_heads: i32,- pub num_kv_heads: i32,- pub head_dim: i32,- pub epsilon: f32,- pub scale_offset: f32,- pub target: QKNormTarget,-}--#[derive(Debug, thiserror::Error)]-pub enum RMSNormError {- #[error("Metal error: {0}")]- MetalError(#[from] MTLError),- #[error(- "Unsupported data type combination: input={input:?}, scales={scales:?}, output={output:?}, accumulation={accumulation:?}"- )]- UnsupportedDataType {- input: DataType,- scales: DataType,- output: DataType,- accumulation: DataType,- },- #[error("Invalid kernel type for operation")]- InvalidKernelType,-}--impl RMSNormKernel {- pub fn new(- context: &MTLContext,- input_data_type: DataType,- scales_data_type: DataType,- output_data_type: DataType,- accumulation_data_type: DataType,- kernel_type: RMSNormKernelType,- ) -> Result<Self, RMSNormError> {- Self::new_with_mode(- context,- input_data_type,- scales_data_type,- output_data_type,- accumulation_data_type,- kernel_type,- false,- )- }-- pub fn new_with_mode(- context: &MTLContext,- input_data_type: DataType,- scales_data_type: DataType,- output_data_type: DataType,- accumulation_data_type: DataType,- kernel_type: RMSNormKernelType,- full_layer: bool,- ) -> Result<Self, RMSNormError> {- let function_name = Self::kernel_name_for_types(- input_data_type,- scales_data_type,- output_data_type,- accumulation_data_type,- kernel_type,- full_layer,- )?;-- let pipeline = context.compute_pipeline_state(&function_name, None).map_err(RMSNormError::MetalError)?;-- Ok(Self {- pipeline,- kernel_type,- })- }-- /// Generate kernel name from data type combination- fn kernel_name_for_types(- input_dt: DataType,- scales_dt: DataType,- output_dt: DataType,- accum_dt: DataType,- kernel_type: RMSNormKernelType,- full_layer: bool,- ) -> Result<String, RMSNormError> {- let input_suffix = Self::data_type_to_suffix(input_dt)?;- let scales_suffix = Self::data_type_to_suffix(scales_dt)?;- let output_suffix = Self::data_type_to_suffix(output_dt)?;- let accum_suffix = Self::accum_type_to_suffix(accum_dt)?;-- let base_name = match kernel_type {- RMSNormKernelType::Standard => "rms_norm",- RMSNormKernelType::QueryKey => "qk_norm",- };-- let mode_suffix = if full_layer {- "_full"- } else {- "_norm"- };-- Ok(format!(- "{}_{}_{}_{}_{}{}",- base_name, input_suffix, scales_suffix, output_suffix, accum_suffix, mode_suffix- ))- }-- fn data_type_to_suffix(data_type: DataType) -> Result<&'static str, RMSNormError> {- match data_type {- DataType::F32 => Ok("f32"),- DataType::F16 => Ok("f16"),- DataType::BF16 => Ok("bf16"),- _ => Err(RMSNormError::UnsupportedDataType {- input: data_type,- scales: data_type,- output: data_type,- accumulation: data_type,- }),- }- }-- fn accum_type_to_suffix(data_type: DataType) -> Result<&'static str, RMSNormError> {- match data_type {- DataType::F32 => Ok("f32"),- DataType::F16 => Ok("f16"),- _ => Err(RMSNormError::UnsupportedDataType {- input: data_type,- scales: data_type,- output: data_type,- accumulation: data_type,- }),- }- }-- pub fn encode(- &self,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: RMSNormArguments,- ) -> Result<(), RMSNormError> {- if self.kernel_type != RMSNormKernelType::Standard {- return Err(RMSNormError::InvalidKernelType);- }-- compute_encoder.set_compute_pipeline_state(&self.pipeline);-- // Set buffers- compute_encoder.set_buffer(Some(args.input_buffer), args.input_offset as usize, 0);- compute_encoder.set_buffer(Some(args.scales_buffer), 0, 1);- compute_encoder.set_buffer(Some(args.output_buffer), args.output_offset as usize, 2);-- // Set parameters- compute_encoder.set_value(&args.batch_size, 3);- compute_encoder.set_value(&args.model_dim, 4);- compute_encoder.set_value(&args.epsilon, 5);- compute_encoder.set_value(&args.scale_offset, 6);-- let threads_per_threadgroup = MTLSize {- width: 1024,- height: 1,- depth: 1,- };-- let threadgroups_per_grid = MTLSize {- width: args.batch_size as usize,- height: 1,- depth: 1,- };-- compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);-- Ok(())- }-- pub fn encode_qk_norm(- &self,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: QKNormArguments,- ) -> Result<(), RMSNormError> {- if self.kernel_type != RMSNormKernelType::QueryKey {- return Err(RMSNormError::InvalidKernelType);- }-- compute_encoder.set_compute_pipeline_state(&self.pipeline);-- // Set buffers- compute_encoder.set_buffer(Some(args.qkv_input_buffer), 0, 0);- compute_encoder.set_buffer(Some(args.scales_buffer), 0, 1);- compute_encoder.set_buffer(Some(args.qkv_output_buffer), 0, 2);-- // Set parameters- compute_encoder.set_value(&args.batch_size, 3);- compute_encoder.set_value(&args.num_q_heads, 4);- compute_encoder.set_value(&args.num_kv_heads, 5);- compute_encoder.set_value(&args.head_dim, 6);- compute_encoder.set_value(&args.epsilon, 7);- compute_encoder.set_value(&args.scale_offset, 8);-- // Determine which contiguous head range to normalize.- //- // QKV layout per token: [Q heads][K heads][V heads]- // We only ever normalize within [Q] and/or [K].- let (head_offset, head_count): (u32, u32) = match args.target {- QKNormTarget::QueryHeads => (0, args.num_q_heads as u32),- QKNormTarget::KeyHeads => (args.num_q_heads as u32, args.num_kv_heads as u32),- QKNormTarget::Both => (0, (args.num_q_heads + args.num_kv_heads) as u32),- };-- if args.batch_size <= 0 || args.head_dim <= 0 || head_count == 0 {- return Ok(());- }-- // Pass head range to the kernel.- compute_encoder.set_value(&head_offset, 9);- compute_encoder.set_value(&head_count, 10);-- // One SIMD-group per head, multiple heads per threadgroup.- let simd_width = self.pipeline.thread_execution_width();- let max_threads = self.pipeline.max_total_threads_per_threadgroup();- let max_heads_per_threadgroup = (max_threads / simd_width).max(1);- let heads_per_threadgroup = (head_count as usize).min(max_heads_per_threadgroup).max(1);-- let num_head_tiles = (head_count as usize).div_ceil(heads_per_threadgroup);-- let threadgroups_per_grid = MTLSize {- width: args.batch_size as usize,- height: num_head_tiles,- depth: 1,- };-- let threads_per_threadgroup = MTLSize {- width: heads_per_threadgroup * simd_width,- height: 1,- depth: 1,- };-- compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);-- Ok(())- }-}diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metalnew file mode 100644index 000000000..e87ca4a89--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal@@ -0,0 +1,108 @@+#include <metal_stdlib>+#include "../definitions.metal"++using namespace metal;++#define SIMD_SIZE 32+#define GRAIN_SIZE 4++// QK norm: normalize per-head vectors (small head_dim) efficiently.+//+// Strategy:+// - One SIMD-group (32 threads) processes one head.+// - One threadgroup (one SIMD-group) is dispatched per head.+template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>+VARIANTS(InputT, float, half, bfloat)+VARIANTS(ScaleT, float, half, bfloat)+VARIANTS(OutputT, float, half, bfloat)+VARIANTS(AccumT, float, half)+KERNEL(QKNorm)(+ const device InputT* qkv_input,+ const device ScaleT* scales,+ device OutputT* qkv_output,+ constant uint& batch_size,+ constant uint& num_q_heads,+ constant uint& num_kv_heads,+ constant uint& head_dim,+ constant float& epsilon,+ constant float& scale_offset,+ constant uint& head_offset,+ constant uint& head_count,+ constant bool& full_layer,+ const uint batch_idx GROUPS(batch_size),+ const uint head_idx GROUPS(head_count),+ const uint lane_id THREADS(SIMD_SIZE)+) {+ if (head_count == 0u || head_dim == 0u)+ return;++ const uint total_heads_in_buffer = num_q_heads + 2u * num_kv_heads;+ const uint logical_head_idx = head_offset + head_idx;+ if (logical_head_idx >= total_heads_in_buffer)+ return;++ const ulong slice_offset =+ (ulong)batch_idx * (ulong)total_heads_in_buffer * (ulong)head_dim ++ (ulong)logical_head_idx * (ulong)head_dim;++ const device InputT* input_data = qkv_input + slice_offset;+ const device ScaleT* scales_data = scales;+ device OutputT* output_data = qkv_output + slice_offset;+ const uint element_count = head_dim;++ AccumT partial_sum = static_cast<AccumT>(0.0f);++ // Sum of squares: each lane processes GRAIN_SIZE elements per iteration.+ for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;+ base_i += SIMD_SIZE * GRAIN_SIZE) {+ AccumT vals[GRAIN_SIZE];+ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;+ }+ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ partial_sum += vals[j] * vals[j];+ }+ }++ // SIMD-group reduction.+ AccumT total_sum = simd_sum(partial_sum);++ // RMS factor.+ AccumT mean_square =+ static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);+ AccumT rms_norm = rsqrt(mean_square + static_cast<AccumT>(epsilon));++ // Normalize + scale.+ for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;+ base_i += SIMD_SIZE * GRAIN_SIZE) {+ AccumT vals[GRAIN_SIZE];+ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;+ }++ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ if (i >= element_count)+ continue;++ AccumT normalized_high = vals[j] * rms_norm;++ if (full_layer) {+ AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) ++ static_cast<AccumT>(scale_offset);+ output_data[i] =+ static_cast<OutputT>(normalized_high * scale_value_high);+ } else {+ OutputT normalized_low = static_cast<OutputT>(normalized_high);+ OutputT scale_value_low = static_cast<OutputT>(+ static_cast<AccumT>(scales_data[i]) ++ static_cast<AccumT>(scale_offset)+ );+ OutputT product_low = normalized_low * scale_value_low;+ output_data[i] = static_cast<OutputT>(product_low);+ }+ }+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex 5cfda4c7e..a744e065f 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -7,19 +7,29 @@ using namespace metal;#define SIMD_SIZE 32#define GRAIN_SIZE 4-// Main template kernel - single implementation for all typestemplate <typename InputT, typename ScaleT, typename OutputT, typename AccumT>-void rms_norm_core(- const device InputT* input_data,- const device ScaleT* scales_data,- device OutputT* output_data,- uint element_count,+VARIANTS(InputT, float, half, bfloat)+VARIANTS(ScaleT, float, half, bfloat)+VARIANTS(OutputT, float, half, bfloat)+VARIANTS(AccumT, float, half)+KERNEL(RMSNorm)(+ const device InputT* input,+ const device ScaleT* scales,+ device OutputT* output,+ constant uint& batch_size,+ constant uint& element_count,constant float& epsilon,constant float& scale_offset,- threadgroup AccumT* shared_sum,- uint thread_in_row,- bool full_layer+ constant bool& full_layer,+ threadgroup AccumT shared_sum[SIMD_SIZE],+ const uint batch_idx GROUPS(batch_size),+ const uint thread_in_row THREADS(1024)) {+ const uint input_offset = batch_idx * element_count;+ const device InputT* input_data = input + input_offset;+ const device ScaleT* scales_data = scales;+ device OutputT* output_data = output + input_offset;+AccumT partial_sum = static_cast<AccumT>(0.0f);// Compute thread local partial sum@@ -96,302 +106,3 @@ void rms_norm_core(}}}--// X-macro table defining all supported type combinations-// Format: (InputType, ScaleType, OutputType, AccumType, full_layer_mode,-// suffix)-#define FOREACH_RMS_COMBO(_) \- /* float combinations */ \- _(float, float, float, float, false, f32_f32_f32_f32_norm) \- _(float, float, float, float, true, f32_f32_f32_f32_full) \- _(float, float, half, float, false, f32_f32_f16_f32_norm) \- _(float, float, half, float, true, f32_f32_f16_f32_full) \- _(float, float, bfloat, float, false, f32_f32_bf16_f32_norm) \- _(float, float, bfloat, float, true, f32_f32_bf16_f32_full) \- _(float, half, float, float, false, f32_f16_f32_f32_norm) \- _(float, half, float, float, true, f32_f16_f32_f32_full) \- _(float, half, half, float, false, f32_f16_f16_f32_norm) \- _(float, half, half, float, true, f32_f16_f16_f32_full) \- _(float, half, bfloat, float, false, f32_f16_bf16_f32_norm) \- _(float, half, bfloat, float, true, f32_f16_bf16_f32_full) \- _(float, bfloat, float, float, false, f32_bf16_f32_f32_norm) \- _(float, bfloat, float, float, true, f32_bf16_f32_f32_full) \- _(float, bfloat, half, float, false, f32_bf16_f16_f32_norm) \- _(float, bfloat, half, float, true, f32_bf16_f16_f32_full) \- _(float, bfloat, bfloat, float, false, f32_bf16_bf16_f32_norm) \- _(float, bfloat, bfloat, float, true, f32_bf16_bf16_f32_full) \- \- /* half combinations */ \- _(half, float, float, float, false, f16_f32_f32_f32_norm) \- _(half, float, float, float, true, f16_f32_f32_f32_full) \- _(half, float, half, float, false, f16_f32_f16_f32_norm) \- _(half, float, half, float, true, f16_f32_f16_f32_full) \- _(half, float, bfloat, float, false, f16_f32_bf16_f32_norm) \- _(half, float, bfloat, float, true, f16_f32_bf16_f32_full) \- _(half, half, float, float, false, f16_f16_f32_f32_norm) \- _(half, half, float, float, true, f16_f16_f32_f32_full) \- _(half, half, half, float, false, f16_f16_f16_f32_norm) \- _(half, half, half, float, true, f16_f16_f16_f32_full) \- _(half, half, bfloat, float, false, f16_f16_bf16_f32_norm) \- _(half, half, bfloat, float, true, f16_f16_bf16_f32_full) \- _(half, bfloat, float, float, false, f16_bf16_f32_f32_norm) \- _(half, bfloat, float, float, true, f16_bf16_f32_f32_full) \- _(half, bfloat, half, float, false, f16_bf16_f16_f32_norm) \- _(half, bfloat, half, float, true, f16_bf16_f16_f32_full) \- _(half, bfloat, bfloat, float, false, f16_bf16_bf16_f32_norm) \- _(half, bfloat, bfloat, float, true, f16_bf16_bf16_f32_full) \- \- /* bfloat combinations */ \- _(bfloat, float, float, float, false, bf16_f32_f32_f32_norm) \- _(bfloat, float, float, float, true, bf16_f32_f32_f32_full) \- _(bfloat, float, half, float, false, bf16_f32_f16_f32_norm) \- _(bfloat, float, half, float, true, bf16_f32_f16_f32_full) \- _(bfloat, float, bfloat, float, false, bf16_f32_bf16_f32_norm) \- _(bfloat, float, bfloat, float, true, bf16_f32_bf16_f32_full) \- _(bfloat, half, float, float, false, bf16_f16_f32_f32_norm) \- _(bfloat, half, float, float, true, bf16_f16_f32_f32_full) \- _(bfloat, half, half, float, false, bf16_f16_f16_f32_norm) \- _(bfloat, half, half, float, true, bf16_f16_f16_f32_full) \- _(bfloat, half, bfloat, float, false, bf16_f16_bf16_f32_norm) \- _(bfloat, half, bfloat, float, true, bf16_f16_bf16_f32_full) \- _(bfloat, bfloat, float, float, false, bf16_bf16_f32_f32_norm) \- _(bfloat, bfloat, float, float, true, bf16_bf16_f32_f32_full) \- _(bfloat, bfloat, half, float, false, bf16_bf16_f16_f32_norm) \- _(bfloat, bfloat, half, float, true, bf16_bf16_f16_f32_full) \- _(bfloat, bfloat, bfloat, float, false, bf16_bf16_bf16_f32_norm) \- _(bfloat, bfloat, bfloat, float, true, bf16_bf16_bf16_f32_full) \- \- /* All combinations with half accumulation */ \- _(float, float, float, half, false, f32_f32_f32_f16_norm) \- _(float, float, float, half, true, f32_f32_f32_f16_full) \- _(float, float, half, half, false, f32_f32_f16_f16_norm) \- _(float, float, half, half, true, f32_f32_f16_f16_full) \- _(float, float, bfloat, half, false, f32_f32_bf16_f16_norm) \- _(float, float, bfloat, half, true, f32_f32_bf16_f16_full) \- _(float, half, float, half, false, f32_f16_f32_f16_norm) \- _(float, half, float, half, true, f32_f16_f32_f16_full) \- _(float, half, half, half, false, f32_f16_f16_f16_norm) \- _(float, half, half, half, true, f32_f16_f16_f16_full) \- _(float, half, bfloat, half, false, f32_f16_bf16_f16_norm) \- _(float, half, bfloat, half, true, f32_f16_bf16_f16_full) \- _(float, bfloat, float, half, false, f32_bf16_f32_f16_norm) \- _(float, bfloat, float, half, true, f32_bf16_f32_f16_full) \- _(float, bfloat, half, half, false, f32_bf16_f16_f16_norm) \- _(float, bfloat, half, half, true, f32_bf16_f16_f16_full) \- _(float, bfloat, bfloat, half, false, f32_bf16_bf16_f16_norm) \- _(float, bfloat, bfloat, half, true, f32_bf16_bf16_f16_full) \- \- _(half, float, float, half, false, f16_f32_f32_f16_norm) \- _(half, float, float, half, true, f16_f32_f32_f16_full) \- _(half, float, half, half, false, f16_f32_f16_f16_norm) \- _(half, float, half, half, true, f16_f32_f16_f16_full) \- _(half, float, bfloat, half, false, f16_f32_bf16_f16_norm) \- _(half, float, bfloat, half, true, f16_f32_bf16_f16_full) \- _(half, half, float, half, false, f16_f16_f32_f16_norm) \- _(half, half, float, half, true, f16_f16_f32_f16_full) \- _(half, half, half, half, false, f16_f16_f16_f16_norm) \- _(half, half, half, half, true, f16_f16_f16_f16_full) \- _(half, half, bfloat, half, false, f16_f16_bf16_f16_norm) \- _(half, half, bfloat, half, true, f16_f16_bf16_f16_full) \- _(half, bfloat, float, half, false, f16_bf16_f32_f16_norm) \- _(half, bfloat, float, half, true, f16_bf16_f32_f16_full) \- _(half, bfloat, half, half, false, f16_bf16_f16_f16_norm) \- _(half, bfloat, half, half, true, f16_bf16_f16_f16_full) \- _(half, bfloat, bfloat, half, false, f16_bf16_bf16_f16_norm) \- _(half, bfloat, bfloat, half, true, f16_bf16_bf16_f16_full) \- \- _(bfloat, float, float, half, false, bf16_f32_f32_f16_norm) \- _(bfloat, float, float, half, true, bf16_f32_f32_f16_full) \- _(bfloat, float, half, half, false, bf16_f32_f16_f16_norm) \- _(bfloat, float, half, half, true, bf16_f32_f16_f16_full) \- _(bfloat, float, bfloat, half, false, bf16_f32_bf16_f16_norm) \- _(bfloat, float, bfloat, half, true, bf16_f32_bf16_f16_full) \- _(bfloat, half, float, half, false, bf16_f16_f32_f16_norm) \- _(bfloat, half, float, half, true, bf16_f16_f32_f16_full) \- _(bfloat, half, half, half, false, bf16_f16_f16_f16_norm) \- _(bfloat, half, half, half, true, bf16_f16_f16_f16_full) \- _(bfloat, half, bfloat, half, false, bf16_f16_bf16_f16_norm) \- _(bfloat, half, bfloat, half, true, bf16_f16_bf16_f16_full) \- _(bfloat, bfloat, float, half, false, bf16_bf16_f32_f16_norm) \- _(bfloat, bfloat, float, half, true, bf16_bf16_f32_f16_full) \- _(bfloat, bfloat, half, half, false, bf16_bf16_f16_f16_norm) \- _(bfloat, bfloat, half, half, true, bf16_bf16_f16_f16_full) \- _(bfloat, bfloat, bfloat, half, false, bf16_bf16_bf16_f16_norm) \- _(bfloat, bfloat, bfloat, half, true, bf16_bf16_bf16_f16_full)--// Generate RMS norm kernels-#define DEFINE_RMS_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \- [[max_total_threads_per_threadgroup(1024)]] kernel void rms_norm_##SUF( \- const device IN* input [[buffer(0)]], \- const device SC* scales [[buffer(1)]], \- device OUT* output [[buffer(2)]], \- constant uint& batch_size [[buffer(3)]], \- constant uint& model_dim [[buffer(4)]], \- constant float& epsilon [[buffer(5)]], \- constant float& scale_offset [[buffer(6)]], \- uint batch_idx [[threadgroup_position_in_grid]], \- uint thread_in_row [[thread_position_in_threadgroup]] \- ) { \- if (batch_idx >= batch_size) \- return; \- \- threadgroup ACC shared_sum[SIMD_SIZE]; \- const uint input_offset = batch_idx * model_dim; \- \- rms_norm_core<IN, SC, OUT, ACC>( \- input + input_offset, \- scales, \- output + input_offset, \- model_dim, \- epsilon, \- scale_offset, \- shared_sum, \- thread_in_row, \- FULL_LAYER \- ); \- }--FOREACH_RMS_COMBO(DEFINE_RMS_KERNEL)-#undef DEFINE_RMS_KERNEL--// QK norm: normalize per-head vectors (small head_dim) efficiently.-//-// Strategy:-// - One SIMD-group (32 threads) processes one head.-// - A threadgroup processes multiple heads for a given token (batch).-// - No threadgroup-wide reductions/barriers: we only use simd_sum within a-// SIMD-group.-template <- typename InputT,- typename ScaleT,- typename OutputT,- typename AccumT,- bool FULL_LAYER>-static inline void qk_norm_head_core(- const device InputT* input_data,- const device ScaleT* scales_data,- device OutputT* output_data,- uint element_count,- constant float& epsilon,- constant float& scale_offset,- uint lane_id-) {- if (element_count == 0u)- return;-- AccumT partial_sum = static_cast<AccumT>(0.0f);-- // Sum of squares: each lane processes GRAIN_SIZE elements per iteration.- for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;- base_i += SIMD_SIZE * GRAIN_SIZE) {- AccumT vals[GRAIN_SIZE];- for (uint j = 0; j < GRAIN_SIZE; ++j) {- uint i = base_i + j;- vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;- }- for (uint j = 0; j < GRAIN_SIZE; ++j) {- partial_sum += vals[j] * vals[j];- }- }-- // SIMD-group reduction.- AccumT total_sum = simd_sum(partial_sum);-- // RMS factor.- AccumT mean_square =- static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);- AccumT rms_norm = rsqrt(mean_square + static_cast<AccumT>(epsilon));-- // Normalize + scale.- for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;- base_i += SIMD_SIZE * GRAIN_SIZE) {- AccumT vals[GRAIN_SIZE];- for (uint j = 0; j < GRAIN_SIZE; ++j) {- uint i = base_i + j;- vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;- }-- for (uint j = 0; j < GRAIN_SIZE; ++j) {- uint i = base_i + j;- if (i >= element_count)- continue;-- AccumT normalized_high = vals[j] * rms_norm;-- if (FULL_LAYER) {- AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) +- static_cast<AccumT>(scale_offset);- output_data[i] =- static_cast<OutputT>(normalized_high * scale_value_high);- } else {- OutputT normalized_low = static_cast<OutputT>(normalized_high);- OutputT scale_value_low = static_cast<OutputT>(- static_cast<AccumT>(scales_data[i]) +- static_cast<AccumT>(scale_offset)- );- OutputT product_low = normalized_low * scale_value_low;- output_data[i] = static_cast<OutputT>(product_low);- }- }- }-}--// Generate QK norm kernels (batched, head-tiled).-#define DEFINE_QK_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \- [[max_total_threads_per_threadgroup(1024)]] kernel void qk_norm_##SUF( \- const device IN* qkv_input [[buffer(0)]], \- const device SC* scales [[buffer(1)]], \- device OUT* qkv_output [[buffer(2)]], \- constant uint& batch_size [[buffer(3)]], \- constant uint& num_q_heads [[buffer(4)]], \- constant uint& num_kv_heads [[buffer(5)]], \- constant uint& head_dim [[buffer(6)]], \- constant float& epsilon [[buffer(7)]], \- constant float& scale_offset [[buffer(8)]], \- constant uint& head_offset [[buffer(9)]], \- constant uint& head_count [[buffer(10)]], \- uint3 tgid [[threadgroup_position_in_grid]], \- uint lid [[thread_index_in_threadgroup]], \- uint3 threads_per_threadgroup [[threads_per_threadgroup]] \- ) { \- const uint batch_idx = tgid.x; \- const uint tile_idx = tgid.y; \- \- if (batch_idx >= batch_size) \- return; \- if (head_count == 0u || head_dim == 0u) \- return; \- \- const uint heads_per_tg = threads_per_threadgroup.x / SIMD_SIZE; \- if (heads_per_tg == 0u) \- return; \- \- const uint simd_group_id = lid / SIMD_SIZE; \- const uint lane_id = lid % SIMD_SIZE; \- const uint head_idx = tile_idx * heads_per_tg + simd_group_id; \- \- if (head_idx >= head_count) \- return; \- \- const uint total_heads_in_buffer = num_q_heads + 2u * num_kv_heads; \- const uint logical_head_idx = head_offset + head_idx; \- if (logical_head_idx >= total_heads_in_buffer) \- return; \- \- const ulong slice_offset = \- (ulong)batch_idx * (ulong)total_heads_in_buffer * (ulong)head_dim + \- (ulong)logical_head_idx * (ulong)head_dim; \- \- qk_norm_head_core<IN, SC, OUT, ACC, FULL_LAYER>( \- qkv_input + slice_offset, \- scales, \- qkv_output + slice_offset, \- head_dim, \- epsilon, \- scale_offset, \- lane_id \- ); \- }--FOREACH_RMS_COMBO(DEFINE_QK_KERNEL)-#undef DEFINE_QK_KERNELdiff --git a/crates/uzu/tests/rms_norm_kernel_test.rs b/crates/uzu/tests/rms_norm_kernel_test.rsindex 40c9e8dd3..f8bdd4d08 100644--- a/crates/uzu/tests/rms_norm_kernel_test.rs+++ b/crates/uzu/tests/rms_norm_kernel_test.rs@@ -7,10 +7,13 @@ use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLuse uzu::{DataType,backends::{- common::Context,+ common::{+ Context,+ kernel::{QKNormKernel, RMSNormKernel},+ },metal::{MTLContext,- kernel::rms_norm::{QKNormArguments, QKNormTarget, RMSNormArguments, RMSNormKernel, RMSNormKernelType},+ kernel::dsl::{QKNormMetalKernel, RMSNormMetalKernel},metal_extensions::CommandBufferTimingExt,},},@@ -301,34 +304,23 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(.expect("Failed to create buffer");// Create RMS norm kernel- let kernel = RMSNormKernel::new(- &mtl_context,- input_type,- scale_type,- output_type,- accumulation_type,- RMSNormKernelType::Standard,- )- .expect("Failed to create RMS norm kernel");-+ let kernel = RMSNormMetalKernel::new(&mtl_context, input_type, scale_type, output_type, accumulation_type)+ .expect("Failed to create RMS norm kernel");// Create command buffer and encodelet command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- let _ = kernel.encode(+ kernel.encode(+ &input_buffer,+ &scale_buffer,+ &output_buffer,+ batch_size as u32,+ model_dim as u32,+ epsilon,+ 0.0,+ false,&compute_encoder,- RMSNormArguments {- input_buffer: &input_buffer,- input_offset: 0,- scales_buffer: &scale_buffer,- output_buffer: &output_buffer,- output_offset: 0,- batch_size,- model_dim,- epsilon,- scale_offset: 0.0,- },);compute_encoder.end_encoding();@@ -423,33 +415,23 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(.new_buffer(4 * OutputT::size_of(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let kernel = RMSNormKernel::new(- &mtl_context,- input_type,- scale_type,- output_type,- accumulation_type,- RMSNormKernelType::Standard,- )- .expect("Failed to create RMS norm kernel");+ let kernel = RMSNormMetalKernel::new(&mtl_context, input_type, scale_type, output_type, accumulation_type)+ .expect("Failed to create RMS norm kernel");let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- let _ = kernel.encode(+ kernel.encode(+ &input_buffer,+ &scale_buffer,+ &output_buffer,+ batch_size as u32,+ model_dim as u32,+ epsilon,+ 0.0,+ false,&compute_encoder,- RMSNormArguments {- input_buffer: &input_buffer,- input_offset: 0,- scales_buffer: &scale_buffer,- output_buffer: &output_buffer,- output_offset: 0,- batch_size,- model_dim,- epsilon,- scale_offset: 0.0,- },);compute_encoder.end_encoding();@@ -602,15 +584,8 @@ fn perf_rms_norm_with_size(const EPSILON: f32 = 1e-6;// ---- Create kernel ----- let kernel = RMSNormKernel::new(- &mtl_context,- DataType::F32,- DataType::F32,- DataType::F32,- DataType::F32,- RMSNormKernelType::Standard,- )- .expect("Failed to create RMS norm kernel");+ let kernel = RMSNormMetalKernel::new(&mtl_context, DataType::F32, DataType::F32, DataType::F32, DataType::F32)+ .expect("Failed to create RMS norm kernel");// ---- Generate random data ----let mut rng = StdRng::seed_from_u64(42);@@ -648,19 +623,16 @@ fn perf_rms_norm_with_size(let command_buffer = command_buffer_ref.to_owned();let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- let _ = kernel.encode(+ kernel.encode(+ &input_buffer,+ &scale_buffer,+ &output_buffer,+ batch_size as u32,+ model_dim as u32,+ EPSILON,+ 0.0,+ false,&compute_encoder,- RMSNormArguments {- input_buffer: &input_buffer,- input_offset: 0,- scales_buffer: &scale_buffer,- output_buffer: &output_buffer,- output_offset: 0,- batch_size,- model_dim,- epsilon: EPSILON,- scale_offset: 0.0,- },);compute_encoder.end_encoding();@@ -795,47 +767,31 @@ fn qk_norm_test() {.expect("Failed to create buffer");// Create QK norm kernels- let q_kernel = RMSNormKernel::new_with_mode(- &mtl_context,- DataType::F32,- DataType::F32,- DataType::F32,- DataType::F32,- RMSNormKernelType::QueryKey,- false,- )- .expect("Failed to create Q norm kernel");-- let k_kernel = RMSNormKernel::new_with_mode(- &mtl_context,- DataType::F32,- DataType::F32,- DataType::F32,- DataType::F32,- RMSNormKernelType::QueryKey,- false,- )- .expect("Failed to create K norm kernel");+ let q_kernel = QKNormMetalKernel::new(&mtl_context, DataType::F32, DataType::F32, DataType::F32, DataType::F32)+ .expect("Failed to create Q norm kernel");++ let k_kernel = QKNormMetalKernel::new(&mtl_context, DataType::F32, DataType::F32, DataType::F32, DataType::F32)+ .expect("Failed to create K norm kernel");// Test Q head normalization{let command_buffer = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- let _ = q_kernel.encode_qk_norm(+ q_kernel.encode(+ &qkv_buffer,+ &q_scales_buffer,+ &qkv_buffer,+ batch_size as u32,+ num_q_heads as u32,+ num_kv_heads as u32,+ head_dim as u32,+ epsilon,+ 0.0,+ 0,+ num_q_heads as u32,+ false,&compute_encoder,- QKNormArguments {- qkv_input_buffer: &qkv_buffer,- scales_buffer: &q_scales_buffer,- qkv_output_buffer: &qkv_buffer,- batch_size,- num_q_heads, // Now correctly passes actual head count- num_kv_heads, // Now correctly passes actual head count- head_dim,- epsilon,- scale_offset: 0.0,- target: QKNormTarget::QueryHeads, // Only process Q heads- },);compute_encoder.end_encoding();@@ -884,20 +840,20 @@ fn qk_norm_test() {let command_buffer = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- let _ = k_kernel.encode_qk_norm(+ k_kernel.encode(+ &qkv_buffer,+ &k_scales_buffer,+ &qkv_buffer,+ batch_size as u32,+ num_q_heads as u32,+ num_kv_heads as u32,+ head_dim as u32,+ epsilon,+ 0.0,+ num_q_heads as u32,+ num_kv_heads as u32,+ false,&compute_encoder,- QKNormArguments {- qkv_input_buffer: &qkv_buffer,- scales_buffer: &k_scales_buffer,- qkv_output_buffer: &qkv_buffer,- batch_size,- num_q_heads, // Now correctly passes actual head count- num_kv_heads, // Now correctly passes actual head count- head_dim,- epsilon,- scale_offset: 0.0,- target: QKNormTarget::KeyHeads, // Only process K heads- },);compute_encoder.end_encoding();