Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

dslify-rms-norm (#166)

Aleksandr Golokoz616dcb1605cd
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
index 798c31d93..f0b8bf99d 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs
@@ -41,7 +41,7 @@ pub use linear::{FullPrecisionLinear, QuantizedLinear};
pub(crate) use mamba_mixer::MambaMixer;
pub use mlp::{MlpBlock, MlpFusedBlock, MlpFusedUpKernel};
pub use moe_block::{MoeBlock, SharedMoeWeights};
-pub use normalization::{LayerNorm, Normalization, NormalizationError, QKNorm, RMSNorm};
+pub use normalization::{LayerNorm, Normalization, QKNorm, RMSNorm};
pub use pooling::Pooling;
pub use prediction_head::ClassifierPredictionHead;
pub use rope::Rope;
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
index 8c281fb0d..8a0e233fb 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
@@ -11,25 +11,13 @@ pub use rms_norm::RMSNorm;
use super::{EncodableBlock, Metal};
use crate::{
DataType,
- backends::metal::{
- MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject, Retained,
- kernel::RMSNormError,
- },
+ backends::metal::{MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject, Retained},
config::NormalizationConfig,
encodable_block::EncodingParameters,
forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
-/// Unified error type for normalization operations.
-#[derive(Debug, thiserror::Error)]
-pub enum NormalizationError {
- #[error("LayerNorm error: {0}")]
- LayerNorm(#[from] MTLError),
- #[error("RMSNorm error: {0}")]
- RMSNorm(#[from] RMSNormError),
-}
-
/// Unified normalization encodable that can be either LayerNorm or RMSNorm.
pub enum Normalization {
LayerNorm(LayerNorm),
@@ -46,7 +34,7 @@ impl Normalization {
input_array_id: ArrayId,
output_array_id: ArrayId,
parameter_tree: &ParameterTree<MTLContext>,
- ) -> Result<Self, NormalizationError> {
+ ) -> Result<Self, MTLError> {
if config.subtract_mean {
// Use LayerNorm (subtract mean before normalization)
Ok(Normalization::LayerNorm(LayerNorm::new(
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
index b429d68d0..2f1277456 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
@@ -3,10 +3,12 @@
use super::super::{EncodableBlock, Metal};
use crate::{
DataType,
- backends::metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,
- MTLResourceOptions, ProtocolObject, Retained,
- kernel::rms_norm::{QKNormArguments, QKNormTarget, RMSNormError, RMSNormKernel, RMSNormKernelType},
+ backends::{
+ common::kernel::QKNormKernel,
+ metal::{
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt,
+ MTLError, MTLResourceOptions, ProtocolObject, Retained, kernel::dsl::QKNormMetalKernel,
+ },
},
config::{NormalizationConfig, UpcastMode},
encodable_block::EncodingParameters,
@@ -15,8 +17,8 @@ use crate::{
};
pub struct QKNorm {
- query_kernel: Option<RMSNormKernel>,
- key_kernel: Option<RMSNormKernel>,
+ query_kernel: Option<QKNormMetalKernel>,
+ key_kernel: Option<QKNormMetalKernel>,
query_config: Option<NormalizationConfig>,
key_config: Option<NormalizationConfig>,
qkv_array_id: ArrayId,
@@ -38,7 +40,7 @@ impl QKNorm {
num_q_heads: usize,
num_kv_heads: usize,
head_dim: usize,
- ) -> Result<Self, RMSNormError> {
+ ) -> Result<Self, MTLError> {
let mut query_kernel = None;
let mut key_kernel = None;
let mut query_scales_buffer = None;
@@ -47,8 +49,9 @@ impl QKNorm {
// Setup query normalization if configured
if let Some(ref q_config) = query_config {
let scales_param = parameter_tree.leaf("query_norm.scales").map_err(|e| {
- RMSNormError::MetalError(MTLError::Library(crate::backends::metal::error::LibraryError::Custom(
- format!("Failed to load query scales: {:?}", e),
+ MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(
+ "Failed to load query scales: {:?}",
+ e
)))
})?;
@@ -64,15 +67,7 @@ impl QKNorm {
UpcastMode::FullLayer => (intermediate_data_type, scale_data_type, scale_data_type),
};
- let kernel = RMSNormKernel::new_with_mode(
- context,
- input_type,
- scales_type,
- output_type,
- accumulation_data_type,
- RMSNormKernelType::QueryKey,
- q_config.upcast_mode == UpcastMode::FullLayer,
- )?;
+ let kernel = QKNormMetalKernel::new(context, input_type, scales_type, output_type, accumulation_data_type)?;
query_kernel = Some(kernel);
query_scales_buffer = Some(scales_buffer);
@@ -81,8 +76,9 @@ impl QKNorm {
// Setup key normalization if configured
if let Some(ref k_config) = key_config {
let scales_param = parameter_tree.leaf("key_norm.scales").map_err(|e| {
- RMSNormError::MetalError(MTLError::Library(crate::backends::metal::error::LibraryError::Custom(
- format!("Failed to load key scales: {:?}", e),
+ MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(
+ "Failed to load key scales: {:?}",
+ e
)))
})?;
@@ -98,15 +94,7 @@ impl QKNorm {
UpcastMode::FullLayer => (intermediate_data_type, scale_data_type, scale_data_type),
};
- let kernel = RMSNormKernel::new_with_mode(
- context,
- input_type,
- scales_type,
- output_type,
- accumulation_data_type,
- RMSNormKernelType::QueryKey,
- k_config.upcast_mode == UpcastMode::FullLayer,
- )?;
+ let kernel = QKNormMetalKernel::new(context, input_type, scales_type, output_type, accumulation_data_type)?;
key_kernel = Some(kernel);
key_scales_buffer = Some(scales_buffer);
@@ -171,48 +159,44 @@ impl EncodableBlock<Metal> for QKNorm {
if let (Some(query_kernel), Some(query_scales_buffer), Some(query_config)) =
(&self.query_kernel, &self.query_scales_buffer, &self.query_config)
{
- if let Err(e) = query_kernel.encode_qk_norm(
+ let query_head_count = self.num_q_heads as u32;
+ query_kernel.encode(
+ qkv_buffer,
+ query_scales_buffer,
+ qkv_buffer,
+ batch_size as u32,
+ self.num_q_heads as u32,
+ self.num_kv_heads as u32,
+ head_dim as u32,
+ query_config.epsilon,
+ query_config.scale_offset.unwrap_or(0.0),
+ 0,
+ query_head_count,
+ query_config.upcast_mode == UpcastMode::FullLayer,
compute_encoder,
- QKNormArguments {
- qkv_input_buffer: &qkv_buffer,
- scales_buffer: query_scales_buffer,
- qkv_output_buffer: &qkv_buffer,
- batch_size,
- // Always pass actual head counts (needed for correct buffer addressing)
- num_q_heads: self.num_q_heads as i32,
- num_kv_heads: self.num_kv_heads as i32,
- head_dim,
- epsilon: query_config.epsilon,
- scale_offset: query_config.scale_offset.unwrap_or(0.0),
- target: QKNormTarget::QueryHeads,
- },
- ) {
- eprintln!("Failed to encode query normalization kernel: {:?}", e);
- }
+ );
}
// Process key normalization if configured
if let (Some(key_kernel), Some(key_scales_buffer), Some(key_config)) =
(&self.key_kernel, &self.key_scales_buffer, &self.key_config)
{
- if let Err(e) = key_kernel.encode_qk_norm(
+ let key_head_count = self.num_kv_heads as u32;
+ key_kernel.encode(
+ qkv_buffer,
+ key_scales_buffer,
+ qkv_buffer,
+ batch_size as u32,
+ self.num_q_heads as u32,
+ self.num_kv_heads as u32,
+ head_dim as u32,
+ key_config.epsilon,
+ key_config.scale_offset.unwrap_or(0.0),
+ self.num_q_heads as u32,
+ key_head_count,
+ key_config.upcast_mode == UpcastMode::FullLayer,
compute_encoder,
- QKNormArguments {
- qkv_input_buffer: &qkv_buffer,
- scales_buffer: key_scales_buffer,
- qkv_output_buffer: &qkv_buffer,
- batch_size,
- // Always pass actual head counts (needed for correct buffer addressing)
- num_q_heads: self.num_q_heads as i32,
- num_kv_heads: self.num_kv_heads as i32,
- head_dim,
- epsilon: key_config.epsilon,
- scale_offset: key_config.scale_offset.unwrap_or(0.0),
- target: QKNormTarget::KeyHeads,
- },
- ) {
- eprintln!("Failed to encode key normalization kernel: {:?}", e);
- }
+ );
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
index fb54be5be..558b7f041 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
@@ -1,12 +1,13 @@
//! RMS Normalization encodable.
-
use super::super::{EncodableBlock, Metal};
use crate::{
DataType,
- backends::metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt, MTLError,
- MTLResourceOptions, ProtocolObject, Retained,
- kernel::rms_norm::{RMSNormArguments, RMSNormError, RMSNormKernel, RMSNormKernelType},
+ backends::{
+ common::kernel::RMSNormKernel,
+ metal::{
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt,
+ MTLError, MTLResourceOptions, ProtocolObject, Retained, kernel::dsl::RMSNormMetalKernel,
+ },
},
config::{NormalizationConfig, UpcastMode},
encodable_block::EncodingParameters,
@@ -15,7 +16,7 @@ use crate::{
};
pub struct RMSNorm {
- kernel: RMSNormKernel,
+ kernel: RMSNormMetalKernel,
config: NormalizationConfig,
input_array_id: ArrayId,
output_array_id: ArrayId,
@@ -31,13 +32,13 @@ impl RMSNorm {
input_array_id: ArrayId,
output_array_id: ArrayId,
parameter_tree: &ParameterTree<MTLContext>,
- ) -> Result<Self, RMSNormError> {
+ ) -> Result<Self, MTLError> {
// Load scales from parameter tree
let scales_param = parameter_tree.leaf("scales").map_err(|e| {
- RMSNormError::MetalError(MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(
+ MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(
"Failed to load scales: {:?}",
e
- ))))
+ )))
})?;
// TODO: Don't create buffers dynamically, we need to use forward pass storage for thing like this
@@ -61,16 +62,7 @@ impl RMSNorm {
},
};
- let kernel = RMSNormKernel::new_with_mode(
- context,
- input_type,
- scales_type,
- output_type,
- accumulation_data_type,
- RMSNormKernelType::Standard,
- config.upcast_mode == UpcastMode::FullLayer,
- )?;
-
+ let kernel = RMSNormMetalKernel::new(context, input_type, scales_type, output_type, accumulation_data_type)?;
Ok(Self {
kernel,
config,
@@ -145,29 +137,24 @@ impl EncodableBlock<Metal> for RMSNorm {
}
let row_size_in_bytes = input_shape[1] * input_elem_size;
- let input_offset = (batch_start * row_size_in_bytes) as u64;
+ let input_offset = batch_start * row_size_in_bytes;
let output_row_size_in_bytes = input_shape[1] * output_elem_size;
- let output_offset = (batch_start * output_row_size_in_bytes) as u64;
+ let output_offset = batch_start * output_row_size_in_bytes;
let batch_size = batch_len as i32;
let model_dim = input_shape[1] as i32;
- if let Err(e) = self.kernel.encode(
+ self.kernel.encode(
+ (input_buffer, input_offset),
+ &self.scales_buffer,
+ (output_buffer, output_offset),
+ batch_size as u32,
+ model_dim as u32,
+ self.config.epsilon,
+ self.config.scale_offset.unwrap_or(0.0),
+ self.config.upcast_mode == UpcastMode::FullLayer,
compute_encoder,
- RMSNormArguments {
- input_buffer: &input_buffer,
- input_offset,
- scales_buffer: &self.scales_buffer,
- output_buffer: &output_buffer,
- output_offset,
- batch_size,
- model_dim,
- epsilon: self.config.epsilon,
- scale_offset: self.config.scale_offset.unwrap_or(0.0),
- },
- ) {
- eprintln!("Failed to encode RMS norm kernel: {:?}", e);
- }
+ )
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index c9bd68bed..9cb03bdbd 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -15,7 +15,6 @@ pub use mlp::{
pub mod mlp_fused;
pub mod moe;
pub mod quant_matmul;
-pub mod rms_norm;
pub mod rope;
pub mod short_conv;
pub mod ssm;
@@ -33,7 +32,6 @@ pub use moe::{
MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError, MoeScatterKernels,
MoeScatterWithMapArguments,
};
-pub use rms_norm::{QKNormArguments, QKNormTarget, RMSNormArguments, RMSNormError, RMSNormKernel, RMSNormKernelType};
pub use rope::{RopeError, RopeKernel, RopeKernelArguments};
pub use short_conv::{ShortConvDecodeArguments, ShortConvKernel, ShortConvKernelError, ShortConvPrefillArguments};
pub use ssm::{
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
deleted file mode 100644
index facef02b7..000000000
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
+++ /dev/null
@@ -1,280 +0,0 @@
-use crate::{
- DataType,
- backends::metal::{
- ComputeEncoderSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,
- MTLSize, ProtocolObject, Retained,
- },
-};
-
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum RMSNormKernelType {
- Standard,
- QueryKey,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum QKNormTarget {
- QueryHeads,
- KeyHeads,
- Both,
-}
-
-pub struct RMSNormKernel {
- pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- kernel_type: RMSNormKernelType,
-}
-
-#[derive(Debug)]
-pub struct RMSNormArguments<'a> {
- pub input_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub input_offset: u64,
- pub scales_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub output_offset: u64,
- pub batch_size: i32,
- pub model_dim: i32,
- pub epsilon: f32,
- pub scale_offset: f32,
-}
-
-#[derive(Debug)]
-pub struct QKNormArguments<'a> {
- pub qkv_input_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub scales_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub qkv_output_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub batch_size: i32,
- pub num_q_heads: i32,
- pub num_kv_heads: i32,
- pub head_dim: i32,
- pub epsilon: f32,
- pub scale_offset: f32,
- pub target: QKNormTarget,
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum RMSNormError {
- #[error("Metal error: {0}")]
- MetalError(#[from] MTLError),
- #[error(
- "Unsupported data type combination: input={input:?}, scales={scales:?}, output={output:?}, accumulation={accumulation:?}"
- )]
- UnsupportedDataType {
- input: DataType,
- scales: DataType,
- output: DataType,
- accumulation: DataType,
- },
- #[error("Invalid kernel type for operation")]
- InvalidKernelType,
-}
-
-impl RMSNormKernel {
- pub fn new(
- context: &MTLContext,
- input_data_type: DataType,
- scales_data_type: DataType,
- output_data_type: DataType,
- accumulation_data_type: DataType,
- kernel_type: RMSNormKernelType,
- ) -> Result<Self, RMSNormError> {
- Self::new_with_mode(
- context,
- input_data_type,
- scales_data_type,
- output_data_type,
- accumulation_data_type,
- kernel_type,
- false,
- )
- }
-
- pub fn new_with_mode(
- context: &MTLContext,
- input_data_type: DataType,
- scales_data_type: DataType,
- output_data_type: DataType,
- accumulation_data_type: DataType,
- kernel_type: RMSNormKernelType,
- full_layer: bool,
- ) -> Result<Self, RMSNormError> {
- let function_name = Self::kernel_name_for_types(
- input_data_type,
- scales_data_type,
- output_data_type,
- accumulation_data_type,
- kernel_type,
- full_layer,
- )?;
-
- let pipeline = context.compute_pipeline_state(&function_name, None).map_err(RMSNormError::MetalError)?;
-
- Ok(Self {
- pipeline,
- kernel_type,
- })
- }
-
- /// Generate kernel name from data type combination
- fn kernel_name_for_types(
- input_dt: DataType,
- scales_dt: DataType,
- output_dt: DataType,
- accum_dt: DataType,
- kernel_type: RMSNormKernelType,
- full_layer: bool,
- ) -> Result<String, RMSNormError> {
- let input_suffix = Self::data_type_to_suffix(input_dt)?;
- let scales_suffix = Self::data_type_to_suffix(scales_dt)?;
- let output_suffix = Self::data_type_to_suffix(output_dt)?;
- let accum_suffix = Self::accum_type_to_suffix(accum_dt)?;
-
- let base_name = match kernel_type {
- RMSNormKernelType::Standard => "rms_norm",
- RMSNormKernelType::QueryKey => "qk_norm",
- };
-
- let mode_suffix = if full_layer {
- "_full"
- } else {
- "_norm"
- };
-
- Ok(format!(
- "{}_{}_{}_{}_{}{}",
- base_name, input_suffix, scales_suffix, output_suffix, accum_suffix, mode_suffix
- ))
- }
-
- fn data_type_to_suffix(data_type: DataType) -> Result<&'static str, RMSNormError> {
- match data_type {
- DataType::F32 => Ok("f32"),
- DataType::F16 => Ok("f16"),
- DataType::BF16 => Ok("bf16"),
- _ => Err(RMSNormError::UnsupportedDataType {
- input: data_type,
- scales: data_type,
- output: data_type,
- accumulation: data_type,
- }),
- }
- }
-
- fn accum_type_to_suffix(data_type: DataType) -> Result<&'static str, RMSNormError> {
- match data_type {
- DataType::F32 => Ok("f32"),
- DataType::F16 => Ok("f16"),
- _ => Err(RMSNormError::UnsupportedDataType {
- input: data_type,
- scales: data_type,
- output: data_type,
- accumulation: data_type,
- }),
- }
- }
-
- pub fn encode(
- &self,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: RMSNormArguments,
- ) -> Result<(), RMSNormError> {
- if self.kernel_type != RMSNormKernelType::Standard {
- return Err(RMSNormError::InvalidKernelType);
- }
-
- compute_encoder.set_compute_pipeline_state(&self.pipeline);
-
- // Set buffers
- compute_encoder.set_buffer(Some(args.input_buffer), args.input_offset as usize, 0);
- compute_encoder.set_buffer(Some(args.scales_buffer), 0, 1);
- compute_encoder.set_buffer(Some(args.output_buffer), args.output_offset as usize, 2);
-
- // Set parameters
- compute_encoder.set_value(&args.batch_size, 3);
- compute_encoder.set_value(&args.model_dim, 4);
- compute_encoder.set_value(&args.epsilon, 5);
- compute_encoder.set_value(&args.scale_offset, 6);
-
- let threads_per_threadgroup = MTLSize {
- width: 1024,
- height: 1,
- depth: 1,
- };
-
- let threadgroups_per_grid = MTLSize {
- width: args.batch_size as usize,
- height: 1,
- depth: 1,
- };
-
- compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);
-
- Ok(())
- }
-
- pub fn encode_qk_norm(
- &self,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: QKNormArguments,
- ) -> Result<(), RMSNormError> {
- if self.kernel_type != RMSNormKernelType::QueryKey {
- return Err(RMSNormError::InvalidKernelType);
- }
-
- compute_encoder.set_compute_pipeline_state(&self.pipeline);
-
- // Set buffers
- compute_encoder.set_buffer(Some(args.qkv_input_buffer), 0, 0);
- compute_encoder.set_buffer(Some(args.scales_buffer), 0, 1);
- compute_encoder.set_buffer(Some(args.qkv_output_buffer), 0, 2);
-
- // Set parameters
- compute_encoder.set_value(&args.batch_size, 3);
- compute_encoder.set_value(&args.num_q_heads, 4);
- compute_encoder.set_value(&args.num_kv_heads, 5);
- compute_encoder.set_value(&args.head_dim, 6);
- compute_encoder.set_value(&args.epsilon, 7);
- compute_encoder.set_value(&args.scale_offset, 8);
-
- // Determine which contiguous head range to normalize.
- //
- // QKV layout per token: [Q heads][K heads][V heads]
- // We only ever normalize within [Q] and/or [K].
- let (head_offset, head_count): (u32, u32) = match args.target {
- QKNormTarget::QueryHeads => (0, args.num_q_heads as u32),
- QKNormTarget::KeyHeads => (args.num_q_heads as u32, args.num_kv_heads as u32),
- QKNormTarget::Both => (0, (args.num_q_heads + args.num_kv_heads) as u32),
- };
-
- if args.batch_size <= 0 || args.head_dim <= 0 || head_count == 0 {
- return Ok(());
- }
-
- // Pass head range to the kernel.
- compute_encoder.set_value(&head_offset, 9);
- compute_encoder.set_value(&head_count, 10);
-
- // One SIMD-group per head, multiple heads per threadgroup.
- let simd_width = self.pipeline.thread_execution_width();
- let max_threads = self.pipeline.max_total_threads_per_threadgroup();
- let max_heads_per_threadgroup = (max_threads / simd_width).max(1);
- let heads_per_threadgroup = (head_count as usize).min(max_heads_per_threadgroup).max(1);
-
- let num_head_tiles = (head_count as usize).div_ceil(heads_per_threadgroup);
-
- let threadgroups_per_grid = MTLSize {
- width: args.batch_size as usize,
- height: num_head_tiles,
- depth: 1,
- };
-
- let threads_per_threadgroup = MTLSize {
- width: heads_per_threadgroup * simd_width,
- height: 1,
- depth: 1,
- };
-
- compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);
-
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal
new file mode 100644
index 000000000..e87ca4a89
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal
@@ -0,0 +1,108 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+
+using namespace metal;
+
+#define SIMD_SIZE 32
+#define GRAIN_SIZE 4
+
+// QK norm: normalize per-head vectors (small head_dim) efficiently.
+//
+// Strategy:
+// - One SIMD-group (32 threads) processes one head.
+// - One threadgroup (one SIMD-group) is dispatched per head.
+template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>
+VARIANTS(InputT, float, half, bfloat)
+VARIANTS(ScaleT, float, half, bfloat)
+VARIANTS(OutputT, float, half, bfloat)
+VARIANTS(AccumT, float, half)
+KERNEL(QKNorm)(
+ const device InputT* qkv_input,
+ const device ScaleT* scales,
+ device OutputT* qkv_output,
+ constant uint& batch_size,
+ constant uint& num_q_heads,
+ constant uint& num_kv_heads,
+ constant uint& head_dim,
+ constant float& epsilon,
+ constant float& scale_offset,
+ constant uint& head_offset,
+ constant uint& head_count,
+ constant bool& full_layer,
+ const uint batch_idx GROUPS(batch_size),
+ const uint head_idx GROUPS(head_count),
+ const uint lane_id THREADS(SIMD_SIZE)
+) {
+ if (head_count == 0u || head_dim == 0u)
+ return;
+
+ const uint total_heads_in_buffer = num_q_heads + 2u * num_kv_heads;
+ const uint logical_head_idx = head_offset + head_idx;
+ if (logical_head_idx >= total_heads_in_buffer)
+ return;
+
+ const ulong slice_offset =
+ (ulong)batch_idx * (ulong)total_heads_in_buffer * (ulong)head_dim +
+ (ulong)logical_head_idx * (ulong)head_dim;
+
+ const device InputT* input_data = qkv_input + slice_offset;
+ const device ScaleT* scales_data = scales;
+ device OutputT* output_data = qkv_output + slice_offset;
+ const uint element_count = head_dim;
+
+ AccumT partial_sum = static_cast<AccumT>(0.0f);
+
+ // Sum of squares: each lane processes GRAIN_SIZE elements per iteration.
+ for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;
+ base_i += SIMD_SIZE * GRAIN_SIZE) {
+ AccumT vals[GRAIN_SIZE];
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
+ }
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ partial_sum += vals[j] * vals[j];
+ }
+ }
+
+ // SIMD-group reduction.
+ AccumT total_sum = simd_sum(partial_sum);
+
+ // RMS factor.
+ AccumT mean_square =
+ static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);
+ AccumT rms_norm = rsqrt(mean_square + static_cast<AccumT>(epsilon));
+
+ // Normalize + scale.
+ for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;
+ base_i += SIMD_SIZE * GRAIN_SIZE) {
+ AccumT vals[GRAIN_SIZE];
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
+ }
+
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ if (i >= element_count)
+ continue;
+
+ AccumT normalized_high = vals[j] * rms_norm;
+
+ if (full_layer) {
+ AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) +
+ static_cast<AccumT>(scale_offset);
+ output_data[i] =
+ static_cast<OutputT>(normalized_high * scale_value_high);
+ } else {
+ OutputT normalized_low = static_cast<OutputT>(normalized_high);
+ OutputT scale_value_low = static_cast<OutputT>(
+ static_cast<AccumT>(scales_data[i]) +
+ static_cast<AccumT>(scale_offset)
+ );
+ OutputT product_low = normalized_low * scale_value_low;
+ output_data[i] = static_cast<OutputT>(product_low);
+ }
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index 5cfda4c7e..a744e065f 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -7,19 +7,29 @@ using namespace metal;
#define SIMD_SIZE 32
#define GRAIN_SIZE 4
-// Main template kernel - single implementation for all types
template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>
-void rms_norm_core(
- const device InputT* input_data,
- const device ScaleT* scales_data,
- device OutputT* output_data,
- uint element_count,
+VARIANTS(InputT, float, half, bfloat)
+VARIANTS(ScaleT, float, half, bfloat)
+VARIANTS(OutputT, float, half, bfloat)
+VARIANTS(AccumT, float, half)
+KERNEL(RMSNorm)(
+ const device InputT* input,
+ const device ScaleT* scales,
+ device OutputT* output,
+ constant uint& batch_size,
+ constant uint& element_count,
constant float& epsilon,
constant float& scale_offset,
- threadgroup AccumT* shared_sum,
- uint thread_in_row,
- bool full_layer
+ constant bool& full_layer,
+ threadgroup AccumT shared_sum[SIMD_SIZE],
+ const uint batch_idx GROUPS(batch_size),
+ const uint thread_in_row THREADS(1024)
) {
+ const uint input_offset = batch_idx * element_count;
+ const device InputT* input_data = input + input_offset;
+ const device ScaleT* scales_data = scales;
+ device OutputT* output_data = output + input_offset;
+
AccumT partial_sum = static_cast<AccumT>(0.0f);
// Compute thread local partial sum
@@ -96,302 +106,3 @@ void rms_norm_core(
}
}
}
-
-// X-macro table defining all supported type combinations
-// Format: (InputType, ScaleType, OutputType, AccumType, full_layer_mode,
-// suffix)
-#define FOREACH_RMS_COMBO(_) \
- /* float combinations */ \
- _(float, float, float, float, false, f32_f32_f32_f32_norm) \
- _(float, float, float, float, true, f32_f32_f32_f32_full) \
- _(float, float, half, float, false, f32_f32_f16_f32_norm) \
- _(float, float, half, float, true, f32_f32_f16_f32_full) \
- _(float, float, bfloat, float, false, f32_f32_bf16_f32_norm) \
- _(float, float, bfloat, float, true, f32_f32_bf16_f32_full) \
- _(float, half, float, float, false, f32_f16_f32_f32_norm) \
- _(float, half, float, float, true, f32_f16_f32_f32_full) \
- _(float, half, half, float, false, f32_f16_f16_f32_norm) \
- _(float, half, half, float, true, f32_f16_f16_f32_full) \
- _(float, half, bfloat, float, false, f32_f16_bf16_f32_norm) \
- _(float, half, bfloat, float, true, f32_f16_bf16_f32_full) \
- _(float, bfloat, float, float, false, f32_bf16_f32_f32_norm) \
- _(float, bfloat, float, float, true, f32_bf16_f32_f32_full) \
- _(float, bfloat, half, float, false, f32_bf16_f16_f32_norm) \
- _(float, bfloat, half, float, true, f32_bf16_f16_f32_full) \
- _(float, bfloat, bfloat, float, false, f32_bf16_bf16_f32_norm) \
- _(float, bfloat, bfloat, float, true, f32_bf16_bf16_f32_full) \
- \
- /* half combinations */ \
- _(half, float, float, float, false, f16_f32_f32_f32_norm) \
- _(half, float, float, float, true, f16_f32_f32_f32_full) \
- _(half, float, half, float, false, f16_f32_f16_f32_norm) \
- _(half, float, half, float, true, f16_f32_f16_f32_full) \
- _(half, float, bfloat, float, false, f16_f32_bf16_f32_norm) \
- _(half, float, bfloat, float, true, f16_f32_bf16_f32_full) \
- _(half, half, float, float, false, f16_f16_f32_f32_norm) \
- _(half, half, float, float, true, f16_f16_f32_f32_full) \
- _(half, half, half, float, false, f16_f16_f16_f32_norm) \
- _(half, half, half, float, true, f16_f16_f16_f32_full) \
- _(half, half, bfloat, float, false, f16_f16_bf16_f32_norm) \
- _(half, half, bfloat, float, true, f16_f16_bf16_f32_full) \
- _(half, bfloat, float, float, false, f16_bf16_f32_f32_norm) \
- _(half, bfloat, float, float, true, f16_bf16_f32_f32_full) \
- _(half, bfloat, half, float, false, f16_bf16_f16_f32_norm) \
- _(half, bfloat, half, float, true, f16_bf16_f16_f32_full) \
- _(half, bfloat, bfloat, float, false, f16_bf16_bf16_f32_norm) \
- _(half, bfloat, bfloat, float, true, f16_bf16_bf16_f32_full) \
- \
- /* bfloat combinations */ \
- _(bfloat, float, float, float, false, bf16_f32_f32_f32_norm) \
- _(bfloat, float, float, float, true, bf16_f32_f32_f32_full) \
- _(bfloat, float, half, float, false, bf16_f32_f16_f32_norm) \
- _(bfloat, float, half, float, true, bf16_f32_f16_f32_full) \
- _(bfloat, float, bfloat, float, false, bf16_f32_bf16_f32_norm) \
- _(bfloat, float, bfloat, float, true, bf16_f32_bf16_f32_full) \
- _(bfloat, half, float, float, false, bf16_f16_f32_f32_norm) \
- _(bfloat, half, float, float, true, bf16_f16_f32_f32_full) \
- _(bfloat, half, half, float, false, bf16_f16_f16_f32_norm) \
- _(bfloat, half, half, float, true, bf16_f16_f16_f32_full) \
- _(bfloat, half, bfloat, float, false, bf16_f16_bf16_f32_norm) \
- _(bfloat, half, bfloat, float, true, bf16_f16_bf16_f32_full) \
- _(bfloat, bfloat, float, float, false, bf16_bf16_f32_f32_norm) \
- _(bfloat, bfloat, float, float, true, bf16_bf16_f32_f32_full) \
- _(bfloat, bfloat, half, float, false, bf16_bf16_f16_f32_norm) \
- _(bfloat, bfloat, half, float, true, bf16_bf16_f16_f32_full) \
- _(bfloat, bfloat, bfloat, float, false, bf16_bf16_bf16_f32_norm) \
- _(bfloat, bfloat, bfloat, float, true, bf16_bf16_bf16_f32_full) \
- \
- /* All combinations with half accumulation */ \
- _(float, float, float, half, false, f32_f32_f32_f16_norm) \
- _(float, float, float, half, true, f32_f32_f32_f16_full) \
- _(float, float, half, half, false, f32_f32_f16_f16_norm) \
- _(float, float, half, half, true, f32_f32_f16_f16_full) \
- _(float, float, bfloat, half, false, f32_f32_bf16_f16_norm) \
- _(float, float, bfloat, half, true, f32_f32_bf16_f16_full) \
- _(float, half, float, half, false, f32_f16_f32_f16_norm) \
- _(float, half, float, half, true, f32_f16_f32_f16_full) \
- _(float, half, half, half, false, f32_f16_f16_f16_norm) \
- _(float, half, half, half, true, f32_f16_f16_f16_full) \
- _(float, half, bfloat, half, false, f32_f16_bf16_f16_norm) \
- _(float, half, bfloat, half, true, f32_f16_bf16_f16_full) \
- _(float, bfloat, float, half, false, f32_bf16_f32_f16_norm) \
- _(float, bfloat, float, half, true, f32_bf16_f32_f16_full) \
- _(float, bfloat, half, half, false, f32_bf16_f16_f16_norm) \
- _(float, bfloat, half, half, true, f32_bf16_f16_f16_full) \
- _(float, bfloat, bfloat, half, false, f32_bf16_bf16_f16_norm) \
- _(float, bfloat, bfloat, half, true, f32_bf16_bf16_f16_full) \
- \
- _(half, float, float, half, false, f16_f32_f32_f16_norm) \
- _(half, float, float, half, true, f16_f32_f32_f16_full) \
- _(half, float, half, half, false, f16_f32_f16_f16_norm) \
- _(half, float, half, half, true, f16_f32_f16_f16_full) \
- _(half, float, bfloat, half, false, f16_f32_bf16_f16_norm) \
- _(half, float, bfloat, half, true, f16_f32_bf16_f16_full) \
- _(half, half, float, half, false, f16_f16_f32_f16_norm) \
- _(half, half, float, half, true, f16_f16_f32_f16_full) \
- _(half, half, half, half, false, f16_f16_f16_f16_norm) \
- _(half, half, half, half, true, f16_f16_f16_f16_full) \
- _(half, half, bfloat, half, false, f16_f16_bf16_f16_norm) \
- _(half, half, bfloat, half, true, f16_f16_bf16_f16_full) \
- _(half, bfloat, float, half, false, f16_bf16_f32_f16_norm) \
- _(half, bfloat, float, half, true, f16_bf16_f32_f16_full) \
- _(half, bfloat, half, half, false, f16_bf16_f16_f16_norm) \
- _(half, bfloat, half, half, true, f16_bf16_f16_f16_full) \
- _(half, bfloat, bfloat, half, false, f16_bf16_bf16_f16_norm) \
- _(half, bfloat, bfloat, half, true, f16_bf16_bf16_f16_full) \
- \
- _(bfloat, float, float, half, false, bf16_f32_f32_f16_norm) \
- _(bfloat, float, float, half, true, bf16_f32_f32_f16_full) \
- _(bfloat, float, half, half, false, bf16_f32_f16_f16_norm) \
- _(bfloat, float, half, half, true, bf16_f32_f16_f16_full) \
- _(bfloat, float, bfloat, half, false, bf16_f32_bf16_f16_norm) \
- _(bfloat, float, bfloat, half, true, bf16_f32_bf16_f16_full) \
- _(bfloat, half, float, half, false, bf16_f16_f32_f16_norm) \
- _(bfloat, half, float, half, true, bf16_f16_f32_f16_full) \
- _(bfloat, half, half, half, false, bf16_f16_f16_f16_norm) \
- _(bfloat, half, half, half, true, bf16_f16_f16_f16_full) \
- _(bfloat, half, bfloat, half, false, bf16_f16_bf16_f16_norm) \
- _(bfloat, half, bfloat, half, true, bf16_f16_bf16_f16_full) \
- _(bfloat, bfloat, float, half, false, bf16_bf16_f32_f16_norm) \
- _(bfloat, bfloat, float, half, true, bf16_bf16_f32_f16_full) \
- _(bfloat, bfloat, half, half, false, bf16_bf16_f16_f16_norm) \
- _(bfloat, bfloat, half, half, true, bf16_bf16_f16_f16_full) \
- _(bfloat, bfloat, bfloat, half, false, bf16_bf16_bf16_f16_norm) \
- _(bfloat, bfloat, bfloat, half, true, bf16_bf16_bf16_f16_full)
-
-// Generate RMS norm kernels
-#define DEFINE_RMS_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \
- [[max_total_threads_per_threadgroup(1024)]] kernel void rms_norm_##SUF( \
- const device IN* input [[buffer(0)]], \
- const device SC* scales [[buffer(1)]], \
- device OUT* output [[buffer(2)]], \
- constant uint& batch_size [[buffer(3)]], \
- constant uint& model_dim [[buffer(4)]], \
- constant float& epsilon [[buffer(5)]], \
- constant float& scale_offset [[buffer(6)]], \
- uint batch_idx [[threadgroup_position_in_grid]], \
- uint thread_in_row [[thread_position_in_threadgroup]] \
- ) { \
- if (batch_idx >= batch_size) \
- return; \
- \
- threadgroup ACC shared_sum[SIMD_SIZE]; \
- const uint input_offset = batch_idx * model_dim; \
- \
- rms_norm_core<IN, SC, OUT, ACC>( \
- input + input_offset, \
- scales, \
- output + input_offset, \
- model_dim, \
- epsilon, \
- scale_offset, \
- shared_sum, \
- thread_in_row, \
- FULL_LAYER \
- ); \
- }
-
-FOREACH_RMS_COMBO(DEFINE_RMS_KERNEL)
-#undef DEFINE_RMS_KERNEL
-
-// QK norm: normalize per-head vectors (small head_dim) efficiently.
-//
-// Strategy:
-// - One SIMD-group (32 threads) processes one head.
-// - A threadgroup processes multiple heads for a given token (batch).
-// - No threadgroup-wide reductions/barriers: we only use simd_sum within a
-// SIMD-group.
-template <
- typename InputT,
- typename ScaleT,
- typename OutputT,
- typename AccumT,
- bool FULL_LAYER>
-static inline void qk_norm_head_core(
- const device InputT* input_data,
- const device ScaleT* scales_data,
- device OutputT* output_data,
- uint element_count,
- constant float& epsilon,
- constant float& scale_offset,
- uint lane_id
-) {
- if (element_count == 0u)
- return;
-
- AccumT partial_sum = static_cast<AccumT>(0.0f);
-
- // Sum of squares: each lane processes GRAIN_SIZE elements per iteration.
- for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;
- base_i += SIMD_SIZE * GRAIN_SIZE) {
- AccumT vals[GRAIN_SIZE];
- for (uint j = 0; j < GRAIN_SIZE; ++j) {
- uint i = base_i + j;
- vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
- }
- for (uint j = 0; j < GRAIN_SIZE; ++j) {
- partial_sum += vals[j] * vals[j];
- }
- }
-
- // SIMD-group reduction.
- AccumT total_sum = simd_sum(partial_sum);
-
- // RMS factor.
- AccumT mean_square =
- static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);
- AccumT rms_norm = rsqrt(mean_square + static_cast<AccumT>(epsilon));
-
- // Normalize + scale.
- for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;
- base_i += SIMD_SIZE * GRAIN_SIZE) {
- AccumT vals[GRAIN_SIZE];
- for (uint j = 0; j < GRAIN_SIZE; ++j) {
- uint i = base_i + j;
- vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
- }
-
- for (uint j = 0; j < GRAIN_SIZE; ++j) {
- uint i = base_i + j;
- if (i >= element_count)
- continue;
-
- AccumT normalized_high = vals[j] * rms_norm;
-
- if (FULL_LAYER) {
- AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) +
- static_cast<AccumT>(scale_offset);
- output_data[i] =
- static_cast<OutputT>(normalized_high * scale_value_high);
- } else {
- OutputT normalized_low = static_cast<OutputT>(normalized_high);
- OutputT scale_value_low = static_cast<OutputT>(
- static_cast<AccumT>(scales_data[i]) +
- static_cast<AccumT>(scale_offset)
- );
- OutputT product_low = normalized_low * scale_value_low;
- output_data[i] = static_cast<OutputT>(product_low);
- }
- }
- }
-}
-
-// Generate QK norm kernels (batched, head-tiled).
-#define DEFINE_QK_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \
- [[max_total_threads_per_threadgroup(1024)]] kernel void qk_norm_##SUF( \
- const device IN* qkv_input [[buffer(0)]], \
- const device SC* scales [[buffer(1)]], \
- device OUT* qkv_output [[buffer(2)]], \
- constant uint& batch_size [[buffer(3)]], \
- constant uint& num_q_heads [[buffer(4)]], \
- constant uint& num_kv_heads [[buffer(5)]], \
- constant uint& head_dim [[buffer(6)]], \
- constant float& epsilon [[buffer(7)]], \
- constant float& scale_offset [[buffer(8)]], \
- constant uint& head_offset [[buffer(9)]], \
- constant uint& head_count [[buffer(10)]], \
- uint3 tgid [[threadgroup_position_in_grid]], \
- uint lid [[thread_index_in_threadgroup]], \
- uint3 threads_per_threadgroup [[threads_per_threadgroup]] \
- ) { \
- const uint batch_idx = tgid.x; \
- const uint tile_idx = tgid.y; \
- \
- if (batch_idx >= batch_size) \
- return; \
- if (head_count == 0u || head_dim == 0u) \
- return; \
- \
- const uint heads_per_tg = threads_per_threadgroup.x / SIMD_SIZE; \
- if (heads_per_tg == 0u) \
- return; \
- \
- const uint simd_group_id = lid / SIMD_SIZE; \
- const uint lane_id = lid % SIMD_SIZE; \
- const uint head_idx = tile_idx * heads_per_tg + simd_group_id; \
- \
- if (head_idx >= head_count) \
- return; \
- \
- const uint total_heads_in_buffer = num_q_heads + 2u * num_kv_heads; \
- const uint logical_head_idx = head_offset + head_idx; \
- if (logical_head_idx >= total_heads_in_buffer) \
- return; \
- \
- const ulong slice_offset = \
- (ulong)batch_idx * (ulong)total_heads_in_buffer * (ulong)head_dim + \
- (ulong)logical_head_idx * (ulong)head_dim; \
- \
- qk_norm_head_core<IN, SC, OUT, ACC, FULL_LAYER>( \
- qkv_input + slice_offset, \
- scales, \
- qkv_output + slice_offset, \
- head_dim, \
- epsilon, \
- scale_offset, \
- lane_id \
- ); \
- }
-
-FOREACH_RMS_COMBO(DEFINE_QK_KERNEL)
-#undef DEFINE_QK_KERNEL
diff --git a/crates/uzu/tests/rms_norm_kernel_test.rs b/crates/uzu/tests/rms_norm_kernel_test.rs
index 40c9e8dd3..f8bdd4d08 100644
--- a/crates/uzu/tests/rms_norm_kernel_test.rs
+++ b/crates/uzu/tests/rms_norm_kernel_test.rs
@@ -7,10 +7,13 @@ use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTL
use uzu::{
DataType,
backends::{
- common::Context,
+ common::{
+ Context,
+ kernel::{QKNormKernel, RMSNormKernel},
+ },
metal::{
MTLContext,
- kernel::rms_norm::{QKNormArguments, QKNormTarget, RMSNormArguments, RMSNormKernel, RMSNormKernelType},
+ kernel::dsl::{QKNormMetalKernel, RMSNormMetalKernel},
metal_extensions::CommandBufferTimingExt,
},
},
@@ -301,34 +304,23 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(
.expect("Failed to create buffer");
// Create RMS norm kernel
- let kernel = RMSNormKernel::new(
- &mtl_context,
- input_type,
- scale_type,
- output_type,
- accumulation_type,
- RMSNormKernelType::Standard,
- )
- .expect("Failed to create RMS norm kernel");
-
+ let kernel = RMSNormMetalKernel::new(&mtl_context, input_type, scale_type, output_type, accumulation_type)
+ .expect("Failed to create RMS norm kernel");
// Create command buffer and encode
let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- let _ = kernel.encode(
+ kernel.encode(
+ &input_buffer,
+ &scale_buffer,
+ &output_buffer,
+ batch_size as u32,
+ model_dim as u32,
+ epsilon,
+ 0.0,
+ false,
&compute_encoder,
- RMSNormArguments {
- input_buffer: &input_buffer,
- input_offset: 0,
- scales_buffer: &scale_buffer,
- output_buffer: &output_buffer,
- output_offset: 0,
- batch_size,
- model_dim,
- epsilon,
- scale_offset: 0.0,
- },
);
compute_encoder.end_encoding();
@@ -423,33 +415,23 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(
.new_buffer(4 * OutputT::size_of(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let kernel = RMSNormKernel::new(
- &mtl_context,
- input_type,
- scale_type,
- output_type,
- accumulation_type,
- RMSNormKernelType::Standard,
- )
- .expect("Failed to create RMS norm kernel");
+ let kernel = RMSNormMetalKernel::new(&mtl_context, input_type, scale_type, output_type, accumulation_type)
+ .expect("Failed to create RMS norm kernel");
let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- let _ = kernel.encode(
+ kernel.encode(
+ &input_buffer,
+ &scale_buffer,
+ &output_buffer,
+ batch_size as u32,
+ model_dim as u32,
+ epsilon,
+ 0.0,
+ false,
&compute_encoder,
- RMSNormArguments {
- input_buffer: &input_buffer,
- input_offset: 0,
- scales_buffer: &scale_buffer,
- output_buffer: &output_buffer,
- output_offset: 0,
- batch_size,
- model_dim,
- epsilon,
- scale_offset: 0.0,
- },
);
compute_encoder.end_encoding();
@@ -602,15 +584,8 @@ fn perf_rms_norm_with_size(
const EPSILON: f32 = 1e-6;
// ---- Create kernel ----
- let kernel = RMSNormKernel::new(
- &mtl_context,
- DataType::F32,
- DataType::F32,
- DataType::F32,
- DataType::F32,
- RMSNormKernelType::Standard,
- )
- .expect("Failed to create RMS norm kernel");
+ let kernel = RMSNormMetalKernel::new(&mtl_context, DataType::F32, DataType::F32, DataType::F32, DataType::F32)
+ .expect("Failed to create RMS norm kernel");
// ---- Generate random data ----
let mut rng = StdRng::seed_from_u64(42);
@@ -648,19 +623,16 @@ fn perf_rms_norm_with_size(
let command_buffer = command_buffer_ref.to_owned();
let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- let _ = kernel.encode(
+ kernel.encode(
+ &input_buffer,
+ &scale_buffer,
+ &output_buffer,
+ batch_size as u32,
+ model_dim as u32,
+ EPSILON,
+ 0.0,
+ false,
&compute_encoder,
- RMSNormArguments {
- input_buffer: &input_buffer,
- input_offset: 0,
- scales_buffer: &scale_buffer,
- output_buffer: &output_buffer,
- output_offset: 0,
- batch_size,
- model_dim,
- epsilon: EPSILON,
- scale_offset: 0.0,
- },
);
compute_encoder.end_encoding();
@@ -795,47 +767,31 @@ fn qk_norm_test() {
.expect("Failed to create buffer");
// Create QK norm kernels
- let q_kernel = RMSNormKernel::new_with_mode(
- &mtl_context,
- DataType::F32,
- DataType::F32,
- DataType::F32,
- DataType::F32,
- RMSNormKernelType::QueryKey,
- false,
- )
- .expect("Failed to create Q norm kernel");
-
- let k_kernel = RMSNormKernel::new_with_mode(
- &mtl_context,
- DataType::F32,
- DataType::F32,
- DataType::F32,
- DataType::F32,
- RMSNormKernelType::QueryKey,
- false,
- )
- .expect("Failed to create K norm kernel");
+ let q_kernel = QKNormMetalKernel::new(&mtl_context, DataType::F32, DataType::F32, DataType::F32, DataType::F32)
+ .expect("Failed to create Q norm kernel");
+
+ let k_kernel = QKNormMetalKernel::new(&mtl_context, DataType::F32, DataType::F32, DataType::F32, DataType::F32)
+ .expect("Failed to create K norm kernel");
// Test Q head normalization
{
let command_buffer = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- let _ = q_kernel.encode_qk_norm(
+ q_kernel.encode(
+ &qkv_buffer,
+ &q_scales_buffer,
+ &qkv_buffer,
+ batch_size as u32,
+ num_q_heads as u32,
+ num_kv_heads as u32,
+ head_dim as u32,
+ epsilon,
+ 0.0,
+ 0,
+ num_q_heads as u32,
+ false,
&compute_encoder,
- QKNormArguments {
- qkv_input_buffer: &qkv_buffer,
- scales_buffer: &q_scales_buffer,
- qkv_output_buffer: &qkv_buffer,
- batch_size,
- num_q_heads, // Now correctly passes actual head count
- num_kv_heads, // Now correctly passes actual head count
- head_dim,
- epsilon,
- scale_offset: 0.0,
- target: QKNormTarget::QueryHeads, // Only process Q heads
- },
);
compute_encoder.end_encoding();
@@ -884,20 +840,20 @@ fn qk_norm_test() {
let command_buffer = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- let _ = k_kernel.encode_qk_norm(
+ k_kernel.encode(
+ &qkv_buffer,
+ &k_scales_buffer,
+ &qkv_buffer,
+ batch_size as u32,
+ num_q_heads as u32,
+ num_kv_heads as u32,
+ head_dim as u32,
+ epsilon,
+ 0.0,
+ num_q_heads as u32,
+ num_kv_heads as u32,
+ false,
&compute_encoder,
- QKNormArguments {
- qkv_input_buffer: &qkv_buffer,
- scales_buffer: &k_scales_buffer,
- qkv_output_buffer: &qkv_buffer,
- batch_size,
- num_q_heads, // Now correctly passes actual head count
- num_kv_heads, // Now correctly passes actual head count
- head_dim,
- epsilon,
- scale_offset: 0.0,
- target: QKNormTarget::KeyHeads, // Only process K heads
- },
);
compute_encoder.end_encoding();