A high-performance inference engine for AI models
generalize qk norm
eugene0d862614d95f
diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rsindex 36b9f469d..63f2843e1 100644--- a/crates/uzu/src/backends/metal/classifier_context.rs+++ b/crates/uzu/src/backends/metal/classifier_context.rs@@ -1,7 +1,8 @@use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};use super::{- MTLCommandBuffer, MTLCommandQueue, MTLContext, ProtocolObject, Retained, compilation_parameters::CompilationConfig,+ MTLCommandBuffer, MTLCommandQueue, MTLContext, ProtocolObject, Retained,+ compilation_parameters::CompilationConfig,encodable_block::{Activation, ClassifierLayer, ClassifierPredictionHead, Normalization, Pooling, Rope,transformer_layer::{embed_block, linear_block},diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rsindex daa851c55..583158542 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs@@ -1,14 +1,12 @@//! Unified normalization encodables: LayerNorm, RMSNorm, QKNorm.-mod qk_norm;--pub use qk_norm::QKNorm;-use super::Metal;use crate::encodable_block::{- LayerNorm as GenericLayerNorm, Normalization as GenericNormalization, RMSNorm as GenericRMSNorm,+ LayerNorm as GenericLayerNorm, Normalization as GenericNormalization, QKNorm as GenericQKNorm,+ RMSNorm as GenericRMSNorm,};pub type LayerNorm = GenericLayerNorm<Metal>;pub type RMSNorm = GenericRMSNorm<Metal>;pub type Normalization = GenericNormalization<Metal>;+pub type QKNorm = GenericQKNorm<Metal>;diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rsindex 4953c1a70..478b40a4f 100644--- a/crates/uzu/src/encodable_block/mod.rs+++ b/crates/uzu/src/encodable_block/mod.rs@@ -7,8 +7,9 @@ mod activation;mod layer_norm;mod normalization;mod pooling;-mod rope;+mod qk_norm;mod rms_norm;+mod rope;mod sampling;mod tensor_add_swap;mod tensor_copy;@@ -17,8 +18,9 @@ pub use activation::Activation;pub use layer_norm::{LayerNorm, LayerNormError};pub use normalization::{Normalization, NormalizationError};pub use pooling::Pooling;-pub use rope::Rope;+pub use qk_norm::{QKNorm, QKNormError};pub use rms_norm::{RMSNorm, RMSNormError};+pub use rope::Rope;pub use sampling::Sampling;pub use tensor_add_swap::TensorAddSwap;pub use tensor_copy::TensorCopy;diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rssimilarity index 54%rename from crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rsrename to crates/uzu/src/encodable_block/qk_norm.rsindex bef9bb85e..18d466a48 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs+++ b/crates/uzu/src/encodable_block/qk_norm.rs@@ -1,46 +1,52 @@//! QK Normalization encodable.-use super::super::{EncodableBlock, Metal};+use thiserror::Error;++use super::{EncodableBlock, EncodingParameters};use crate::{DataType,- backends::{- common::kernel::QKNormKernel,- metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt,- MTLError, MTLResourceOptions, ProtocolObject, Retained, kernel::dsl::QKNormMetalKernel,- },+ backends::common::{+ Backend,+ kernel::{Kernels, QKNormKernel},},config::{NormalizationConfig, UpcastMode},- encodable_block::EncodingParameters,forward_pass::state::{ArrayId, ForwardPassState},- parameters::ParameterTree,+ parameters::{ParameterLoaderError, ParameterTree},};-pub struct QKNorm {- query_kernel: Option<QKNormMetalKernel>,- key_kernel: Option<QKNormMetalKernel>,+#[derive(Debug, Error)]+pub enum QKNormError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Parameter loading error: {0}")]+ ParameterError(ParameterLoaderError),+}++pub struct QKNorm<B: Backend> {+ query_kernel: Option<<B::Kernels as Kernels>::QKNormKernel>,+ key_kernel: Option<<B::Kernels as Kernels>::QKNormKernel>,query_config: Option<NormalizationConfig>,key_config: Option<NormalizationConfig>,qkv_array_id: ArrayId,- query_scales_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,- key_scales_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,+ query_scales_buffer: Option<B::NativeBuffer>,+ key_scales_buffer: Option<B::NativeBuffer>,num_q_heads: usize,num_kv_heads: usize,head_dim: usize,}-impl QKNorm {+impl<B: Backend> QKNorm<B> {pub fn new(- context: &MTLContext,+ context: &B::Context,intermediate_data_type: DataType,query_config: Option<NormalizationConfig>,key_config: Option<NormalizationConfig>,qkv_array_id: ArrayId,- parameter_tree: &ParameterTree<MTLContext>,+ parameter_tree: &ParameterTree<B::Context>,num_q_heads: usize,num_kv_heads: usize,head_dim: usize,- ) -> Result<Self, MTLError> {+ ) -> Result<Self, QKNormError<B>> {let mut query_kernel = None;let mut key_kernel = None;let mut query_scales_buffer = None;@@ -48,26 +54,24 @@ impl QKNorm {// Setup query normalization if configuredif let Some(ref q_config) = query_config {- let scales_param = parameter_tree.leaf("query_norm.scales").map_err(|e| {- MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(- "Failed to load query scales: {:?}",- e- )))- })?;-- let scales_data = scales_param.as_bytes();- let scales_buffer =- context.device.new_buffer_with_data(scales_data, MTLResourceOptions::STORAGE_MODE_SHARED);+ let scales = parameter_tree.leaf("query_norm.scales").map_err(QKNormError::ParameterError)?;+ let scales_buffer = scales.buffer().clone();let accumulation_data_type: DataType = q_config.accumulation_precision.into();let scale_data_type: DataType = q_config.scale_precision.into();-let (input_type, scales_type, output_type) = match q_config.upcast_mode {UpcastMode::OnlyNormalization => (intermediate_data_type, scale_data_type, scale_data_type),UpcastMode::FullLayer => (intermediate_data_type, scale_data_type, scale_data_type),};- let kernel = QKNormMetalKernel::new(context, input_type, scales_type, output_type, accumulation_data_type)?;+ let kernel = <B::Kernels as Kernels>::QKNormKernel::new(+ context,+ input_type,+ scales_type,+ output_type,+ accumulation_data_type,+ )+ .map_err(QKNormError::BackendError)?;query_kernel = Some(kernel);query_scales_buffer = Some(scales_buffer);@@ -75,26 +79,24 @@ impl QKNorm {// Setup key normalization if configuredif let Some(ref k_config) = key_config {- let scales_param = parameter_tree.leaf("key_norm.scales").map_err(|e| {- MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(- "Failed to load key scales: {:?}",- e- )))- })?;-- let scales_data = scales_param.as_bytes();- let scales_buffer =- context.device.new_buffer_with_data(scales_data, MTLResourceOptions::STORAGE_MODE_SHARED);+ let scales = parameter_tree.leaf("key_norm.scales").map_err(QKNormError::ParameterError)?;+ let scales_buffer = scales.buffer().clone();let accumulation_data_type: DataType = k_config.accumulation_precision.into();let scale_data_type: DataType = k_config.scale_precision.into();-let (input_type, scales_type, output_type) = match k_config.upcast_mode {UpcastMode::OnlyNormalization => (intermediate_data_type, scale_data_type, scale_data_type),UpcastMode::FullLayer => (intermediate_data_type, scale_data_type, scale_data_type),};- let kernel = QKNormMetalKernel::new(context, input_type, scales_type, output_type, accumulation_data_type)?;+ let kernel = <B::Kernels as Kernels>::QKNormKernel::new(+ context,+ input_type,+ scales_type,+ output_type,+ accumulation_data_type,+ )+ .map_err(QKNormError::BackendError)?;key_kernel = Some(kernel);key_scales_buffer = Some(scales_buffer);@@ -106,8 +108,8 @@ impl QKNorm {query_config,key_config,qkv_array_id,- query_scales_buffer: query_scales_buffer.flatten(),- key_scales_buffer: key_scales_buffer.flatten(),+ query_scales_buffer,+ key_scales_buffer,num_q_heads,num_kv_heads,head_dim,@@ -115,33 +117,16 @@ impl QKNorm {}}-impl EncodableBlock<Metal> for QKNorm {- fn encode(- &self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- ) {- let compute_encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.encode_with_shared_encoder(state, parameters, &compute_encoder);- compute_encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit();- command_buffer.wait_until_completed();- }- }-+impl<B: Backend> EncodableBlock<B> for QKNorm<B> {fn supports_shared_encoder(&self) -> bool {true}fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState<Metal>,- _parameters: &EncodingParameters<Metal>,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ _parameters: &EncodingParameters<B>,+ encoder: &B::ComputeEncoder,) {let qkv_binding = state.arrays(&[self.qkv_array_id]);let qkv_shape = {@@ -151,29 +136,26 @@ impl EncodableBlock<Metal> for QKNorm {let qkv_array = qkv_binding[0].borrow_mut();let qkv_buffer = qkv_array.buffer();-- let batch_size = qkv_shape[0] as i32;- let head_dim = self.head_dim as i32;+ let batch_size = qkv_shape[0] as u32;// Process query normalization if configuredif let (Some(query_kernel), Some(query_scales_buffer), Some(query_config)) =(&self.query_kernel, &self.query_scales_buffer, &self.query_config){- let query_head_count = self.num_q_heads as u32;query_kernel.encode(qkv_buffer,query_scales_buffer,qkv_buffer,- batch_size as u32,+ batch_size,self.num_q_heads as u32,self.num_kv_heads as u32,- head_dim as u32,+ self.head_dim as u32,query_config.epsilon,query_config.scale_offset.unwrap_or(0.0),0,- query_head_count,+ self.num_q_heads as u32,query_config.upcast_mode == UpcastMode::FullLayer,- compute_encoder,+ encoder,);}@@ -181,21 +163,20 @@ impl EncodableBlock<Metal> for QKNorm {if let (Some(key_kernel), Some(key_scales_buffer), Some(key_config)) =(&self.key_kernel, &self.key_scales_buffer, &self.key_config){- let key_head_count = self.num_kv_heads as u32;key_kernel.encode(qkv_buffer,key_scales_buffer,qkv_buffer,- batch_size as u32,+ batch_size,self.num_q_heads as u32,self.num_kv_heads as u32,- head_dim as u32,+ self.head_dim as u32,key_config.epsilon,key_config.scale_offset.unwrap_or(0.0),self.num_q_heads as u32,- key_head_count,+ self.num_kv_heads as u32,key_config.upcast_mode == UpcastMode::FullLayer,- compute_encoder,+ encoder,);}}