Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

generalize qk norm

eugene0d862614d95f
diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rs
index 36b9f469d..63f2843e1 100644
--- a/crates/uzu/src/backends/metal/classifier_context.rs
+++ b/crates/uzu/src/backends/metal/classifier_context.rs
@@ -1,7 +1,8 @@
use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};
use super::{
- MTLCommandBuffer, MTLCommandQueue, MTLContext, ProtocolObject, Retained, compilation_parameters::CompilationConfig,
+ MTLCommandBuffer, MTLCommandQueue, MTLContext, ProtocolObject, Retained,
+ compilation_parameters::CompilationConfig,
encodable_block::{
Activation, ClassifierLayer, ClassifierPredictionHead, Normalization, Pooling, Rope,
transformer_layer::{embed_block, linear_block},
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
index daa851c55..583158542 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
@@ -1,14 +1,12 @@
//! Unified normalization encodables: LayerNorm, RMSNorm, QKNorm.
-mod qk_norm;
-
-pub use qk_norm::QKNorm;
-
use super::Metal;
use crate::encodable_block::{
- LayerNorm as GenericLayerNorm, Normalization as GenericNormalization, RMSNorm as GenericRMSNorm,
+ LayerNorm as GenericLayerNorm, Normalization as GenericNormalization, QKNorm as GenericQKNorm,
+ RMSNorm as GenericRMSNorm,
};
pub type LayerNorm = GenericLayerNorm<Metal>;
pub type RMSNorm = GenericRMSNorm<Metal>;
pub type Normalization = GenericNormalization<Metal>;
+pub type QKNorm = GenericQKNorm<Metal>;
diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rs
index 4953c1a70..478b40a4f 100644
--- a/crates/uzu/src/encodable_block/mod.rs
+++ b/crates/uzu/src/encodable_block/mod.rs
@@ -7,8 +7,9 @@ mod activation;
mod layer_norm;
mod normalization;
mod pooling;
-mod rope;
+mod qk_norm;
mod rms_norm;
+mod rope;
mod sampling;
mod tensor_add_swap;
mod tensor_copy;
@@ -17,8 +18,9 @@ pub use activation::Activation;
pub use layer_norm::{LayerNorm, LayerNormError};
pub use normalization::{Normalization, NormalizationError};
pub use pooling::Pooling;
-pub use rope::Rope;
+pub use qk_norm::{QKNorm, QKNormError};
pub use rms_norm::{RMSNorm, RMSNormError};
+pub use rope::Rope;
pub use sampling::Sampling;
pub use tensor_add_swap::TensorAddSwap;
pub use tensor_copy::TensorCopy;
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rs
similarity index 54%
rename from crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
rename to crates/uzu/src/encodable_block/qk_norm.rs
index bef9bb85e..18d466a48 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
+++ b/crates/uzu/src/encodable_block/qk_norm.rs
@@ -1,46 +1,52 @@
//! QK Normalization encodable.
-use super::super::{EncodableBlock, Metal};
+use thiserror::Error;
+
+use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
- backends::{
- common::kernel::QKNormKernel,
- metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLDeviceExt,
- MTLError, MTLResourceOptions, ProtocolObject, Retained, kernel::dsl::QKNormMetalKernel,
- },
+ backends::common::{
+ Backend,
+ kernel::{Kernels, QKNormKernel},
},
config::{NormalizationConfig, UpcastMode},
- encodable_block::EncodingParameters,
forward_pass::state::{ArrayId, ForwardPassState},
- parameters::ParameterTree,
+ parameters::{ParameterLoaderError, ParameterTree},
};
-pub struct QKNorm {
- query_kernel: Option<QKNormMetalKernel>,
- key_kernel: Option<QKNormMetalKernel>,
+#[derive(Debug, Error)]
+pub enum QKNormError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Parameter loading error: {0}")]
+ ParameterError(ParameterLoaderError),
+}
+
+pub struct QKNorm<B: Backend> {
+ query_kernel: Option<<B::Kernels as Kernels>::QKNormKernel>,
+ key_kernel: Option<<B::Kernels as Kernels>::QKNormKernel>,
query_config: Option<NormalizationConfig>,
key_config: Option<NormalizationConfig>,
qkv_array_id: ArrayId,
- query_scales_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
- key_scales_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
+ query_scales_buffer: Option<B::NativeBuffer>,
+ key_scales_buffer: Option<B::NativeBuffer>,
num_q_heads: usize,
num_kv_heads: usize,
head_dim: usize,
}
-impl QKNorm {
+impl<B: Backend> QKNorm<B> {
pub fn new(
- context: &MTLContext,
+ context: &B::Context,
intermediate_data_type: DataType,
query_config: Option<NormalizationConfig>,
key_config: Option<NormalizationConfig>,
qkv_array_id: ArrayId,
- parameter_tree: &ParameterTree<MTLContext>,
+ parameter_tree: &ParameterTree<B::Context>,
num_q_heads: usize,
num_kv_heads: usize,
head_dim: usize,
- ) -> Result<Self, MTLError> {
+ ) -> Result<Self, QKNormError<B>> {
let mut query_kernel = None;
let mut key_kernel = None;
let mut query_scales_buffer = None;
@@ -48,26 +54,24 @@ impl QKNorm {
// Setup query normalization if configured
if let Some(ref q_config) = query_config {
- let scales_param = parameter_tree.leaf("query_norm.scales").map_err(|e| {
- MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(
- "Failed to load query scales: {:?}",
- e
- )))
- })?;
-
- let scales_data = scales_param.as_bytes();
- let scales_buffer =
- context.device.new_buffer_with_data(scales_data, MTLResourceOptions::STORAGE_MODE_SHARED);
+ let scales = parameter_tree.leaf("query_norm.scales").map_err(QKNormError::ParameterError)?;
+ let scales_buffer = scales.buffer().clone();
let accumulation_data_type: DataType = q_config.accumulation_precision.into();
let scale_data_type: DataType = q_config.scale_precision.into();
-
let (input_type, scales_type, output_type) = match q_config.upcast_mode {
UpcastMode::OnlyNormalization => (intermediate_data_type, scale_data_type, scale_data_type),
UpcastMode::FullLayer => (intermediate_data_type, scale_data_type, scale_data_type),
};
- let kernel = QKNormMetalKernel::new(context, input_type, scales_type, output_type, accumulation_data_type)?;
+ let kernel = <B::Kernels as Kernels>::QKNormKernel::new(
+ context,
+ input_type,
+ scales_type,
+ output_type,
+ accumulation_data_type,
+ )
+ .map_err(QKNormError::BackendError)?;
query_kernel = Some(kernel);
query_scales_buffer = Some(scales_buffer);
@@ -75,26 +79,24 @@ impl QKNorm {
// Setup key normalization if configured
if let Some(ref k_config) = key_config {
- let scales_param = parameter_tree.leaf("key_norm.scales").map_err(|e| {
- MTLError::Library(crate::backends::metal::error::LibraryError::Custom(format!(
- "Failed to load key scales: {:?}",
- e
- )))
- })?;
-
- let scales_data = scales_param.as_bytes();
- let scales_buffer =
- context.device.new_buffer_with_data(scales_data, MTLResourceOptions::STORAGE_MODE_SHARED);
+ let scales = parameter_tree.leaf("key_norm.scales").map_err(QKNormError::ParameterError)?;
+ let scales_buffer = scales.buffer().clone();
let accumulation_data_type: DataType = k_config.accumulation_precision.into();
let scale_data_type: DataType = k_config.scale_precision.into();
-
let (input_type, scales_type, output_type) = match k_config.upcast_mode {
UpcastMode::OnlyNormalization => (intermediate_data_type, scale_data_type, scale_data_type),
UpcastMode::FullLayer => (intermediate_data_type, scale_data_type, scale_data_type),
};
- let kernel = QKNormMetalKernel::new(context, input_type, scales_type, output_type, accumulation_data_type)?;
+ let kernel = <B::Kernels as Kernels>::QKNormKernel::new(
+ context,
+ input_type,
+ scales_type,
+ output_type,
+ accumulation_data_type,
+ )
+ .map_err(QKNormError::BackendError)?;
key_kernel = Some(kernel);
key_scales_buffer = Some(scales_buffer);
@@ -106,8 +108,8 @@ impl QKNorm {
query_config,
key_config,
qkv_array_id,
- query_scales_buffer: query_scales_buffer.flatten(),
- key_scales_buffer: key_scales_buffer.flatten(),
+ query_scales_buffer,
+ key_scales_buffer,
num_q_heads,
num_kv_heads,
head_dim,
@@ -115,33 +117,16 @@ impl QKNorm {
}
}
-impl EncodableBlock<Metal> for QKNorm {
- fn encode(
- &self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- ) {
- let compute_encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.encode_with_shared_encoder(state, parameters, &compute_encoder);
- compute_encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
- }
-
+impl<B: Backend> EncodableBlock<B> for QKNorm<B> {
fn supports_shared_encoder(&self) -> bool {
true
}
fn encode_with_shared_encoder(
&self,
- state: &mut ForwardPassState<Metal>,
- _parameters: &EncodingParameters<Metal>,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ state: &mut ForwardPassState<B>,
+ _parameters: &EncodingParameters<B>,
+ encoder: &B::ComputeEncoder,
) {
let qkv_binding = state.arrays(&[self.qkv_array_id]);
let qkv_shape = {
@@ -151,29 +136,26 @@ impl EncodableBlock<Metal> for QKNorm {
let qkv_array = qkv_binding[0].borrow_mut();
let qkv_buffer = qkv_array.buffer();
-
- let batch_size = qkv_shape[0] as i32;
- let head_dim = self.head_dim as i32;
+ let batch_size = qkv_shape[0] as u32;
// Process query normalization if configured
if let (Some(query_kernel), Some(query_scales_buffer), Some(query_config)) =
(&self.query_kernel, &self.query_scales_buffer, &self.query_config)
{
- let query_head_count = self.num_q_heads as u32;
query_kernel.encode(
qkv_buffer,
query_scales_buffer,
qkv_buffer,
- batch_size as u32,
+ batch_size,
self.num_q_heads as u32,
self.num_kv_heads as u32,
- head_dim as u32,
+ self.head_dim as u32,
query_config.epsilon,
query_config.scale_offset.unwrap_or(0.0),
0,
- query_head_count,
+ self.num_q_heads as u32,
query_config.upcast_mode == UpcastMode::FullLayer,
- compute_encoder,
+ encoder,
);
}
@@ -181,21 +163,20 @@ impl EncodableBlock<Metal> for QKNorm {
if let (Some(key_kernel), Some(key_scales_buffer), Some(key_config)) =
(&self.key_kernel, &self.key_scales_buffer, &self.key_config)
{
- let key_head_count = self.num_kv_heads as u32;
key_kernel.encode(
qkv_buffer,
key_scales_buffer,
qkv_buffer,
- batch_size as u32,
+ batch_size,
self.num_q_heads as u32,
self.num_kv_heads as u32,
- head_dim as u32,
+ self.head_dim as u32,
key_config.epsilon,
key_config.scale_offset.unwrap_or(0.0),
self.num_q_heads as u32,
- key_head_count,
+ self.num_kv_heads as u32,
key_config.upcast_mode == UpcastMode::FullLayer,
- compute_encoder,
+ encoder,
);
}
}