A high-performance inference engine for AI models
Small cleanup part 3 (#953)
uuuvn85dc4df22f81
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rsindex 63a68fec6..44f2706bb 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs@@ -107,22 +107,18 @@ impl<B: Backend> Attention<B> {// TODO: Fix lalamo config, those two must be None if kv sharing.let key_norm_config = (!is_kv_sharing).then(|| config.key_norm_config.clone()).flatten();let value_norm_config = (!is_kv_sharing).then(|| config.value_norm_config()).flatten();- let qkv_norm = (query_norm_config.is_some() || key_norm_config.is_some() || value_norm_config.is_some())- .then(|| {- QKVNorm::new(- config.num_heads,- num_kv_heads.unwrap_or(0), // TODO: should take option- projection_dim,- config.head_dim,- data_type,- query_norm_config,- key_norm_config,- value_norm_config,- parameter_tree,- context,- )- })- .transpose()?;+ let qkv_norm = QKVNorm::new(+ num_q_heads,+ num_kv_heads,+ projection_dim,+ head_dim,+ data_type,+ query_norm_config,+ key_norm_config,+ value_norm_config,+ parameter_tree,+ context,+ )?;let prepare = <B::Kernels as Kernels>::AttentionPrepareKernel::new(context,diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rsindex d9cb25e90..d937a5e3c 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs@@ -16,7 +16,7 @@ use crate::{pub(super) struct LinearProjection<B: Backend> {pub(super) lin: Box<dyn Linear<B>>,- pub(super) norm: Option<QKVNorm<B>>,+ pub(super) norm: QKVNorm<B>,}impl<B: Backend> LinearProjection<B> {@@ -27,9 +27,7 @@ impl<B: Backend> LinearProjection<B> {command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {let mut projected = self.lin.encode(hidden, batch_dim, command_buffer)?;- if let Some(norm) = &self.norm {- norm.encode(&mut projected, batch_dim, command_buffer)?;- }+ self.norm.encode(&mut projected, batch_dim, command_buffer);Ok(projected)}}@@ -121,9 +119,7 @@ impl<B: Backend> Attention<B> {state: &mut AttentionState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- if let Some(norm) = &self.projection.norm {- norm.encode_key_value(key_value.reborrow(), batch_dim, command_buffer)?;- }+ self.projection.norm.encode_key_value(key_value.reborrow(), batch_dim, command_buffer);let prefix_len = state.view().prefix_len();self.prepare_kv_and_queries(key_value.as_ref(),diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rsindex 9a9a1fbde..5fc53ef76 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs@@ -30,7 +30,7 @@ pub struct QKVNorm<B: Backend> {key: Option<Head<B>>,value: Option<Head<B>>,num_q_heads: u32,- num_kv_heads: u32,+ num_kv_heads: Option<u32>,projection_row_stride: u32,head_dim: u32,}@@ -38,7 +38,7 @@ pub struct QKVNorm<B: Backend> {impl<B: Backend> QKVNorm<B> {pub fn new(num_q_heads: u32,- num_kv_heads: u32,+ num_kv_heads: Option<u32>,projection_row_stride: u32,head_dim: u32,intermediate_data_type: DataType,@@ -48,36 +48,41 @@ impl<B: Backend> QKVNorm<B> {parameter_tree: &ParameterTree<B>,context: &B::Context,) -> Result<Self, QKVNormError<B>> {- let query = query_config- .map(|cfg| {- Self::build_head(- context,- intermediate_data_type,- cfg,- Some(¶meter_tree.subtree("query_norm")),- head_dim,- )- })- .transpose()?;- let key = key_config- .map(|cfg| {- Self::build_head(- context,- intermediate_data_type,- cfg,- Some(¶meter_tree.subtree("key_norm")),- head_dim,- )- })- .transpose()?;- let value = value_config- .map(|cfg| Self::build_head(context, intermediate_data_type, cfg, None, head_dim))- .transpose()?;+ let build_head = |config: Option<NormalizationConfig>, parameter_name: Option<&str>| {+ let Some(config) = config else {+ return Ok::<_, QKVNormError<B>>(None);+ };+ let scales = config+ .has_scale+ .then(|| {+ parameter_tree+ .subtree(parameter_name.expect("scaled norm requires parameter tree"))+ .leaf("scales")?+ .validate(&[head_dim], DataType::F32)?+ .read_buffer()+ })+ .transpose()?;+ let kernel = <B::Kernels as Kernels>::QKVNormKernel::new(+ context,+ intermediate_data_type,+ DataType::F32,+ intermediate_data_type,+ DataType::F32,+ true,+ scales.is_some(),+ )+ .map_err(QKVNormError::BackendError)?;+ Ok(Some(Head {+ kernel,+ scales,+ config,+ }))+ };Ok(Self {- query,- key,- value,+ query: build_head(query_config, Some("query_norm"))?,+ key: build_head(key_config, Some("key_norm"))?,+ value: build_head(value_config, None)?,num_q_heads,num_kv_heads,projection_row_stride,@@ -85,47 +90,12 @@ impl<B: Backend> QKVNorm<B> {})}- fn build_head(- context: &B::Context,- intermediate_data_type: DataType,- config: NormalizationConfig,- parameter_tree: Option<&ParameterTree<B>>,- head_dim: u32,- ) -> Result<Head<B>, QKVNormError<B>> {- let scales = if config.has_scale {- Some(- parameter_tree- .expect("scaled norm requires parameter tree")- .leaf("scales")?- .validate(&[head_dim], DataType::F32)?- .read_buffer()?,- )- } else {- None- };- let kernel = <B::Kernels as Kernels>::QKVNormKernel::new(- context,- intermediate_data_type,- DataType::F32,- intermediate_data_type,- DataType::F32,- true,- scales.is_some(),- )- .map_err(QKVNormError::BackendError)?;- Ok(Head {- kernel,- scales,- config,- })- }-pub fn encode(&self,qkvg: impl BufferMut<Backend = B>,batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), B::Error> {+ ) {self.encode_packed(qkvg, batch_dim, self.num_q_heads, self.projection_row_stride, command_buffer)}@@ -134,8 +104,9 @@ impl<B: Backend> QKVNorm<B> {key_value: impl BufferMut<Backend = B>,batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), B::Error> {- self.encode_packed(key_value, batch_dim, 0, 2 * self.num_kv_heads * self.head_dim, command_buffer)+ ) {+ let num_kv_heads = self.num_kv_heads.expect("KV norm requires KV heads");+ self.encode_packed(key_value, batch_dim, 0, 2 * num_kv_heads * self.head_dim, command_buffer)}fn encode_packed(@@ -145,14 +116,14 @@ impl<B: Backend> QKVNorm<B> {q_heads: u32,input_row_stride: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), B::Error> {- let packed_row_width = (q_heads + 2 * self.num_kv_heads) * self.head_dim;+ ) {+ let kv = self.num_kv_heads.unwrap_or(0);+ let packed_row_width = (q_heads + 2 * kv) * self.head_dim;assert!(input_row_stride >= packed_row_width,"QKV norm input row stride ({input_row_stride}) is smaller than its packed row width ({packed_row_width})");- let kv = self.num_kv_heads;let heads = [(&self.query, 0, q_heads), (&self.key, q_heads, kv), (&self.value, q_heads + kv, kv)];for (head, head_offset, head_count) in heads {let Some(head) = head else {@@ -176,8 +147,6 @@ impl<B: Backend> QKVNorm<B> {command_buffer,);}-- Ok(())}}diff --git a/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs b/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rsindex 15a85cc3b..7e21752d0 100644--- a/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs+++ b/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs@@ -1,8 +1,11 @@use uzu_engine_macros::uzu_test;-use super::QKVNorm;+use super::{Head, QKVNorm};use crate::{- backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context},+ backends::common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,+ kernel::QKVNormKernel,+ },config::normalization::{NormalizationConfig, UpcastMode},data_type::DataType,tests::{@@ -27,14 +30,26 @@ fn run_key_value_row_stride_test<B: Backend>() {has_scale: false,has_biases: false,};- let key = QKVNorm::<B>::build_head(&context, DataType::F32, config, None, HEAD_DIM)- .expect("failed to construct key norm");+ let key = Head {+ kernel: <B::Kernels as Kernels>::QKVNormKernel::new(+ &context,+ DataType::F32,+ DataType::F32,+ DataType::F32,+ DataType::F32,+ true,+ false,+ )+ .expect("failed to construct key norm"),+ scales: None,+ config,+ };let norm = QKVNorm {query: None,key: Some(key),value: None,num_q_heads: 0,- num_kv_heads: NUM_KV_HEADS,+ num_kv_heads: Some(NUM_KV_HEADS),projection_row_stride: ROW_WIDTH,head_dim: HEAD_DIM,};@@ -57,7 +72,7 @@ fn run_key_value_row_stride_test<B: Backend>() {let mut key_value = create_buffer_with_data::<B, f32>(&context, &input);let mut command_buffer = context.create_command_buffer(None, None).expect("failed to create command buffer");- norm.encode_key_value(&mut key_value, BATCH_SIZE, &mut command_buffer).expect("failed to encode key/value norm");+ norm.encode_key_value(&mut key_value, BATCH_SIZE, &mut command_buffer);command_buffer.end_encoding().submit().wait_until_completed().expect("failed to execute key/value norm");let output = buffer_to_vec::<B, f32>(&key_value);