Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Small cleanup part 3 (#953)

uuuvn85dc4df22f81
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
index 63a68fec6..44f2706bb 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
@@ -107,22 +107,18 @@ impl<B: Backend> Attention<B> {
// TODO: Fix lalamo config, those two must be None if kv sharing.
let key_norm_config = (!is_kv_sharing).then(|| config.key_norm_config.clone()).flatten();
let value_norm_config = (!is_kv_sharing).then(|| config.value_norm_config()).flatten();
- let qkv_norm = (query_norm_config.is_some() || key_norm_config.is_some() || value_norm_config.is_some())
- .then(|| {
- QKVNorm::new(
- config.num_heads,
- num_kv_heads.unwrap_or(0), // TODO: should take option
- projection_dim,
- config.head_dim,
- data_type,
- query_norm_config,
- key_norm_config,
- value_norm_config,
- parameter_tree,
- context,
- )
- })
- .transpose()?;
+ let qkv_norm = QKVNorm::new(
+ num_q_heads,
+ num_kv_heads,
+ projection_dim,
+ head_dim,
+ data_type,
+ query_norm_config,
+ key_norm_config,
+ value_norm_config,
+ parameter_tree,
+ context,
+ )?;
let prepare = <B::Kernels as Kernels>::AttentionPrepareKernel::new(
context,
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
index d9cb25e90..d937a5e3c 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
@@ -16,7 +16,7 @@ use crate::{
pub(super) struct LinearProjection<B: Backend> {
pub(super) lin: Box<dyn Linear<B>>,
- pub(super) norm: Option<QKVNorm<B>>,
+ pub(super) norm: QKVNorm<B>,
}
impl<B: Backend> LinearProjection<B> {
@@ -27,9 +27,7 @@ impl<B: Backend> LinearProjection<B> {
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
let mut projected = self.lin.encode(hidden, batch_dim, command_buffer)?;
- if let Some(norm) = &self.norm {
- norm.encode(&mut projected, batch_dim, command_buffer)?;
- }
+ self.norm.encode(&mut projected, batch_dim, command_buffer);
Ok(projected)
}
}
@@ -121,9 +119,7 @@ impl<B: Backend> Attention<B> {
state: &mut AttentionState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- if let Some(norm) = &self.projection.norm {
- norm.encode_key_value(key_value.reborrow(), batch_dim, command_buffer)?;
- }
+ self.projection.norm.encode_key_value(key_value.reborrow(), batch_dim, command_buffer);
let prefix_len = state.view().prefix_len();
self.prepare_kv_and_queries(
key_value.as_ref(),
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
index 9a9a1fbde..5fc53ef76 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/qkv_norm.rs
@@ -30,7 +30,7 @@ pub struct QKVNorm<B: Backend> {
key: Option<Head<B>>,
value: Option<Head<B>>,
num_q_heads: u32,
- num_kv_heads: u32,
+ num_kv_heads: Option<u32>,
projection_row_stride: u32,
head_dim: u32,
}
@@ -38,7 +38,7 @@ pub struct QKVNorm<B: Backend> {
impl<B: Backend> QKVNorm<B> {
pub fn new(
num_q_heads: u32,
- num_kv_heads: u32,
+ num_kv_heads: Option<u32>,
projection_row_stride: u32,
head_dim: u32,
intermediate_data_type: DataType,
@@ -48,36 +48,41 @@ impl<B: Backend> QKVNorm<B> {
parameter_tree: &ParameterTree<B>,
context: &B::Context,
) -> Result<Self, QKVNormError<B>> {
- let query = query_config
- .map(|cfg| {
- Self::build_head(
- context,
- intermediate_data_type,
- cfg,
- Some(&parameter_tree.subtree("query_norm")),
- head_dim,
- )
- })
- .transpose()?;
- let key = key_config
- .map(|cfg| {
- Self::build_head(
- context,
- intermediate_data_type,
- cfg,
- Some(&parameter_tree.subtree("key_norm")),
- head_dim,
- )
- })
- .transpose()?;
- let value = value_config
- .map(|cfg| Self::build_head(context, intermediate_data_type, cfg, None, head_dim))
- .transpose()?;
+ let build_head = |config: Option<NormalizationConfig>, parameter_name: Option<&str>| {
+ let Some(config) = config else {
+ return Ok::<_, QKVNormError<B>>(None);
+ };
+ let scales = config
+ .has_scale
+ .then(|| {
+ parameter_tree
+ .subtree(parameter_name.expect("scaled norm requires parameter tree"))
+ .leaf("scales")?
+ .validate(&[head_dim], DataType::F32)?
+ .read_buffer()
+ })
+ .transpose()?;
+ let kernel = <B::Kernels as Kernels>::QKVNormKernel::new(
+ context,
+ intermediate_data_type,
+ DataType::F32,
+ intermediate_data_type,
+ DataType::F32,
+ true,
+ scales.is_some(),
+ )
+ .map_err(QKVNormError::BackendError)?;
+ Ok(Some(Head {
+ kernel,
+ scales,
+ config,
+ }))
+ };
Ok(Self {
- query,
- key,
- value,
+ query: build_head(query_config, Some("query_norm"))?,
+ key: build_head(key_config, Some("key_norm"))?,
+ value: build_head(value_config, None)?,
num_q_heads,
num_kv_heads,
projection_row_stride,
@@ -85,47 +90,12 @@ impl<B: Backend> QKVNorm<B> {
})
}
- fn build_head(
- context: &B::Context,
- intermediate_data_type: DataType,
- config: NormalizationConfig,
- parameter_tree: Option<&ParameterTree<B>>,
- head_dim: u32,
- ) -> Result<Head<B>, QKVNormError<B>> {
- let scales = if config.has_scale {
- Some(
- parameter_tree
- .expect("scaled norm requires parameter tree")
- .leaf("scales")?
- .validate(&[head_dim], DataType::F32)?
- .read_buffer()?,
- )
- } else {
- None
- };
- let kernel = <B::Kernels as Kernels>::QKVNormKernel::new(
- context,
- intermediate_data_type,
- DataType::F32,
- intermediate_data_type,
- DataType::F32,
- true,
- scales.is_some(),
- )
- .map_err(QKVNormError::BackendError)?;
- Ok(Head {
- kernel,
- scales,
- config,
- })
- }
-
pub fn encode(
&self,
qkvg: impl BufferMut<Backend = B>,
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), B::Error> {
+ ) {
self.encode_packed(qkvg, batch_dim, self.num_q_heads, self.projection_row_stride, command_buffer)
}
@@ -134,8 +104,9 @@ impl<B: Backend> QKVNorm<B> {
key_value: impl BufferMut<Backend = B>,
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), B::Error> {
- self.encode_packed(key_value, batch_dim, 0, 2 * self.num_kv_heads * self.head_dim, command_buffer)
+ ) {
+ let num_kv_heads = self.num_kv_heads.expect("KV norm requires KV heads");
+ self.encode_packed(key_value, batch_dim, 0, 2 * num_kv_heads * self.head_dim, command_buffer)
}
fn encode_packed(
@@ -145,14 +116,14 @@ impl<B: Backend> QKVNorm<B> {
q_heads: u32,
input_row_stride: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), B::Error> {
- let packed_row_width = (q_heads + 2 * self.num_kv_heads) * self.head_dim;
+ ) {
+ let kv = self.num_kv_heads.unwrap_or(0);
+ let packed_row_width = (q_heads + 2 * kv) * self.head_dim;
assert!(
input_row_stride >= packed_row_width,
"QKV norm input row stride ({input_row_stride}) is smaller than its packed row width ({packed_row_width})"
);
- let kv = self.num_kv_heads;
let heads = [(&self.query, 0, q_heads), (&self.key, q_heads, kv), (&self.value, q_heads + kv, kv)];
for (head, head_offset, head_count) in heads {
let Some(head) = head else {
@@ -176,8 +147,6 @@ impl<B: Backend> QKVNorm<B> {
command_buffer,
);
}
-
- Ok(())
}
}
diff --git a/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs b/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs
index 15a85cc3b..7e21752d0 100644
--- a/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs
+++ b/crates/uzu-engine/unit/encodable_block/attention/qkv_norm_test.rs
@@ -1,8 +1,11 @@
use uzu_engine_macros::uzu_test;
-use super::QKVNorm;
+use super::{Head, QKVNorm};
use crate::{
- backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context},
+ backends::common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
+ kernel::QKVNormKernel,
+ },
config::normalization::{NormalizationConfig, UpcastMode},
data_type::DataType,
tests::{
@@ -27,14 +30,26 @@ fn run_key_value_row_stride_test<B: Backend>() {
has_scale: false,
has_biases: false,
};
- let key = QKVNorm::<B>::build_head(&context, DataType::F32, config, None, HEAD_DIM)
- .expect("failed to construct key norm");
+ let key = Head {
+ kernel: <B::Kernels as Kernels>::QKVNormKernel::new(
+ &context,
+ DataType::F32,
+ DataType::F32,
+ DataType::F32,
+ DataType::F32,
+ true,
+ false,
+ )
+ .expect("failed to construct key norm"),
+ scales: None,
+ config,
+ };
let norm = QKVNorm {
query: None,
key: Some(key),
value: None,
num_q_heads: 0,
- num_kv_heads: NUM_KV_HEADS,
+ num_kv_heads: Some(NUM_KV_HEADS),
projection_row_stride: ROW_WIDTH,
head_dim: HEAD_DIM,
};
@@ -57,7 +72,7 @@ fn run_key_value_row_stride_test<B: Backend>() {
let mut key_value = create_buffer_with_data::<B, f32>(&context, &input);
let mut command_buffer = context.create_command_buffer(None, None).expect("failed to create command buffer");
- norm.encode_key_value(&mut key_value, BATCH_SIZE, &mut command_buffer).expect("failed to encode key/value norm");
+ norm.encode_key_value(&mut key_value, BATCH_SIZE, &mut command_buffer);
command_buffer.end_encoding().submit().wait_until_completed().expect("failed to execute key/value norm");
let output = buffer_to_vec::<B, f32>(&key_value);