Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Support Gated Attention (#278)

CCYehe80807a20ee0
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu/src/backends/cpu/kernel/attention/mod.rs
index 24f00ec54..4a6321bee 100644
--- a/crates/uzu/src/backends/cpu/kernel/attention/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/attention/mod.rs
@@ -2,3 +2,4 @@ pub mod attention_gemm;
pub mod attention_single_pass;
pub mod attention_two_pass;
pub mod attention_update_kv_cache;
+pub mod sigmoid_gate;
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/sigmoid_gate.rs b/crates/uzu/src/backends/cpu/kernel/attention/sigmoid_gate.rs
new file mode 100644
index 000000000..6f5c8e49b
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/attention/sigmoid_gate.rs
@@ -0,0 +1,22 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(SigmoidGate)]
+#[variants(T, f32, f16, bf16)]
+pub fn sigmoid_gate<T: ArrayElement + Float>(
+ gate: *const T,
+ output: *mut T,
+ total_elements: u32,
+) {
+ for idx in 0..total_elements as usize {
+ unsafe {
+ let g = (*gate.add(idx)).to_f32().unwrap();
+ let sigmoid = 1.0f32 / (1.0f32 + (-g).exp());
+ let out = (*output.add(idx)).to_f32().unwrap();
+ *output.add(idx) = T::from(out * sigmoid).unwrap();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/sigmoid_gate.metal b/crates/uzu/src/backends/metal/kernel/attention/sigmoid_gate.metal
new file mode 100644
index 000000000..876609196
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/attention/sigmoid_gate.metal
@@ -0,0 +1,16 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(SigmoidGate)(
+ const device T* gate,
+ device T* output,
+ const constant uint& total_elements,
+ const uint tid AXIS(total_elements, 256)
+) {
+ if (tid >= total_elements)
+ return;
+ float sigmoid = 1.0f / (1.0f + exp(-float(gate[tid])));
+ output[tid] = T(float(output[tid]) * sigmoid);
+}
diff --git a/crates/uzu/src/config/attention.rs b/crates/uzu/src/config/attention.rs
index d74cdb3fe..a24b72bb4 100644
--- a/crates/uzu/src/config/attention.rs
+++ b/crates/uzu/src/config/attention.rs
@@ -23,6 +23,8 @@ pub struct AttentionConfig {
pub has_qkv_biases: bool,
pub has_out_biases: bool,
#[serde(default)]
+ pub has_gate: bool,
+ #[serde(default)]
pub partial_rope_dim: Option<usize>,
}
diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rs
index aabd892f6..c5dad382e 100644
--- a/crates/uzu/src/encodable_block/attention.rs
+++ b/crates/uzu/src/encodable_block/attention.rs
@@ -11,6 +11,7 @@ use crate::{
Backend, CommandBuffer, Kernels,
kernel::{
AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel,
+ SigmoidGateKernel,
attention::{AttentionGemmArguments, AttentionGemmBlock},
},
},
@@ -40,6 +41,7 @@ pub struct Attention<B: Backend> {
two_pass_2_kernels: HashMap<u32, <B::Kernels as Kernels>::AttentionTwoPass2Kernel>,
update_kv_cache_kernel: <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel,
update_kv_cache_inplace_kernel: <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel,
+ gate_kernel: Option<<B::Kernels as Kernels>::SigmoidGateKernel>,
gemm_block: AttentionGemmBlock<B>,
layer_index: usize,
attention_scale: Option<f32>,
@@ -57,6 +59,7 @@ impl<B: Backend> Attention<B> {
has_sinks: bool,
is_causal: bool,
sliding_window_size: Option<usize>,
+ has_gate: bool,
) -> Result<Self, B::Error> {
let mut single_pass_kernels = HashMap::new();
let mut two_pass_1_kernels = HashMap::new();
@@ -90,6 +93,11 @@ impl<B: Backend> Attention<B> {
<B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, false)?;
let update_kv_cache_inplace_kernel =
<B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, true)?;
+ let gate_kernel = if has_gate {
+ Some(<B::Kernels as Kernels>::SigmoidGateKernel::new(context, data_type)?)
+ } else {
+ None
+ };
let gemm_block = AttentionGemmBlock::new(data_type);
Ok(Self {
@@ -98,6 +106,7 @@ impl<B: Backend> Attention<B> {
two_pass_2_kernels,
update_kv_cache_kernel,
update_kv_cache_inplace_kernel,
+ gate_kernel,
gemm_block,
layer_index,
attention_scale,
@@ -198,6 +207,7 @@ impl<B: Backend> Attention<B> {
let qkv_binding = state.arrays(&[ArrayId::QKV]);
let attention_bias_binding = state.hashmaps(&[HashMapId::AttentionBias]);
let attention_output_binding = state.arrays(&[ArrayId::AttentionOutput]);
+ let gate_binding = self.gate_kernel.as_ref().map(|_| state.arrays(&[ArrayId::Gate]));
let mask_kv_seq_stride = 1;
let mask_q_seq_stride = sequence_length as i32;
@@ -436,6 +446,19 @@ impl<B: Backend> Attention<B> {
},
}
+ if let Some(gate_kernel) = &self.gate_kernel {
+ let gate_array = gate_binding.as_ref().unwrap()[0].borrow();
+ let gate_buf_rc = gate_array.buffer();
+ let gate_buf_borrow = gate_buf_rc.borrow();
+ let total_elements = (suffix_length * num_heads * head_dim) as u32;
+ gate_kernel.encode(
+ gate_buf_borrow.deref(),
+ attention_output_buf_borrow.deref_mut(),
+ total_elements,
+ command_buffer,
+ );
+ }
+
Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rs
index 658d6de62..5ead1cf31 100644
--- a/crates/uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/encodable_block/classifier_layer.rs
@@ -191,6 +191,7 @@ impl<B: Backend> ClassifierLayer<B> {
attention_config.has_sinks,
false,
attention_config.sliding_window_size,
+ false,
)
.expect("Failed to create attention kernel");
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index 0de13c75a..f785afb80 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -84,6 +84,24 @@ impl<B: Backend> LayerExecutables<B> {
)
.expect("Failed to create qkv projection");
+ let gate_projection = if attention_config.has_gate {
+ Some(
+ <dyn Linear<B>>::new(
+ &attention_config.qkv_projection_config,
+ false,
+ model_dim,
+ [num_heads * head_dim],
+ ctx,
+ &decoder_layer_loader.subtree("mixer.gate_projection").unwrap(),
+ ArrayId::Main,
+ ArrayId::Gate,
+ )
+ .expect("Failed to create gate projection"),
+ )
+ } else {
+ None
+ };
+
let qk_norm =
if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {
match QKNorm::new(
@@ -124,11 +142,13 @@ impl<B: Backend> LayerExecutables<B> {
attention_config.has_sinks,
attention_config.is_causal.unwrap_or(true),
attention_config.sliding_window_size,
+ attention_config.has_gate,
)
.expect("Failed to create AttentionWrapper kernel");
MixerExecutables::Attention {
qkv_projection,
+ gate_projection,
qk_norm,
rope: rope_block,
attention,
@@ -264,12 +284,16 @@ impl<B: Backend> LayerExecutables<B> {
match &self.mixer {
MixerExecutables::Attention {
qkv_projection,
+ gate_projection,
qk_norm,
rope,
attention,
out_projection,
} => {
qkv_projection.encode(state, command_buffer)?;
+ if let Some(gate_proj) = gate_projection {
+ gate_proj.encode(state, command_buffer)?;
+ }
if let Some(norm) = qk_norm {
norm.encode(state, command_buffer)?;
}
diff --git a/crates/uzu/src/encodable_block/layer/mixer.rs b/crates/uzu/src/encodable_block/layer/mixer.rs
index e709d25b0..749ebbc0b 100644
--- a/crates/uzu/src/encodable_block/layer/mixer.rs
+++ b/crates/uzu/src/encodable_block/layer/mixer.rs
@@ -11,6 +11,7 @@ use crate::{
pub(crate) enum MixerExecutables<B: Backend> {
Attention {
qkv_projection: Box<dyn Linear<B>>,
+ gate_projection: Option<Box<dyn Linear<B>>>,
qk_norm: Option<QKNorm<B>>,
rope: Rc<Rope<B>>,
attention: Attention<B>,
diff --git a/crates/uzu/src/forward_pass/model_shape.rs b/crates/uzu/src/forward_pass/model_shape.rs
index 634ce39fe..da82e1408 100644
--- a/crates/uzu/src/forward_pass/model_shape.rs
+++ b/crates/uzu/src/forward_pass/model_shape.rs
@@ -26,6 +26,7 @@ pub struct ModelShape {
max_mamba_state_dim: usize,
max_mamba_kernel_size: usize,
max_rope_dim: usize,
+ has_gate: bool,
}
impl ModelShape {
@@ -75,6 +76,7 @@ impl ModelShape {
}
}
let mut max_rope_dim = 0usize;
+ let mut has_gate = false;
let all_layer_configs = decoder_config
.layer_configs
@@ -86,6 +88,7 @@ impl ModelShape {
if let MixerConfig::Attention(attn) = &layer_config.mixer_config {
let hd = attn.head_dim.unwrap_or(decoder_config.head_dim);
max_rope_dim = max_rope_dim.max(attn.partial_rope_dim.unwrap_or(hd));
+ has_gate = has_gate || attn.has_gate;
}
}
@@ -116,6 +119,7 @@ impl ModelShape {
max_mamba_state_dim,
max_mamba_kernel_size,
max_rope_dim,
+ has_gate,
}
}
@@ -208,6 +212,13 @@ impl ModelShape {
[suffix_length, self.num_heads * self.head_dim]
}
+ pub fn gate_shape(
+ &self,
+ suffix_length: usize,
+ ) -> Option<[usize; 2]> {
+ self.has_gate.then(|| [suffix_length, self.num_heads * self.head_dim])
+ }
+
pub fn rotated_queries_shape(
&self,
suffix_length: usize,
diff --git a/crates/uzu/src/forward_pass/scratch_buffers.rs b/crates/uzu/src/forward_pass/scratch_buffers.rs
index a29fd934a..b8d690e33 100644
--- a/crates/uzu/src/forward_pass/scratch_buffers.rs
+++ b/crates/uzu/src/forward_pass/scratch_buffers.rs
@@ -26,6 +26,7 @@ pub struct ScratchBuffers<B: Backend> {
pub main: ArrayCell<B>,
pub shortcut: ArrayCell<B>,
pub qkv: ArrayCell<B>,
+ pub gate: Option<ArrayCell<B>>,
pub attention_output: ArrayCell<B>,
pub mlp_fused_up: ArrayCell<B>,
pub mlp_hidden: ArrayCell<B>,
@@ -128,6 +129,7 @@ impl<B: Backend> ScratchBuffers<B> {
main: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "main"),
shortcut: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "shortcut"),
qkv: alloc(&model_shape.qkv_shape(max_suffix_len), act_ty, "qkv"),
+ gate: model_shape.gate_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "gate")),
attention_output: alloc(&model_shape.attention_output_shape(max_suffix_len), act_ty, "attention_output"),
mlp_fused_up: alloc(&model_shape.mlp_fused_up_shape(max_suffix_len), act_ty, "mlp_fused_up"),
mlp_hidden: alloc(&model_shape.mlp_hidden_shape(max_suffix_len), act_ty, "mlp_hidden"),
diff --git a/crates/uzu/src/forward_pass/state/array_id.rs b/crates/uzu/src/forward_pass/state/array_id.rs
index b0b6af703..09afec89b 100644
--- a/crates/uzu/src/forward_pass/state/array_id.rs
+++ b/crates/uzu/src/forward_pass/state/array_id.rs
@@ -14,6 +14,7 @@ pub enum ArrayId {
Main,
Shortcut,
QKV,
+ Gate,
AttentionOutput,
MlpFusedUp,
MlpHidden,
diff --git a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs b/crates/uzu/src/forward_pass/state/common_aux_buffers.rs
index 7c7b38beb..58af10f2f 100644
--- a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs
+++ b/crates/uzu/src/forward_pass/state/common_aux_buffers.rs
@@ -9,6 +9,7 @@ pub struct CommonAuxBuffers<B: Backend> {
pub main: ArrayCell<B>,
pub shortcut: ArrayCell<B>,
pub qkv: ArrayCell<B>,
+ pub gate: Option<ArrayCell<B>>,
pub attention_output: ArrayCell<B>,
pub mlp_fused_up: ArrayCell<B>,
pub mlp_hidden: ArrayCell<B>,
@@ -31,6 +32,7 @@ impl<B: Backend> CommonAuxBuffers<B> {
main: scratch.main.view(&model_shape.main_shape(suffix_length)),
shortcut: scratch.shortcut.view(&model_shape.main_shape(suffix_length)),
qkv: scratch.qkv.view(&model_shape.qkv_shape(suffix_length)),
+ gate: scratch.gate.as_ref().zip(model_shape.gate_shape(suffix_length)).map(|(buf, shape)| buf.view(&shape)),
attention_output: scratch.attention_output.view(&model_shape.attention_output_shape(suffix_length)),
mlp_fused_up: scratch.mlp_fused_up.view(&model_shape.mlp_fused_up_shape(suffix_length)),
mlp_hidden: scratch.mlp_hidden.view(&model_shape.mlp_hidden_shape(suffix_length)),
diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rs
index 58ac194b3..8112db011 100644
--- a/crates/uzu/src/forward_pass/state/state.rs
+++ b/crates/uzu/src/forward_pass/state/state.rs
@@ -485,6 +485,7 @@ impl<B: Backend> ForwardPassState<B> {
ArrayId::Main => self.common_aux.main.clone(),
ArrayId::Shortcut => self.common_aux.shortcut.clone(),
ArrayId::QKV => self.common_aux.qkv.clone(),
+ ArrayId::Gate => self.common_aux.gate.clone().expect("Gate buffer not available"),
ArrayId::AttentionOutput => self.common_aux.attention_output.clone(),
ArrayId::MlpFusedUp => self.common_aux.mlp_fused_up.clone(),
ArrayId::MlpHidden => self.common_aux.mlp_hidden.clone(),
diff --git a/crates/uzu/tests/unit/config/attention_test.rs b/crates/uzu/tests/unit/config/attention_test.rs
index 6eb9b9016..6a7f86d3f 100644
--- a/crates/uzu/tests/unit/config/attention_test.rs
+++ b/crates/uzu/tests/unit/config/attention_test.rs
@@ -76,6 +76,7 @@ fn test_attention_config() {
has_sinks: false,
has_qkv_biases: false,
has_out_biases: false,
+ has_gate: false,
partial_rope_dim: None,
};
diff --git a/crates/uzu/tests/unit/config/decoder_layer_test.rs b/crates/uzu/tests/unit/config/decoder_layer_test.rs
index f437d261c..adc7e52d4 100644
--- a/crates/uzu/tests/unit/config/decoder_layer_test.rs
+++ b/crates/uzu/tests/unit/config/decoder_layer_test.rs
@@ -132,6 +132,7 @@ fn test_decoder_layer_config() {
has_sinks: false,
has_qkv_biases: false,
has_out_biases: false,
+ has_gate: false,
partial_rope_dim: None,
}),
mlp_config: MLPConfig::Dense(mlp::DenseMLPConfig {
diff --git a/crates/uzu/tests/unit/config/decoder_test.rs b/crates/uzu/tests/unit/config/decoder_test.rs
index 4c5473b97..b64e56a5d 100644
--- a/crates/uzu/tests/unit/config/decoder_test.rs
+++ b/crates/uzu/tests/unit/config/decoder_test.rs
@@ -195,6 +195,7 @@ fn test_decoder_config() {
has_sinks: false,
has_qkv_biases: false,
has_out_biases: false,
+ has_gate: false,
partial_rope_dim: None,
}),
mlp_config: MLPConfig::Dense(DenseMLPConfig {
diff --git a/crates/uzu/tests/unit/config/transformer_layer_test.rs b/crates/uzu/tests/unit/config/transformer_layer_test.rs
index 5889e1f5d..117312912 100644
--- a/crates/uzu/tests/unit/config/transformer_layer_test.rs
+++ b/crates/uzu/tests/unit/config/transformer_layer_test.rs
@@ -140,6 +140,7 @@ fn test_transformer_layer_config() {
has_sinks: false,
has_qkv_biases: false,
has_out_biases: false,
+ has_gate: false,
partial_rope_dim: None,
}),
mlp_config: MLPConfig::Dense(mlp::DenseMLPConfig {
diff --git a/crates/uzu/tests/unit/kernel/attention/mod.rs b/crates/uzu/tests/unit/kernel/attention/mod.rs
index fc725184d..7f8328e4b 100644
--- a/crates/uzu/tests/unit/kernel/attention/mod.rs
+++ b/crates/uzu/tests/unit/kernel/attention/mod.rs
@@ -1,3 +1,4 @@
mod attention_gemm_test;
mod attention_single_pass_test;
mod attention_update_kv_cache_test;
+mod sigmoid_gate_test;
diff --git a/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs b/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs
new file mode 100644
index 000000000..cde59daf1
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs
@@ -0,0 +1,132 @@
+use std::{fmt::Debug, ops::DerefMut};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayContextExt, ArrayElement,
+ backends::{
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels, kernel::SigmoidGateKernel,
+ },
+ cpu::Cpu,
+ },
+};
+
+struct Config {
+ num_heads: u32,
+ head_dim: u32,
+ suffix_length: u32,
+}
+
+fn get_output<T: ArrayElement + Float, B: Backend>(
+ gate_data: &[T],
+ output_data: &[T],
+ config: &Config,
+) -> Vec<T> {
+ let size = gate_data.len();
+ let context = B::Context::new().expect("Failed to create Context");
+ let kernel = <<B as Backend>::Kernels as Kernels>::SigmoidGateKernel::new(&context, T::data_type())
+ .expect("Failed to create SigmoidGateKernel");
+
+ let gate_array = context.create_array_from(&[size], &gate_data.to_vec().into_boxed_slice(), "gate");
+ let output_array = context.create_array_from(&[size], &output_data.to_vec().into_boxed_slice(), "output");
+
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let total_elements = config.suffix_length * config.num_heads * config.head_dim;
+ kernel.encode(
+ &*gate_array.buffer().borrow(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ total_elements,
+ &mut command_buffer,
+ );
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+
+ output_array.as_slice().to_vec()
+}
+
+fn run_test<T: ArrayElement + Float + Debug>(config: &Config) {
+ let size = (config.suffix_length * config.num_heads * config.head_dim) as usize;
+
+ let gate_f32: Vec<f32> = (0..size).map(|i| (i as f32) * 0.1 - 2.0).collect();
+ let output_f32: Vec<f32> = (0..size).map(|i| (i as f32) * 0.05 + 0.5).collect();
+
+ let gate_data: Vec<T> = gate_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+ let output_data: Vec<T> = output_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ let expected = get_output::<T, Cpu>(&gate_data, &output_data, config);
+
+ let rtol = if std::mem::size_of::<T>() <= 2 {
+ 0.01
+ } else {
+ 1e-5
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let result = get_output::<T, B>(&gate_data, &output_data, config);
+
+ for (i, (got, exp)) in result.iter().zip(expected.iter()).enumerate() {
+ let got_f32 = got.to_f32().unwrap();
+ let exp_f32 = exp.to_f32().unwrap();
+ let diff = (got_f32 - exp_f32).abs();
+ let tol = rtol * exp_f32.abs().max(1.0);
+ assert!(
+ diff < tol,
+ "Backend {}: mismatch at index {}: got {} expected {} (diff {}, tol {})",
+ std::any::type_name::<B>(),
+ i,
+ got_f32,
+ exp_f32,
+ diff,
+ tol,
+ );
+ }
+ });
+}
+
+#[test]
+fn test_sigmoid_gate_f32() {
+ run_test::<f32>(&Config {
+ num_heads: 8,
+ head_dim: 64,
+ suffix_length: 4,
+ });
+ run_test::<f32>(&Config {
+ num_heads: 16,
+ head_dim: 256,
+ suffix_length: 1,
+ });
+ run_test::<f32>(&Config {
+ num_heads: 2,
+ head_dim: 64,
+ suffix_length: 8,
+ });
+}
+
+#[test]
+fn test_sigmoid_gate_f16() {
+ run_test::<f16>(&Config {
+ num_heads: 8,
+ head_dim: 64,
+ suffix_length: 4,
+ });
+ run_test::<f16>(&Config {
+ num_heads: 16,
+ head_dim: 256,
+ suffix_length: 1,
+ });
+}
+
+#[test]
+fn test_sigmoid_gate_bf16() {
+ run_test::<bf16>(&Config {
+ num_heads: 8,
+ head_dim: 64,
+ suffix_length: 4,
+ });
+ run_test::<bf16>(&Config {
+ num_heads: 16,
+ head_dim: 256,
+ suffix_length: 1,
+ });
+}