A high-performance inference engine for AI models
Support Gated Attention (#278)
CCYehe80807a20ee0
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu/src/backends/cpu/kernel/attention/mod.rsindex 24f00ec54..4a6321bee 100644--- a/crates/uzu/src/backends/cpu/kernel/attention/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/attention/mod.rs@@ -2,3 +2,4 @@ pub mod attention_gemm;pub mod attention_single_pass;pub mod attention_two_pass;pub mod attention_update_kv_cache;+pub mod sigmoid_gate;diff --git a/crates/uzu/src/backends/cpu/kernel/attention/sigmoid_gate.rs b/crates/uzu/src/backends/cpu/kernel/attention/sigmoid_gate.rsnew file mode 100644index 000000000..6f5c8e49b--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/attention/sigmoid_gate.rs@@ -0,0 +1,22 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(SigmoidGate)]+#[variants(T, f32, f16, bf16)]+pub fn sigmoid_gate<T: ArrayElement + Float>(+ gate: *const T,+ output: *mut T,+ total_elements: u32,+) {+ for idx in 0..total_elements as usize {+ unsafe {+ let g = (*gate.add(idx)).to_f32().unwrap();+ let sigmoid = 1.0f32 / (1.0f32 + (-g).exp());+ let out = (*output.add(idx)).to_f32().unwrap();+ *output.add(idx) = T::from(out * sigmoid).unwrap();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/attention/sigmoid_gate.metal b/crates/uzu/src/backends/metal/kernel/attention/sigmoid_gate.metalnew file mode 100644index 000000000..876609196--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/attention/sigmoid_gate.metal@@ -0,0 +1,16 @@+#include <metal_stdlib>+#include "../common/dsl.h"++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(SigmoidGate)(+ const device T* gate,+ device T* output,+ const constant uint& total_elements,+ const uint tid AXIS(total_elements, 256)+) {+ if (tid >= total_elements)+ return;+ float sigmoid = 1.0f / (1.0f + exp(-float(gate[tid])));+ output[tid] = T(float(output[tid]) * sigmoid);+}diff --git a/crates/uzu/src/config/attention.rs b/crates/uzu/src/config/attention.rsindex d74cdb3fe..a24b72bb4 100644--- a/crates/uzu/src/config/attention.rs+++ b/crates/uzu/src/config/attention.rs@@ -23,6 +23,8 @@ pub struct AttentionConfig {pub has_qkv_biases: bool,pub has_out_biases: bool,#[serde(default)]+ pub has_gate: bool,+ #[serde(default)]pub partial_rope_dim: Option<usize>,}diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rsindex aabd892f6..c5dad382e 100644--- a/crates/uzu/src/encodable_block/attention.rs+++ b/crates/uzu/src/encodable_block/attention.rs@@ -11,6 +11,7 @@ use crate::{Backend, CommandBuffer, Kernels,kernel::{AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel,+ SigmoidGateKernel,attention::{AttentionGemmArguments, AttentionGemmBlock},},},@@ -40,6 +41,7 @@ pub struct Attention<B: Backend> {two_pass_2_kernels: HashMap<u32, <B::Kernels as Kernels>::AttentionTwoPass2Kernel>,update_kv_cache_kernel: <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel,update_kv_cache_inplace_kernel: <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel,+ gate_kernel: Option<<B::Kernels as Kernels>::SigmoidGateKernel>,gemm_block: AttentionGemmBlock<B>,layer_index: usize,attention_scale: Option<f32>,@@ -57,6 +59,7 @@ impl<B: Backend> Attention<B> {has_sinks: bool,is_causal: bool,sliding_window_size: Option<usize>,+ has_gate: bool,) -> Result<Self, B::Error> {let mut single_pass_kernels = HashMap::new();let mut two_pass_1_kernels = HashMap::new();@@ -90,6 +93,11 @@ impl<B: Backend> Attention<B> {<B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, false)?;let update_kv_cache_inplace_kernel =<B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, true)?;+ let gate_kernel = if has_gate {+ Some(<B::Kernels as Kernels>::SigmoidGateKernel::new(context, data_type)?)+ } else {+ None+ };let gemm_block = AttentionGemmBlock::new(data_type);Ok(Self {@@ -98,6 +106,7 @@ impl<B: Backend> Attention<B> {two_pass_2_kernels,update_kv_cache_kernel,update_kv_cache_inplace_kernel,+ gate_kernel,gemm_block,layer_index,attention_scale,@@ -198,6 +207,7 @@ impl<B: Backend> Attention<B> {let qkv_binding = state.arrays(&[ArrayId::QKV]);let attention_bias_binding = state.hashmaps(&[HashMapId::AttentionBias]);let attention_output_binding = state.arrays(&[ArrayId::AttentionOutput]);+ let gate_binding = self.gate_kernel.as_ref().map(|_| state.arrays(&[ArrayId::Gate]));let mask_kv_seq_stride = 1;let mask_q_seq_stride = sequence_length as i32;@@ -436,6 +446,19 @@ impl<B: Backend> Attention<B> {},}+ if let Some(gate_kernel) = &self.gate_kernel {+ let gate_array = gate_binding.as_ref().unwrap()[0].borrow();+ let gate_buf_rc = gate_array.buffer();+ let gate_buf_borrow = gate_buf_rc.borrow();+ let total_elements = (suffix_length * num_heads * head_dim) as u32;+ gate_kernel.encode(+ gate_buf_borrow.deref(),+ attention_output_buf_borrow.deref_mut(),+ total_elements,+ command_buffer,+ );+ }+Ok(())}}diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rsindex 658d6de62..5ead1cf31 100644--- a/crates/uzu/src/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/encodable_block/classifier_layer.rs@@ -191,6 +191,7 @@ impl<B: Backend> ClassifierLayer<B> {attention_config.has_sinks,false,attention_config.sliding_window_size,+ false,).expect("Failed to create attention kernel");diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex 0de13c75a..f785afb80 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -84,6 +84,24 @@ impl<B: Backend> LayerExecutables<B> {).expect("Failed to create qkv projection");+ let gate_projection = if attention_config.has_gate {+ Some(+ <dyn Linear<B>>::new(+ &attention_config.qkv_projection_config,+ false,+ model_dim,+ [num_heads * head_dim],+ ctx,+ &decoder_layer_loader.subtree("mixer.gate_projection").unwrap(),+ ArrayId::Main,+ ArrayId::Gate,+ )+ .expect("Failed to create gate projection"),+ )+ } else {+ None+ };+let qk_norm =if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {match QKNorm::new(@@ -124,11 +142,13 @@ impl<B: Backend> LayerExecutables<B> {attention_config.has_sinks,attention_config.is_causal.unwrap_or(true),attention_config.sliding_window_size,+ attention_config.has_gate,).expect("Failed to create AttentionWrapper kernel");MixerExecutables::Attention {qkv_projection,+ gate_projection,qk_norm,rope: rope_block,attention,@@ -264,12 +284,16 @@ impl<B: Backend> LayerExecutables<B> {match &self.mixer {MixerExecutables::Attention {qkv_projection,+ gate_projection,qk_norm,rope,attention,out_projection,} => {qkv_projection.encode(state, command_buffer)?;+ if let Some(gate_proj) = gate_projection {+ gate_proj.encode(state, command_buffer)?;+ }if let Some(norm) = qk_norm {norm.encode(state, command_buffer)?;}diff --git a/crates/uzu/src/encodable_block/layer/mixer.rs b/crates/uzu/src/encodable_block/layer/mixer.rsindex e709d25b0..749ebbc0b 100644--- a/crates/uzu/src/encodable_block/layer/mixer.rs+++ b/crates/uzu/src/encodable_block/layer/mixer.rs@@ -11,6 +11,7 @@ use crate::{pub(crate) enum MixerExecutables<B: Backend> {Attention {qkv_projection: Box<dyn Linear<B>>,+ gate_projection: Option<Box<dyn Linear<B>>>,qk_norm: Option<QKNorm<B>>,rope: Rc<Rope<B>>,attention: Attention<B>,diff --git a/crates/uzu/src/forward_pass/model_shape.rs b/crates/uzu/src/forward_pass/model_shape.rsindex 634ce39fe..da82e1408 100644--- a/crates/uzu/src/forward_pass/model_shape.rs+++ b/crates/uzu/src/forward_pass/model_shape.rs@@ -26,6 +26,7 @@ pub struct ModelShape {max_mamba_state_dim: usize,max_mamba_kernel_size: usize,max_rope_dim: usize,+ has_gate: bool,}impl ModelShape {@@ -75,6 +76,7 @@ impl ModelShape {}}let mut max_rope_dim = 0usize;+ let mut has_gate = false;let all_layer_configs = decoder_config.layer_configs@@ -86,6 +88,7 @@ impl ModelShape {if let MixerConfig::Attention(attn) = &layer_config.mixer_config {let hd = attn.head_dim.unwrap_or(decoder_config.head_dim);max_rope_dim = max_rope_dim.max(attn.partial_rope_dim.unwrap_or(hd));+ has_gate = has_gate || attn.has_gate;}}@@ -116,6 +119,7 @@ impl ModelShape {max_mamba_state_dim,max_mamba_kernel_size,max_rope_dim,+ has_gate,}}@@ -208,6 +212,13 @@ impl ModelShape {[suffix_length, self.num_heads * self.head_dim]}+ pub fn gate_shape(+ &self,+ suffix_length: usize,+ ) -> Option<[usize; 2]> {+ self.has_gate.then(|| [suffix_length, self.num_heads * self.head_dim])+ }+pub fn rotated_queries_shape(&self,suffix_length: usize,diff --git a/crates/uzu/src/forward_pass/scratch_buffers.rs b/crates/uzu/src/forward_pass/scratch_buffers.rsindex a29fd934a..b8d690e33 100644--- a/crates/uzu/src/forward_pass/scratch_buffers.rs+++ b/crates/uzu/src/forward_pass/scratch_buffers.rs@@ -26,6 +26,7 @@ pub struct ScratchBuffers<B: Backend> {pub main: ArrayCell<B>,pub shortcut: ArrayCell<B>,pub qkv: ArrayCell<B>,+ pub gate: Option<ArrayCell<B>>,pub attention_output: ArrayCell<B>,pub mlp_fused_up: ArrayCell<B>,pub mlp_hidden: ArrayCell<B>,@@ -128,6 +129,7 @@ impl<B: Backend> ScratchBuffers<B> {main: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "main"),shortcut: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "shortcut"),qkv: alloc(&model_shape.qkv_shape(max_suffix_len), act_ty, "qkv"),+ gate: model_shape.gate_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "gate")),attention_output: alloc(&model_shape.attention_output_shape(max_suffix_len), act_ty, "attention_output"),mlp_fused_up: alloc(&model_shape.mlp_fused_up_shape(max_suffix_len), act_ty, "mlp_fused_up"),mlp_hidden: alloc(&model_shape.mlp_hidden_shape(max_suffix_len), act_ty, "mlp_hidden"),diff --git a/crates/uzu/src/forward_pass/state/array_id.rs b/crates/uzu/src/forward_pass/state/array_id.rsindex b0b6af703..09afec89b 100644--- a/crates/uzu/src/forward_pass/state/array_id.rs+++ b/crates/uzu/src/forward_pass/state/array_id.rs@@ -14,6 +14,7 @@ pub enum ArrayId {Main,Shortcut,QKV,+ Gate,AttentionOutput,MlpFusedUp,MlpHidden,diff --git a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs b/crates/uzu/src/forward_pass/state/common_aux_buffers.rsindex 7c7b38beb..58af10f2f 100644--- a/crates/uzu/src/forward_pass/state/common_aux_buffers.rs+++ b/crates/uzu/src/forward_pass/state/common_aux_buffers.rs@@ -9,6 +9,7 @@ pub struct CommonAuxBuffers<B: Backend> {pub main: ArrayCell<B>,pub shortcut: ArrayCell<B>,pub qkv: ArrayCell<B>,+ pub gate: Option<ArrayCell<B>>,pub attention_output: ArrayCell<B>,pub mlp_fused_up: ArrayCell<B>,pub mlp_hidden: ArrayCell<B>,@@ -31,6 +32,7 @@ impl<B: Backend> CommonAuxBuffers<B> {main: scratch.main.view(&model_shape.main_shape(suffix_length)),shortcut: scratch.shortcut.view(&model_shape.main_shape(suffix_length)),qkv: scratch.qkv.view(&model_shape.qkv_shape(suffix_length)),+ gate: scratch.gate.as_ref().zip(model_shape.gate_shape(suffix_length)).map(|(buf, shape)| buf.view(&shape)),attention_output: scratch.attention_output.view(&model_shape.attention_output_shape(suffix_length)),mlp_fused_up: scratch.mlp_fused_up.view(&model_shape.mlp_fused_up_shape(suffix_length)),mlp_hidden: scratch.mlp_hidden.view(&model_shape.mlp_hidden_shape(suffix_length)),diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rsindex 58ac194b3..8112db011 100644--- a/crates/uzu/src/forward_pass/state/state.rs+++ b/crates/uzu/src/forward_pass/state/state.rs@@ -485,6 +485,7 @@ impl<B: Backend> ForwardPassState<B> {ArrayId::Main => self.common_aux.main.clone(),ArrayId::Shortcut => self.common_aux.shortcut.clone(),ArrayId::QKV => self.common_aux.qkv.clone(),+ ArrayId::Gate => self.common_aux.gate.clone().expect("Gate buffer not available"),ArrayId::AttentionOutput => self.common_aux.attention_output.clone(),ArrayId::MlpFusedUp => self.common_aux.mlp_fused_up.clone(),ArrayId::MlpHidden => self.common_aux.mlp_hidden.clone(),diff --git a/crates/uzu/tests/unit/config/attention_test.rs b/crates/uzu/tests/unit/config/attention_test.rsindex 6eb9b9016..6a7f86d3f 100644--- a/crates/uzu/tests/unit/config/attention_test.rs+++ b/crates/uzu/tests/unit/config/attention_test.rs@@ -76,6 +76,7 @@ fn test_attention_config() {has_sinks: false,has_qkv_biases: false,has_out_biases: false,+ has_gate: false,partial_rope_dim: None,};diff --git a/crates/uzu/tests/unit/config/decoder_layer_test.rs b/crates/uzu/tests/unit/config/decoder_layer_test.rsindex f437d261c..adc7e52d4 100644--- a/crates/uzu/tests/unit/config/decoder_layer_test.rs+++ b/crates/uzu/tests/unit/config/decoder_layer_test.rs@@ -132,6 +132,7 @@ fn test_decoder_layer_config() {has_sinks: false,has_qkv_biases: false,has_out_biases: false,+ has_gate: false,partial_rope_dim: None,}),mlp_config: MLPConfig::Dense(mlp::DenseMLPConfig {diff --git a/crates/uzu/tests/unit/config/decoder_test.rs b/crates/uzu/tests/unit/config/decoder_test.rsindex 4c5473b97..b64e56a5d 100644--- a/crates/uzu/tests/unit/config/decoder_test.rs+++ b/crates/uzu/tests/unit/config/decoder_test.rs@@ -195,6 +195,7 @@ fn test_decoder_config() {has_sinks: false,has_qkv_biases: false,has_out_biases: false,+ has_gate: false,partial_rope_dim: None,}),mlp_config: MLPConfig::Dense(DenseMLPConfig {diff --git a/crates/uzu/tests/unit/config/transformer_layer_test.rs b/crates/uzu/tests/unit/config/transformer_layer_test.rsindex 5889e1f5d..117312912 100644--- a/crates/uzu/tests/unit/config/transformer_layer_test.rs+++ b/crates/uzu/tests/unit/config/transformer_layer_test.rs@@ -140,6 +140,7 @@ fn test_transformer_layer_config() {has_sinks: false,has_qkv_biases: false,has_out_biases: false,+ has_gate: false,partial_rope_dim: None,}),mlp_config: MLPConfig::Dense(mlp::DenseMLPConfig {diff --git a/crates/uzu/tests/unit/kernel/attention/mod.rs b/crates/uzu/tests/unit/kernel/attention/mod.rsindex fc725184d..7f8328e4b 100644--- a/crates/uzu/tests/unit/kernel/attention/mod.rs+++ b/crates/uzu/tests/unit/kernel/attention/mod.rs@@ -1,3 +1,4 @@mod attention_gemm_test;mod attention_single_pass_test;mod attention_update_kv_cache_test;+mod sigmoid_gate_test;diff --git a/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs b/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rsnew file mode 100644index 000000000..cde59daf1--- /dev/null+++ b/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs@@ -0,0 +1,132 @@+use std::{fmt::Debug, ops::DerefMut};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayContextExt, ArrayElement,+ backends::{+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels, kernel::SigmoidGateKernel,+ },+ cpu::Cpu,+ },+};++struct Config {+ num_heads: u32,+ head_dim: u32,+ suffix_length: u32,+}++fn get_output<T: ArrayElement + Float, B: Backend>(+ gate_data: &[T],+ output_data: &[T],+ config: &Config,+) -> Vec<T> {+ let size = gate_data.len();+ let context = B::Context::new().expect("Failed to create Context");+ let kernel = <<B as Backend>::Kernels as Kernels>::SigmoidGateKernel::new(&context, T::data_type())+ .expect("Failed to create SigmoidGateKernel");++ let gate_array = context.create_array_from(&[size], &gate_data.to_vec().into_boxed_slice(), "gate");+ let output_array = context.create_array_from(&[size], &output_data.to_vec().into_boxed_slice(), "output");++ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let total_elements = config.suffix_length * config.num_heads * config.head_dim;+ kernel.encode(+ &*gate_array.buffer().borrow(),+ output_array.buffer().borrow_mut().deref_mut(),+ total_elements,+ &mut command_buffer,+ );+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();++ output_array.as_slice().to_vec()+}++fn run_test<T: ArrayElement + Float + Debug>(config: &Config) {+ let size = (config.suffix_length * config.num_heads * config.head_dim) as usize;++ let gate_f32: Vec<f32> = (0..size).map(|i| (i as f32) * 0.1 - 2.0).collect();+ let output_f32: Vec<f32> = (0..size).map(|i| (i as f32) * 0.05 + 0.5).collect();++ let gate_data: Vec<T> = gate_f32.iter().map(|&v| T::from(v).unwrap()).collect();+ let output_data: Vec<T> = output_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ let expected = get_output::<T, Cpu>(&gate_data, &output_data, config);++ let rtol = if std::mem::size_of::<T>() <= 2 {+ 0.01+ } else {+ 1e-5+ };++ for_each_non_cpu_backend!(|B| {+ let result = get_output::<T, B>(&gate_data, &output_data, config);++ for (i, (got, exp)) in result.iter().zip(expected.iter()).enumerate() {+ let got_f32 = got.to_f32().unwrap();+ let exp_f32 = exp.to_f32().unwrap();+ let diff = (got_f32 - exp_f32).abs();+ let tol = rtol * exp_f32.abs().max(1.0);+ assert!(+ diff < tol,+ "Backend {}: mismatch at index {}: got {} expected {} (diff {}, tol {})",+ std::any::type_name::<B>(),+ i,+ got_f32,+ exp_f32,+ diff,+ tol,+ );+ }+ });+}++#[test]+fn test_sigmoid_gate_f32() {+ run_test::<f32>(&Config {+ num_heads: 8,+ head_dim: 64,+ suffix_length: 4,+ });+ run_test::<f32>(&Config {+ num_heads: 16,+ head_dim: 256,+ suffix_length: 1,+ });+ run_test::<f32>(&Config {+ num_heads: 2,+ head_dim: 64,+ suffix_length: 8,+ });+}++#[test]+fn test_sigmoid_gate_f16() {+ run_test::<f16>(&Config {+ num_heads: 8,+ head_dim: 64,+ suffix_length: 4,+ });+ run_test::<f16>(&Config {+ num_heads: 16,+ head_dim: 256,+ suffix_length: 1,+ });+}++#[test]+fn test_sigmoid_gate_bf16() {+ run_test::<bf16>(&Config {+ num_heads: 8,+ head_dim: 64,+ suffix_length: 4,+ });+ run_test::<bf16>(&Config {+ num_heads: 16,+ head_dim: 256,+ suffix_length: 1,+ });+}