A high-performance inference engine for AI models
Fuse sigmoid gate into Attention (#930)
CCYehf9d747746e8a
diff --git a/crates/uzu-engine/src/backends/common/kernel/attention.rs b/crates/uzu-engine/src/backends/common/kernel/attention.rsindex 1ca7225d9..823389935 100644--- a/crates/uzu-engine/src/backends/common/kernel/attention.rs+++ b/crates/uzu-engine/src/backends/common/kernel/attention.rs@@ -21,6 +21,7 @@ pub trait AttentionKernel: Sized + Send + Sync {impl BufferRef<Backend = Self::Backend>,impl BufferRef<Backend = Self::Backend>,impl BufferRef<Backend = Self::Backend>,+ impl BufferRef<Backend = Self::Backend>,>,command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,) -> Result<<Self::Backend as Backend>::ScratchBuffer, <Self::Backend as Backend>::Error>;@@ -32,6 +33,7 @@ pub struct AttentionKernelConfig {pub num_groups: u32,pub num_q_heads: u32,pub has_sinks: bool,+ pub gate_row_stride: Option<u32>,pub is_kv_cache_ring: bool,pub is_causal: bool,pub sliding_window_size: Option<u32>,@@ -46,6 +48,7 @@ pub struct AttentionArguments<TT: BufferRef<Backend = B>,KT: BufferRef<Backend = B>,VT: BufferRef<Backend = B>,+ GT: BufferRef<Backend = B>,> {pub queries: QT,pub keys: KT,@@ -54,4 +57,5 @@ pub struct AttentionArguments<pub trie: Option<TT>,pub sinks: Option<&'a B::GlobalBuffer>,pub cache: KVCacheView,+ pub gate: Option<GT>,}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rsindex 18ad128b4..21b07f2d1 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs@@ -12,7 +12,9 @@ mod mask;use crate::backends::{common::{BufferRef, CommandBufferEncoding,- kernel::{AttentionArguments, AttentionKernel, AttentionKernelConfig, AttentionSinglePassKernel},+ kernel::{+ AttentionArguments, AttentionKernel, AttentionKernelConfig, AttentionSinglePassKernel, SigmoidGateKernel,+ },},cpu::{Cpu, command_buffer::CpuCommandBufferEncoding, context::CpuContext, error::CpuError},};@@ -42,6 +44,7 @@ impl AttentionKernel for AttentionCpuKernel {impl BufferRef<Backend = Cpu>,impl BufferRef<Backend = Cpu>,impl BufferRef<Backend = Cpu>,+ impl BufferRef<Backend = Cpu>,>,command_buffer: &mut CpuCommandBufferEncoding,) -> Result<<Cpu as Backend>::ScratchBuffer, CpuError> {@@ -83,6 +86,17 @@ impl AttentionKernel for AttentionCpuKernel {command_buffer,);command_buffer.pop_debug_group();+ if let (Some(gate), Some(gate_row_stride)) = (arguments.gate, config.gate_row_stride) {+ <sigmoid_gate::SigmoidGateCpuKernel as SigmoidGateKernel>::new(command_buffer.context(), config.data_type)?+ .encode(+ gate,+ &mut output,+ config.num_q_heads * config.head_dim,+ arguments.suffix_length,+ gate_row_stride,+ command_buffer,+ );+ }Ok(output)}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gate.h b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gate.hnew file mode 100644index 000000000..8222b4242--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gate.h@@ -0,0 +1,9 @@+#pragma once++#include <metal_stdlib>++template <typename T>+METAL_FUNC T attention_apply_gate(const T result, const T gate) {+ const float sigmoid = 1.0f / (1.0f + metal::exp(-float(gate)));+ return T(float(result) * sigmoid);+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metalindex 3594bcbce..a34d453ef 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -5,6 +5,7 @@#include "../common/thread_context.h"#include "../generated/ring.h"#include "../generated/trie.h"+#include "attention_gate.h"#include "mask.h"#define TOTAL_BLOCKS_COUNT 32@@ -146,8 +147,11 @@ KERNEL(AttentionTwoPass2)(const device float* sums,const device float* maxs,device T* out,+ const device T* gate OPTIONAL(has_gate),const constant uint& num_heads,const constant uint& suffix_length,+ const constant uint& gate_row_stride OPTIONAL(has_gate),+ const bool has_gate SPECIALIZE,const ThreadContext thread_context,const uint head_idx GROUPS(num_heads),const uint q_seq_idx GROUPS(suffix_length),@@ -197,7 +201,12 @@ KERNEL(AttentionTwoPass2)(if (thread_context.simd_lane_id == 0) {out += o_offset * HEAD_DIM + thread_context.simdgroup_index * elements_per_thread;for (uint i = 0; i < elements_per_thread; i++) {- out[i] = static_cast<T>(o[i] / sum_exp_score);+ T result = static_cast<T>(o[i] / sum_exp_score);+ if (has_gate) {+ const uint channel = thread_context.simdgroup_index * elements_per_thread + i;+ result = attention_apply_gate(result, gate[q_seq_idx * gate_row_stride + head_idx * HEAD_DIM + channel]);+ }+ out[i] = result;}}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rsindex bf19f9ea0..b8a0b01c9 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs@@ -89,6 +89,7 @@ impl AttentionFallback {impl BufferRef<Backend = Metal>,impl BufferRef<Backend = Metal>,impl BufferRef<Backend = Metal>,+ impl BufferRef<Backend = Metal>,>,command_buffer: &mut MetalCommandBufferEncoding,) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rsindex d8e93eb1a..957e25657 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs@@ -120,6 +120,7 @@ impl AttentionGemm {impl BufferRef<Backend = Metal>,impl BufferRef<Backend = Metal>,impl BufferRef<Backend = Metal>,+ impl BufferRef<Backend = Metal>,>,command_buffer: &mut MetalCommandBufferEncoding,) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_gemm_grouped.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_gemm_grouped.metalindex 8478518d5..2aeb9fbfd 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_gemm_grouped.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_gemm_grouped.metal@@ -7,6 +7,7 @@#include "../../matmul/common/mxu_fragment/ops.h"#include "../../generated/attention.h"#include "../../generated/trie.h"+#include "../attention_gate.h"using namespace metal;using namespace uzu::matmul;@@ -402,6 +403,7 @@ KERNEL(AttentionGemmGroupedCombine)(const device float* partial_maxs,const device float* partial_sums,device T* o,+ const device T* gate OPTIONAL(has_gate),const constant AttnParams& params,const constant uint& grouped_rows,const constant uint& block_rows,@@ -409,6 +411,8 @@ KERNEL(AttentionGemmGroupedCombine)(const constant uint& kv_heads,const constant uint& batch_count,const constant uint& num_splits,+ const constant uint& gate_row_stride OPTIONAL(has_gate),+ const bool has_gate SPECIALIZE,const uint grouped_row_idx GROUPS(grouped_rows),const uint kv_head_idx GROUPS(kv_heads),const uint batch_idx GROUPS(batch_count),@@ -446,7 +450,14 @@ KERNEL(AttentionGemmGroupedCombine)(const uint head_in_group = grouped_row_idx / params.q_len;const uint query_position = grouped_row_idx - head_in_group * params.q_len;const uint q_head_idx = kv_head_idx * params.gqa_factor + head_in_group;+ T result = static_cast<T>(denominator > 0.0f ? numerator / denominator : 0.0f);+ if (has_gate) {+ result = attention_apply_gate(+ result,+ gate[(batch_idx * params.q_len + query_position) * gate_row_stride + q_head_idx * BD + column]+ );+ }o += size_t(batch_idx) * params.o_strides[0] + size_t(q_head_idx) * params.o_strides[1] +size_t(query_position) * params.o_strides[2] + column;- o[0] = static_cast<T>(denominator > 0.0f ? numerator / denominator : 0.0f);+ o[0] = result;}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rsindex bd3697228..a26465005 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs@@ -36,6 +36,7 @@ struct AttentionGemmGroupedMetal {num_groups: u32,num_q_heads: u32,scale: Option<f32>,+ gate_row_stride: Option<u32>,block_rows: u32,mask: MaskKind,}@@ -47,6 +48,7 @@ impl AttentionGemmGroupedMetal {num_groups: u32,num_q_heads: u32,scale: Option<f32>,+ gate_row_stride: Option<u32>,mask: MaskKind,) -> Result<Self, MetalError> {assert!(matches!(head_dim, 128 | 256), "head_dim must be 128 or 256");@@ -65,7 +67,8 @@ impl AttentionGemmGroupedMetal {mask.is_trie(),)};- let combine = AttentionGemmGroupedCombineMetalKernel::new(context, DataType::BF16, head_dim)?;+ let combine =+ AttentionGemmGroupedCombineMetalKernel::new(context, DataType::BF16, head_dim, gate_row_stride.is_some())?;Ok(Self {kernel: new_kernel(false)?,split_kernel: new_kernel(true)?,@@ -74,6 +77,7 @@ impl AttentionGemmGroupedMetal {num_groups,num_q_heads,scale,+ gate_row_stride,block_rows,mask,})@@ -86,12 +90,13 @@ impl AttentionGemmGroupedMetal {values: impl BufferRef<Backend = Metal>,output: impl BufferMut<Backend = Metal>,trie: Option<impl BufferRef<Backend = Metal>>,+ gate: Option<impl BufferRef<Backend = Metal>>,suffix_length: u32,kv_length: u32,q_replicas: u32,num_splits: u32,command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<(), MetalError> {+ ) -> Result<bool, MetalError> {let params = self.params(suffix_length, kv_length);let grouped_rows = self.num_q_heads / self.num_groups * suffix_length;let m_tiles = grouped_rows.div_ceil(self.block_rows);@@ -116,7 +121,7 @@ impl AttentionGemmGroupedMetal {1,command_buffer,);- return Ok(());+ return Ok(false);}let partial_rows = q_replicas * self.num_groups * m_tiles * num_splits * self.block_rows;@@ -146,6 +151,7 @@ impl AttentionGemmGroupedMetal {&partial_maxs,&partial_sums,output,+ gate,params,grouped_rows,self.block_rows,@@ -153,9 +159,10 @@ impl AttentionGemmGroupedMetal {self.num_groups,q_replicas,num_splits,+ self.gate_row_stride,command_buffer,);- Ok(())+ Ok(gate.is_some())}fn params(@@ -194,6 +201,7 @@ pub struct AttentionGemmGrouped {num_groups: u32,num_q_heads: u32,scale: Option<f32>,+ gate_row_stride: Option<u32>,}impl AttentionGemmGrouped {@@ -215,6 +223,7 @@ impl AttentionGemmGrouped {self.num_groups,self.num_q_heads,self.scale,+ self.gate_row_stride,mask,)?);}@@ -236,6 +245,7 @@ impl AttentionGemmGrouped {num_groups: config.num_groups,num_q_heads: config.num_q_heads,scale: config.scale,+ gate_row_stride: config.gate_row_stride,}}@@ -258,9 +268,10 @@ impl AttentionGemmGrouped {impl BufferRef<Backend = Metal>,impl BufferRef<Backend = Metal>,impl BufferRef<Backend = Metal>,+ impl BufferRef<Backend = Metal>,>,command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {+ ) -> Result<(<Metal as Backend>::ScratchBuffer, bool), MetalError> {let suffix_length = arguments.suffix_length;assert!(arguments.cache.ring_params().is_none(), "ring KV cache is unsupported");assert!(arguments.sinks.is_none(), "attention sinks are unsupported");@@ -282,18 +293,19 @@ impl AttentionGemmGrouped {kv_length,gpu_core_count,);- core.encode(+ let gate_applied = core.encode(arguments.queries,arguments.keys,arguments.values,&mut output,arguments.trie,+ arguments.gate,suffix_length,kv_length,1,num_splits,command_buffer,)?;- Ok(output)+ Ok((output, gate_applied))}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rsindex 5bbafd69e..d16465c6f 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs@@ -9,9 +9,12 @@ mod two_pass;use crate::backends::{common::{BufferRef, CommandBufferEncoding,- kernel::{AttentionArguments, AttentionKernel, AttentionKernelConfig},+ kernel::{AttentionArguments, AttentionKernel, AttentionKernelConfig, SigmoidGateKernel},+ },+ metal::{+ Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError,+ kernel::SigmoidGateMetalKernel,},- metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError},};#[derive(Clone, Copy, Debug, PartialEq, Eq)]@@ -48,8 +51,8 @@ const D256_SINGLE_PASS_KV_THRESHOLD: u32 = 150_000;const D128_SHORT_KV_GEMM_THRESHOLD: u32 = 2_048;pub struct AttentionMetalKernel {- head_dim: u32,- is_causal: bool,+ config: AttentionKernelConfig,+ gate: Option<(SigmoidGateMetalKernel, u32)>,grouped: Option<gemm_grouped::AttentionGemmGrouped>,gemm: Option<gemm::AttentionGemm>,fallback: Option<fallback::AttentionFallback>,@@ -70,9 +73,13 @@ impl AttentionKernel for AttentionMetalKernel {let fallback = fallback::AttentionFallback::is_supported(&config).then(|| fallback::AttentionFallback::new(&config, context)).transpose()?;+ let gate = config+ .gate_row_stride+ .map(|stride| SigmoidGateMetalKernel::new(context, config.data_type).map(|kernel| (kernel, stride)))+ .transpose()?;Ok(Self {- head_dim: config.head_dim,- is_causal: config.is_causal,+ config,+ gate,grouped,gemm,fallback,@@ -90,11 +97,25 @@ impl AttentionKernel for AttentionMetalKernel {impl BufferRef<Backend = Metal>,impl BufferRef<Backend = Metal>,impl BufferRef<Backend = Metal>,+ impl BufferRef<Backend = Metal>,>,command_buffer: &mut MetalCommandBufferEncoding,) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {command_buffer.push_debug_group("attention core");- let result = self.encode_impl(arguments, command_buffer);+ let (gate, suffix_length) = (arguments.gate, arguments.suffix_length);+ let result = self.encode_impl(arguments, command_buffer).map(|(mut output, gate_applied)| {+ if let (Some(gate), Some((kernel, stride)), false) = (gate, &self.gate, gate_applied) {+ kernel.encode(+ gate,+ &mut output,+ self.config.num_q_heads * self.config.head_dim,+ suffix_length,+ *stride,+ command_buffer,+ );+ }+ output+ });command_buffer.pop_debug_group();result}@@ -110,14 +131,16 @@ impl AttentionMetalKernel {impl BufferRef<Backend = Metal>,impl BufferRef<Backend = Metal>,impl BufferRef<Backend = Metal>,+ impl BufferRef<Backend = Metal>,>,command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {+ ) -> Result<(<Metal as Backend>::ScratchBuffer, bool), MetalError> {let suffix_length = arguments.suffix_length;+ let gate_applied_by_two_pass = arguments.gate.is_some();let kv_length = arguments.cache.prefix_len() + suffix_length;let is_trie = arguments.trie.is_some();let is_ring = arguments.cache.ring_params().is_some();- let mask = MaskKind::for_attention(self.is_causal, is_trie);+ let mask = MaskKind::for_attention(self.config.is_causal, is_trie);if !is_ring&& let Some(mask) = mask@@ -129,31 +152,34 @@ impl AttentionMetalKernel {// These measured sibling exceptions apply only when the// grouped core was built for this layer.- if self.head_dim == 256 && (1..=16).contains(&suffix_length) && kv_length > D256_SINGLE_PASS_KV_THRESHOLD {- return self.single_pass.encode(arguments, command_buffer);+ if self.config.head_dim == 256+ && (1..=16).contains(&suffix_length)+ && kv_length > D256_SINGLE_PASS_KV_THRESHOLD+ {+ return Ok((self.single_pass.encode(arguments, command_buffer)?, false));}- if (self.head_dim == 256 && (9..=16).contains(&suffix_length))- || (self.head_dim == 128+ if (self.config.head_dim == 256 && (9..=16).contains(&suffix_length))+ || (self.config.head_dim == 128&& (9..=16).contains(&suffix_length)&& kv_length > D128_SHORT_KV_GEMM_THRESHOLD){- return self.two_pass.encode(arguments, command_buffer);+ return Ok((self.two_pass.encode(arguments, command_buffer)?, gate_applied_by_two_pass));}}if suffix_length > 8 {if let Some(gemm) = &self.gemm {- return gemm.encode(arguments, command_buffer);+ return Ok((gemm.encode(arguments, command_buffer)?, false));}if !is_trie && let Some(fallback) = &self.fallback {- return fallback.encode(arguments, command_buffer);+ return Ok((fallback.encode(arguments, command_buffer)?, false));}}if kv_length > SINGLE_PASS_KV_THRESHOLD {- self.two_pass.encode(arguments, command_buffer)+ Ok((self.two_pass.encode(arguments, command_buffer)?, gate_applied_by_two_pass))} else {- self.single_pass.encode(arguments, command_buffer)+ Ok((self.single_pass.encode(arguments, command_buffer)?, false))}}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metalindex 656d96237..bb66922da 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal@@ -1,5 +1,6 @@#include <metal_stdlib>#include "../common/dsl.h"+#include "attention_gate.h"template <typename T>VARIANTS(T, float, bfloat)@@ -13,6 +14,5 @@ PUBLIC KERNEL(SigmoidGate)(const uint batch_idx AXIS(batch_dim, 1)) {const uint output_idx = batch_idx * gate_dim + gate_idx;- float sigmoid = 1.0f / (1.0f + exp(-float(gate[batch_idx * gate_row_stride + gate_idx])));- output[output_idx] = T(float(output[output_idx]) * sigmoid);+ output[output_idx] = attention_apply_gate(output[output_idx], gate[batch_idx * gate_row_stride + gate_idx]);}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/single_pass.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/single_pass.rsindex aa034a94a..c3ce7d9eb 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/single_pass.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/single_pass.rs@@ -57,6 +57,7 @@ impl AttentionSinglePass {impl BufferRef<Backend = Metal>,impl BufferRef<Backend = Metal>,impl BufferRef<Backend = Metal>,+ impl BufferRef<Backend = Metal>,>,command_buffer: &mut MetalCommandBufferEncoding,) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/two_pass.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/two_pass.rsindex 8a7f93c64..0c9b1a1ee 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/two_pass.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/two_pass.rs@@ -64,7 +64,12 @@ impl AttentionTwoPass {) -> Result<MappedMutexGuard<'_, AttentionTwoPass2MetalKernel>, MetalError> {let mut second = self.second.lock();if second.is_none() {- *second = Some(AttentionTwoPass2MetalKernel::new(context, self.config.data_type, self.config.head_dim)?);+ *second = Some(AttentionTwoPass2MetalKernel::new(+ context,+ self.config.data_type,+ self.config.head_dim,+ self.config.gate_row_stride.is_some(),+ )?);}Ok(MutexGuard::map(second, |second| second.as_mut().expect("pass was just initialized")))}@@ -78,6 +83,7 @@ impl AttentionTwoPass {impl BufferRef<Backend = Metal>,impl BufferRef<Backend = Metal>,impl BufferRef<Backend = Metal>,+ impl BufferRef<Backend = Metal>,>,command_buffer: &mut MetalCommandBufferEncoding,) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {@@ -127,8 +133,10 @@ impl AttentionTwoPass {&sums,&maxs,&mut output,+ arguments.gate,config.num_q_heads,arguments.suffix_length,+ config.gate_row_stride,command_buffer,);Ok(output)diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rsindex 9c2b21056..d9b72f349 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs@@ -4,7 +4,7 @@ use thiserror::Error;use crate::{backends::common::{Backend, CommandBuffer, CommandBufferEncoding, Kernels,- kernel::{AttentionKernel, AttentionKernelConfig, AttentionPrepareKernel, SigmoidGateKernel},+ kernel::{AttentionKernel, AttentionKernelConfig, AttentionPrepareKernel},},config::{rope::AnyRoPEConfig, token_mixer::attention::AttentionConfig},data_type::DataType,@@ -39,12 +39,12 @@ pub struct Attention<B: Backend> {ring_capacity: Option<u32>,max_rope_length: Option<u32>,data_type: DataType,+ has_gate: bool,projection_dim: u32,projection: LinearProjection<B>,prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,sinks: Option<B::GlobalBuffer>,kernel: <B::Kernels as Kernels>::AttentionKernel,- gate_kernel: Option<<B::Kernels as Kernels>::SigmoidGateKernel>,out_projection: Box<dyn Linear<B>>,}@@ -148,6 +148,7 @@ impl<B: Backend> Attention<B> {num_groups,num_q_heads,has_sinks: sinks.is_some(),+ gate_row_stride: config.has_gate.then_some(projection_dim),is_kv_cache_ring,is_causal,sliding_window_size,@@ -157,12 +158,6 @@ impl<B: Backend> Attention<B> {).map_err(AttentionNewError::Backend)?;- let gate_kernel = config- .has_gate- .then(|| <B::Kernels as Kernels>::SigmoidGateKernel::new(context, data_type))- .transpose()- .map_err(AttentionNewError::Backend)?;-let out_projection = <dyn Linear<B>>::new(q_dim,[hidden_dim],@@ -180,6 +175,7 @@ impl<B: Backend> Attention<B> {ring_capacity,max_rope_length,data_type,+ has_gate: config.has_gate,projection_dim,projection: LinearProjection {lin: projection,@@ -188,7 +184,6 @@ impl<B: Backend> Attention<B> {prepare,sinks,kernel,- gate_kernel,out_projection,},in_projection_input_hadamard_factors,diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rsindex 75b337ac1..d9cb25e90 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs@@ -1,7 +1,7 @@use crate::{backends::common::{Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding,- kernel::{AttentionArguments, AttentionKernel, AttentionPrepareKernel, SigmoidGateKernel},+ kernel::{AttentionArguments, AttentionKernel, AttentionPrepareKernel},},encodable_block::{batch_topology::BatchTopology,@@ -44,8 +44,13 @@ impl<B: Backend> Attention<B> {command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {let qkvg = self.projection.project(hidden, batch_dim.size(), command_buffer)?;+ let gate = self.has_gate.then(|| {+ qkvg.subrange(+ (self.projection_dim - self.num_q_heads * self.head_dim) as usize * self.data_type.size_in_bytes()..,+ )+ });- let mut attention_output = match state {+ let attention_output = match state {Some(MaybeMut::Mut(state)) => {let prefix_len = state.view().prefix_len();let queries = self.prepare_kv_and_queries(@@ -58,12 +63,12 @@ impl<B: Backend> Attention<B> {batch_dim.size(),command_buffer,)?;- self.run_core(&queries, batch_dim, state, command_buffer)?+ self.run_core(&queries, gate, batch_dim, state, command_buffer)?},Some(MaybeMut::Const(state)) => {// KV sharing: QKVG contains queries and an optional gate only.let queries = self.prepare_queries(&qkvg, precalculated_rope, batch_dim.size(), command_buffer)?;- self.run_core(&queries, batch_dim, state, command_buffer)?+ self.run_core(&queries, gate, batch_dim, state, command_buffer)?},None => {let Some(num_kv_heads) = self.num_kv_heads else {@@ -98,24 +103,13 @@ impl<B: Backend> Attention<B> {trie: None::<&B::ConstantBuffer>,sinks: self.sinks.as_ref(),cache,+ gate,},command_buffer,)?},};- if let Some(gate_kernel) = &self.gate_kernel {- let gate_dim = self.num_q_heads * self.head_dim;- let gate_offset = self.projection_dim - gate_dim;- gate_kernel.encode(- qkvg.subrange(gate_offset as usize * self.data_type.size_in_bytes()..),- &mut attention_output,- gate_dim,- batch_dim.size(),- self.projection_dim,- command_buffer,- );- }self.out_projection.encode(attention_output, batch_dim.size(), command_buffer)}@@ -148,6 +142,7 @@ impl<B: Backend> Attention<B> {fn run_core(&self,queries: impl BufferRef<Backend = B>,+ gate: Option<impl BufferRef<Backend = B>>,batch_dim: &BatchTopology,state: &AttentionState<B>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,@@ -167,6 +162,7 @@ impl<B: Backend> Attention<B> {trie: trie.as_ref(),sinks: self.sinks.as_ref(),cache: state.view(),+ gate,},command_buffer,)diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rsindex d446df08c..bb054c5cf 100644--- a/crates/uzu-engine/src/encodable_block/weaver.rs+++ b/crates/uzu-engine/src/encodable_block/weaver.rs@@ -343,6 +343,7 @@ impl<B: Backend> Weaver<B> {trie: None::<&B::ConstantBuffer>,sinks: None,cache,+ gate: None::<&B::ConstantBuffer>,},command_buffer,)diff --git a/crates/uzu-engine/src/encodable_block/weaver_layer.rs b/crates/uzu-engine/src/encodable_block/weaver_layer.rsindex 2b9080e6b..691cdf286 100644--- a/crates/uzu-engine/src/encodable_block/weaver_layer.rs+++ b/crates/uzu-engine/src/encodable_block/weaver_layer.rs@@ -81,6 +81,7 @@ impl<B: Backend> WeaverLayer<B> {num_groups: num_heads,num_q_heads: num_heads,has_sinks: false,+ gate_row_stride: None,is_kv_cache_ring: false,is_causal: true,sliding_window_size: None,diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rsindex e820571a7..48a16d455 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs@@ -98,6 +98,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {trie: None::<&B::GlobalBuffer>,sinks: None,cache: KVCacheView::full(segment_prefix_length as u32),+ gate: None::<&B::GlobalBuffer>,};let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");@@ -132,6 +133,7 @@ fn get_gemm_output<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {trie: None::<&<Metal as Backend>::GlobalBuffer>,sinks: None,cache,+ gate: None::<&<Metal as Backend>::GlobalBuffer>,},&mut command_buffer,)diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rsindex 31e9f540e..9598b8344 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs@@ -3,11 +3,12 @@ use ndarray::{Array4, s};use uzu_engine_macros::uzu_test;use crate::{+ array::ArrayElement,backends::{common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,+ Backend, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,gpu_types::trie::TrieNode as GpuTrieNode,- kernel::{AttentionArguments, AttentionKernel},+ kernel::{AttentionArguments, AttentionKernel, SigmoidGateKernel},},cpu::Cpu,metal::Metal,@@ -237,6 +238,7 @@ fn run_single_pass_attention(trie: None::<&<Metal as Backend>::GlobalBuffer>,sinks: sinks_buffer.as_ref(),cache: KVCacheView::full(0),+ gate: None::<&<Metal as Backend>::GlobalBuffer>,},&mut command_buffer,)?;@@ -306,6 +308,7 @@ fn run_gemm_attention(trie: None::<&<Metal as Backend>::GlobalBuffer>,sinks: sinks_buffer.as_ref(),cache: KVCacheView::full(0),+ gate: None::<&<Metal as Backend>::GlobalBuffer>,};let pooled_output = kernel.encode(args, &mut command_buffer)?;@@ -489,6 +492,7 @@ fn run_two_pass_attention(trie: None::<&<Metal as Backend>::GlobalBuffer>,sinks: sinks_buffer.as_ref(),cache: KVCacheView::full(0),+ gate: None::<&<Metal as Backend>::GlobalBuffer>,},&mut command_buffer,)?;@@ -612,6 +616,7 @@ fn run_attention_with_kernel<B: Backend>(trie: trie.as_ref(),sinks: None,cache,+ gate: None::<&B::GlobalBuffer>,};let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");let pooled = kernel.encode(arguments, &mut command_buffer).expect("encode");@@ -744,3 +749,73 @@ fn attention_kernel_ring_matches_full_on_wrap() {);assert_eq_float::<bf16>(&full, &ring, 1e-2, "wrapped ring attention");}++fn check_attention_gate<B: Backend>(+ context: &B::Context,+ suffix_length: usize,+ prefix_length: usize,+) {+ const HEAD_DIM: usize = 256;+ const Q_HEADS: usize = 24;+ const KV_HEADS: usize = 4;+ const GATE_DIM: usize = Q_HEADS * HEAD_DIM;+ const PROJECTION_DIM: usize = 2 * (Q_HEADS + KV_HEADS) * HEAD_DIM;+ let mut config = super::default_attention_config::<bf16>(HEAD_DIM, Q_HEADS, KV_HEADS, true);+ let plain = <B::Kernels as Kernels>::AttentionKernel::new(context, config).expect("attention kernel");+ config.gate_row_stride = Some(PROJECTION_DIM as u32);+ let gated = <B::Kernels as Kernels>::AttentionKernel::new(context, config).expect("gated attention kernel");+ let sigmoid_gate =+ <B::Kernels as Kernels>::SigmoidGateKernel::new(context, bf16::data_type()).expect("sigmoid gate kernel");+ let shape = (HEAD_DIM, Q_HEADS, KV_HEADS, suffix_length, prefix_length, true);+ let (keys, values) = attention_data(shape);+ let queries = create_buffer_with_data::<B, bf16>(context, &fill_attention(Q_HEADS * suffix_length * HEAD_DIM, 0.5));+ let keys = create_buffer_with_data::<B, bf16>(context, &keys);+ let values = create_buffer_with_data::<B, bf16>(context, &values);+ let qkvg = create_buffer_with_data::<B, bf16>(context, &fill_attention(suffix_length * PROJECTION_DIM, 3.0));+ let arguments = |gate| AttentionArguments {+ queries: &queries,+ keys: &keys,+ values: &values,+ suffix_length: suffix_length as u32,+ trie: None::<&B::GlobalBuffer>,+ sinks: None,+ cache: KVCacheView::full(prefix_length as u32),+ gate,+ };+ let gate_segment = qkvg.subrange((PROJECTION_DIM - GATE_DIM) * size_of::<bf16>()..);+ let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");+ let mut expected = plain.encode(arguments(None), &mut command_buffer).expect("plain attention");+ sigmoid_gate.encode(+ gate_segment,+ &mut expected,+ GATE_DIM as u32,+ suffix_length as u32,+ PROJECTION_DIM as u32,+ &mut command_buffer,+ );+ let fused = gated.encode(arguments(Some(gate_segment)), &mut command_buffer).expect("gated attention");+ let mut expected_out = create_buffer::<B, bf16>(context, suffix_length * GATE_DIM);+ let mut fused_out = create_buffer::<B, bf16>(context, suffix_length * GATE_DIM);+ command_buffer.encode_copy(&expected, &mut expected_out);+ command_buffer.encode_copy(&fused, &mut fused_out);+ submit_command_buffer(command_buffer);+ let expected = buffer_to_vec::<B, bf16>(&expected_out);+ let fused = buffer_to_vec::<B, bf16>(&fused_out);+ assert_eq!(expected, fused, "gated attention differs at S{suffix_length} L{prefix_length}");+}++#[uzu_test]+fn attention_gate_matches_standalone_gate_on_cpu() {+ let context = <Cpu as Backend>::Context::new().expect("CPU attention context");+ check_attention_gate::<Cpu>(context.as_ref(), 3, 29);+}++#[uzu_test]+fn attention_gate_matches_standalone_gate_on_metal() {+ let context = <Metal as Backend>::Context::new().expect("Metal attention context");+ for (suffix_length, prefix_length) in+ [(1, 100), (1, 2000), (200, 800), (2, 1022), (16, 16368), (65, 959), (65, 16319), (1024, 0)]+ {+ check_attention_gate::<Metal>(context.as_ref(), suffix_length, prefix_length);+ }+}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rsindex 888095db5..f13b664d7 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs@@ -17,6 +17,7 @@ fn default_attention_config<T: ArrayElement>(num_groups: num_groups as u32,num_q_heads: num_q_heads as u32,has_sinks: false,+ gate_row_stride: None,is_kv_cache_ring: false,is_causal,sliding_window_size: None,