Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Fuse sigmoid gate into Attention (#930)

CCYehf9d747746e8a
diff --git a/crates/uzu-engine/src/backends/common/kernel/attention.rs b/crates/uzu-engine/src/backends/common/kernel/attention.rs
index 1ca7225d9..823389935 100644
--- a/crates/uzu-engine/src/backends/common/kernel/attention.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/attention.rs
@@ -21,6 +21,7 @@ pub trait AttentionKernel: Sized + Send + Sync {
impl BufferRef<Backend = Self::Backend>,
impl BufferRef<Backend = Self::Backend>,
impl BufferRef<Backend = Self::Backend>,
+ impl BufferRef<Backend = Self::Backend>,
>,
command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<<Self::Backend as Backend>::ScratchBuffer, <Self::Backend as Backend>::Error>;
@@ -32,6 +33,7 @@ pub struct AttentionKernelConfig {
pub num_groups: u32,
pub num_q_heads: u32,
pub has_sinks: bool,
+ pub gate_row_stride: Option<u32>,
pub is_kv_cache_ring: bool,
pub is_causal: bool,
pub sliding_window_size: Option<u32>,
@@ -46,6 +48,7 @@ pub struct AttentionArguments<
TT: BufferRef<Backend = B>,
KT: BufferRef<Backend = B>,
VT: BufferRef<Backend = B>,
+ GT: BufferRef<Backend = B>,
> {
pub queries: QT,
pub keys: KT,
@@ -54,4 +57,5 @@ pub struct AttentionArguments<
pub trie: Option<TT>,
pub sinks: Option<&'a B::GlobalBuffer>,
pub cache: KVCacheView,
+ pub gate: Option<GT>,
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
index 18ad128b4..21b07f2d1 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
@@ -12,7 +12,9 @@ mod mask;
use crate::backends::{
common::{
BufferRef, CommandBufferEncoding,
- kernel::{AttentionArguments, AttentionKernel, AttentionKernelConfig, AttentionSinglePassKernel},
+ kernel::{
+ AttentionArguments, AttentionKernel, AttentionKernelConfig, AttentionSinglePassKernel, SigmoidGateKernel,
+ },
},
cpu::{Cpu, command_buffer::CpuCommandBufferEncoding, context::CpuContext, error::CpuError},
};
@@ -42,6 +44,7 @@ impl AttentionKernel for AttentionCpuKernel {
impl BufferRef<Backend = Cpu>,
impl BufferRef<Backend = Cpu>,
impl BufferRef<Backend = Cpu>,
+ impl BufferRef<Backend = Cpu>,
>,
command_buffer: &mut CpuCommandBufferEncoding,
) -> Result<<Cpu as Backend>::ScratchBuffer, CpuError> {
@@ -83,6 +86,17 @@ impl AttentionKernel for AttentionCpuKernel {
command_buffer,
);
command_buffer.pop_debug_group();
+ if let (Some(gate), Some(gate_row_stride)) = (arguments.gate, config.gate_row_stride) {
+ <sigmoid_gate::SigmoidGateCpuKernel as SigmoidGateKernel>::new(command_buffer.context(), config.data_type)?
+ .encode(
+ gate,
+ &mut output,
+ config.num_q_heads * config.head_dim,
+ arguments.suffix_length,
+ gate_row_stride,
+ command_buffer,
+ );
+ }
Ok(output)
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gate.h b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gate.h
new file mode 100644
index 000000000..8222b4242
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gate.h
@@ -0,0 +1,9 @@
+#pragma once
+
+#include <metal_stdlib>
+
+template <typename T>
+METAL_FUNC T attention_apply_gate(const T result, const T gate) {
+ const float sigmoid = 1.0f / (1.0f + metal::exp(-float(gate)));
+ return T(float(result) * sigmoid);
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal
index 3594bcbce..a34d453ef 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -5,6 +5,7 @@
#include "../common/thread_context.h"
#include "../generated/ring.h"
#include "../generated/trie.h"
+#include "attention_gate.h"
#include "mask.h"
#define TOTAL_BLOCKS_COUNT 32
@@ -146,8 +147,11 @@ KERNEL(AttentionTwoPass2)(
const device float* sums,
const device float* maxs,
device T* out,
+ const device T* gate OPTIONAL(has_gate),
const constant uint& num_heads,
const constant uint& suffix_length,
+ const constant uint& gate_row_stride OPTIONAL(has_gate),
+ const bool has_gate SPECIALIZE,
const ThreadContext thread_context,
const uint head_idx GROUPS(num_heads),
const uint q_seq_idx GROUPS(suffix_length),
@@ -197,7 +201,12 @@ KERNEL(AttentionTwoPass2)(
if (thread_context.simd_lane_id == 0) {
out += o_offset * HEAD_DIM + thread_context.simdgroup_index * elements_per_thread;
for (uint i = 0; i < elements_per_thread; i++) {
- out[i] = static_cast<T>(o[i] / sum_exp_score);
+ T result = static_cast<T>(o[i] / sum_exp_score);
+ if (has_gate) {
+ const uint channel = thread_context.simdgroup_index * elements_per_thread + i;
+ result = attention_apply_gate(result, gate[q_seq_idx * gate_row_stride + head_idx * HEAD_DIM + channel]);
+ }
+ out[i] = result;
}
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
index bf19f9ea0..b8a0b01c9 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
@@ -89,6 +89,7 @@ impl AttentionFallback {
impl BufferRef<Backend = Metal>,
impl BufferRef<Backend = Metal>,
impl BufferRef<Backend = Metal>,
+ impl BufferRef<Backend = Metal>,
>,
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs
index d8e93eb1a..957e25657 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs
@@ -120,6 +120,7 @@ impl AttentionGemm {
impl BufferRef<Backend = Metal>,
impl BufferRef<Backend = Metal>,
impl BufferRef<Backend = Metal>,
+ impl BufferRef<Backend = Metal>,
>,
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_gemm_grouped.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_gemm_grouped.metal
index 8478518d5..2aeb9fbfd 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_gemm_grouped.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_gemm_grouped.metal
@@ -7,6 +7,7 @@
#include "../../matmul/common/mxu_fragment/ops.h"
#include "../../generated/attention.h"
#include "../../generated/trie.h"
+#include "../attention_gate.h"
using namespace metal;
using namespace uzu::matmul;
@@ -402,6 +403,7 @@ KERNEL(AttentionGemmGroupedCombine)(
const device float* partial_maxs,
const device float* partial_sums,
device T* o,
+ const device T* gate OPTIONAL(has_gate),
const constant AttnParams& params,
const constant uint& grouped_rows,
const constant uint& block_rows,
@@ -409,6 +411,8 @@ KERNEL(AttentionGemmGroupedCombine)(
const constant uint& kv_heads,
const constant uint& batch_count,
const constant uint& num_splits,
+ const constant uint& gate_row_stride OPTIONAL(has_gate),
+ const bool has_gate SPECIALIZE,
const uint grouped_row_idx GROUPS(grouped_rows),
const uint kv_head_idx GROUPS(kv_heads),
const uint batch_idx GROUPS(batch_count),
@@ -446,7 +450,14 @@ KERNEL(AttentionGemmGroupedCombine)(
const uint head_in_group = grouped_row_idx / params.q_len;
const uint query_position = grouped_row_idx - head_in_group * params.q_len;
const uint q_head_idx = kv_head_idx * params.gqa_factor + head_in_group;
+ T result = static_cast<T>(denominator > 0.0f ? numerator / denominator : 0.0f);
+ if (has_gate) {
+ result = attention_apply_gate(
+ result,
+ gate[(batch_idx * params.q_len + query_position) * gate_row_stride + q_head_idx * BD + column]
+ );
+ }
o += size_t(batch_idx) * params.o_strides[0] + size_t(q_head_idx) * params.o_strides[1] +
size_t(query_position) * params.o_strides[2] + column;
- o[0] = static_cast<T>(denominator > 0.0f ? numerator / denominator : 0.0f);
+ o[0] = result;
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
index bd3697228..a26465005 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
@@ -36,6 +36,7 @@ struct AttentionGemmGroupedMetal {
num_groups: u32,
num_q_heads: u32,
scale: Option<f32>,
+ gate_row_stride: Option<u32>,
block_rows: u32,
mask: MaskKind,
}
@@ -47,6 +48,7 @@ impl AttentionGemmGroupedMetal {
num_groups: u32,
num_q_heads: u32,
scale: Option<f32>,
+ gate_row_stride: Option<u32>,
mask: MaskKind,
) -> Result<Self, MetalError> {
assert!(matches!(head_dim, 128 | 256), "head_dim must be 128 or 256");
@@ -65,7 +67,8 @@ impl AttentionGemmGroupedMetal {
mask.is_trie(),
)
};
- let combine = AttentionGemmGroupedCombineMetalKernel::new(context, DataType::BF16, head_dim)?;
+ let combine =
+ AttentionGemmGroupedCombineMetalKernel::new(context, DataType::BF16, head_dim, gate_row_stride.is_some())?;
Ok(Self {
kernel: new_kernel(false)?,
split_kernel: new_kernel(true)?,
@@ -74,6 +77,7 @@ impl AttentionGemmGroupedMetal {
num_groups,
num_q_heads,
scale,
+ gate_row_stride,
block_rows,
mask,
})
@@ -86,12 +90,13 @@ impl AttentionGemmGroupedMetal {
values: impl BufferRef<Backend = Metal>,
output: impl BufferMut<Backend = Metal>,
trie: Option<impl BufferRef<Backend = Metal>>,
+ gate: Option<impl BufferRef<Backend = Metal>>,
suffix_length: u32,
kv_length: u32,
q_replicas: u32,
num_splits: u32,
command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<(), MetalError> {
+ ) -> Result<bool, MetalError> {
let params = self.params(suffix_length, kv_length);
let grouped_rows = self.num_q_heads / self.num_groups * suffix_length;
let m_tiles = grouped_rows.div_ceil(self.block_rows);
@@ -116,7 +121,7 @@ impl AttentionGemmGroupedMetal {
1,
command_buffer,
);
- return Ok(());
+ return Ok(false);
}
let partial_rows = q_replicas * self.num_groups * m_tiles * num_splits * self.block_rows;
@@ -146,6 +151,7 @@ impl AttentionGemmGroupedMetal {
&partial_maxs,
&partial_sums,
output,
+ gate,
params,
grouped_rows,
self.block_rows,
@@ -153,9 +159,10 @@ impl AttentionGemmGroupedMetal {
self.num_groups,
q_replicas,
num_splits,
+ self.gate_row_stride,
command_buffer,
);
- Ok(())
+ Ok(gate.is_some())
}
fn params(
@@ -194,6 +201,7 @@ pub struct AttentionGemmGrouped {
num_groups: u32,
num_q_heads: u32,
scale: Option<f32>,
+ gate_row_stride: Option<u32>,
}
impl AttentionGemmGrouped {
@@ -215,6 +223,7 @@ impl AttentionGemmGrouped {
self.num_groups,
self.num_q_heads,
self.scale,
+ self.gate_row_stride,
mask,
)?);
}
@@ -236,6 +245,7 @@ impl AttentionGemmGrouped {
num_groups: config.num_groups,
num_q_heads: config.num_q_heads,
scale: config.scale,
+ gate_row_stride: config.gate_row_stride,
}
}
@@ -258,9 +268,10 @@ impl AttentionGemmGrouped {
impl BufferRef<Backend = Metal>,
impl BufferRef<Backend = Metal>,
impl BufferRef<Backend = Metal>,
+ impl BufferRef<Backend = Metal>,
>,
command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {
+ ) -> Result<(<Metal as Backend>::ScratchBuffer, bool), MetalError> {
let suffix_length = arguments.suffix_length;
assert!(arguments.cache.ring_params().is_none(), "ring KV cache is unsupported");
assert!(arguments.sinks.is_none(), "attention sinks are unsupported");
@@ -282,18 +293,19 @@ impl AttentionGemmGrouped {
kv_length,
gpu_core_count,
);
- core.encode(
+ let gate_applied = core.encode(
arguments.queries,
arguments.keys,
arguments.values,
&mut output,
arguments.trie,
+ arguments.gate,
suffix_length,
kv_length,
1,
num_splits,
command_buffer,
)?;
- Ok(output)
+ Ok((output, gate_applied))
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
index 5bbafd69e..d16465c6f 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
@@ -9,9 +9,12 @@ mod two_pass;
use crate::backends::{
common::{
BufferRef, CommandBufferEncoding,
- kernel::{AttentionArguments, AttentionKernel, AttentionKernelConfig},
+ kernel::{AttentionArguments, AttentionKernel, AttentionKernelConfig, SigmoidGateKernel},
+ },
+ metal::{
+ Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError,
+ kernel::SigmoidGateMetalKernel,
},
- metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError},
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -48,8 +51,8 @@ const D256_SINGLE_PASS_KV_THRESHOLD: u32 = 150_000;
const D128_SHORT_KV_GEMM_THRESHOLD: u32 = 2_048;
pub struct AttentionMetalKernel {
- head_dim: u32,
- is_causal: bool,
+ config: AttentionKernelConfig,
+ gate: Option<(SigmoidGateMetalKernel, u32)>,
grouped: Option<gemm_grouped::AttentionGemmGrouped>,
gemm: Option<gemm::AttentionGemm>,
fallback: Option<fallback::AttentionFallback>,
@@ -70,9 +73,13 @@ impl AttentionKernel for AttentionMetalKernel {
let fallback = fallback::AttentionFallback::is_supported(&config)
.then(|| fallback::AttentionFallback::new(&config, context))
.transpose()?;
+ let gate = config
+ .gate_row_stride
+ .map(|stride| SigmoidGateMetalKernel::new(context, config.data_type).map(|kernel| (kernel, stride)))
+ .transpose()?;
Ok(Self {
- head_dim: config.head_dim,
- is_causal: config.is_causal,
+ config,
+ gate,
grouped,
gemm,
fallback,
@@ -90,11 +97,25 @@ impl AttentionKernel for AttentionMetalKernel {
impl BufferRef<Backend = Metal>,
impl BufferRef<Backend = Metal>,
impl BufferRef<Backend = Metal>,
+ impl BufferRef<Backend = Metal>,
>,
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {
command_buffer.push_debug_group("attention core");
- let result = self.encode_impl(arguments, command_buffer);
+ let (gate, suffix_length) = (arguments.gate, arguments.suffix_length);
+ let result = self.encode_impl(arguments, command_buffer).map(|(mut output, gate_applied)| {
+ if let (Some(gate), Some((kernel, stride)), false) = (gate, &self.gate, gate_applied) {
+ kernel.encode(
+ gate,
+ &mut output,
+ self.config.num_q_heads * self.config.head_dim,
+ suffix_length,
+ *stride,
+ command_buffer,
+ );
+ }
+ output
+ });
command_buffer.pop_debug_group();
result
}
@@ -110,14 +131,16 @@ impl AttentionMetalKernel {
impl BufferRef<Backend = Metal>,
impl BufferRef<Backend = Metal>,
impl BufferRef<Backend = Metal>,
+ impl BufferRef<Backend = Metal>,
>,
command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {
+ ) -> Result<(<Metal as Backend>::ScratchBuffer, bool), MetalError> {
let suffix_length = arguments.suffix_length;
+ let gate_applied_by_two_pass = arguments.gate.is_some();
let kv_length = arguments.cache.prefix_len() + suffix_length;
let is_trie = arguments.trie.is_some();
let is_ring = arguments.cache.ring_params().is_some();
- let mask = MaskKind::for_attention(self.is_causal, is_trie);
+ let mask = MaskKind::for_attention(self.config.is_causal, is_trie);
if !is_ring
&& let Some(mask) = mask
@@ -129,31 +152,34 @@ impl AttentionMetalKernel {
// These measured sibling exceptions apply only when the
// grouped core was built for this layer.
- if self.head_dim == 256 && (1..=16).contains(&suffix_length) && kv_length > D256_SINGLE_PASS_KV_THRESHOLD {
- return self.single_pass.encode(arguments, command_buffer);
+ if self.config.head_dim == 256
+ && (1..=16).contains(&suffix_length)
+ && kv_length > D256_SINGLE_PASS_KV_THRESHOLD
+ {
+ return Ok((self.single_pass.encode(arguments, command_buffer)?, false));
}
- if (self.head_dim == 256 && (9..=16).contains(&suffix_length))
- || (self.head_dim == 128
+ if (self.config.head_dim == 256 && (9..=16).contains(&suffix_length))
+ || (self.config.head_dim == 128
&& (9..=16).contains(&suffix_length)
&& kv_length > D128_SHORT_KV_GEMM_THRESHOLD)
{
- return self.two_pass.encode(arguments, command_buffer);
+ return Ok((self.two_pass.encode(arguments, command_buffer)?, gate_applied_by_two_pass));
}
}
if suffix_length > 8 {
if let Some(gemm) = &self.gemm {
- return gemm.encode(arguments, command_buffer);
+ return Ok((gemm.encode(arguments, command_buffer)?, false));
}
if !is_trie && let Some(fallback) = &self.fallback {
- return fallback.encode(arguments, command_buffer);
+ return Ok((fallback.encode(arguments, command_buffer)?, false));
}
}
if kv_length > SINGLE_PASS_KV_THRESHOLD {
- self.two_pass.encode(arguments, command_buffer)
+ Ok((self.two_pass.encode(arguments, command_buffer)?, gate_applied_by_two_pass))
} else {
- self.single_pass.encode(arguments, command_buffer)
+ Ok((self.single_pass.encode(arguments, command_buffer)?, false))
}
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal
index 656d96237..bb66922da 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal
@@ -1,5 +1,6 @@
#include <metal_stdlib>
#include "../common/dsl.h"
+#include "attention_gate.h"
template <typename T>
VARIANTS(T, float, bfloat)
@@ -13,6 +14,5 @@ PUBLIC KERNEL(SigmoidGate)(
const uint batch_idx AXIS(batch_dim, 1)
) {
const uint output_idx = batch_idx * gate_dim + gate_idx;
- float sigmoid = 1.0f / (1.0f + exp(-float(gate[batch_idx * gate_row_stride + gate_idx])));
- output[output_idx] = T(float(output[output_idx]) * sigmoid);
+ output[output_idx] = attention_apply_gate(output[output_idx], gate[batch_idx * gate_row_stride + gate_idx]);
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/single_pass.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/single_pass.rs
index aa034a94a..c3ce7d9eb 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/single_pass.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/single_pass.rs
@@ -57,6 +57,7 @@ impl AttentionSinglePass {
impl BufferRef<Backend = Metal>,
impl BufferRef<Backend = Metal>,
impl BufferRef<Backend = Metal>,
+ impl BufferRef<Backend = Metal>,
>,
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/two_pass.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/two_pass.rs
index 8a7f93c64..0c9b1a1ee 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/two_pass.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/two_pass.rs
@@ -64,7 +64,12 @@ impl AttentionTwoPass {
) -> Result<MappedMutexGuard<'_, AttentionTwoPass2MetalKernel>, MetalError> {
let mut second = self.second.lock();
if second.is_none() {
- *second = Some(AttentionTwoPass2MetalKernel::new(context, self.config.data_type, self.config.head_dim)?);
+ *second = Some(AttentionTwoPass2MetalKernel::new(
+ context,
+ self.config.data_type,
+ self.config.head_dim,
+ self.config.gate_row_stride.is_some(),
+ )?);
}
Ok(MutexGuard::map(second, |second| second.as_mut().expect("pass was just initialized")))
}
@@ -78,6 +83,7 @@ impl AttentionTwoPass {
impl BufferRef<Backend = Metal>,
impl BufferRef<Backend = Metal>,
impl BufferRef<Backend = Metal>,
+ impl BufferRef<Backend = Metal>,
>,
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {
@@ -127,8 +133,10 @@ impl AttentionTwoPass {
&sums,
&maxs,
&mut output,
+ arguments.gate,
config.num_q_heads,
arguments.suffix_length,
+ config.gate_row_stride,
command_buffer,
);
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
index 9c2b21056..d9b72f349 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
@@ -4,7 +4,7 @@ use thiserror::Error;
use crate::{
backends::common::{
Backend, CommandBuffer, CommandBufferEncoding, Kernels,
- kernel::{AttentionKernel, AttentionKernelConfig, AttentionPrepareKernel, SigmoidGateKernel},
+ kernel::{AttentionKernel, AttentionKernelConfig, AttentionPrepareKernel},
},
config::{rope::AnyRoPEConfig, token_mixer::attention::AttentionConfig},
data_type::DataType,
@@ -39,12 +39,12 @@ pub struct Attention<B: Backend> {
ring_capacity: Option<u32>,
max_rope_length: Option<u32>,
data_type: DataType,
+ has_gate: bool,
projection_dim: u32,
projection: LinearProjection<B>,
prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,
sinks: Option<B::GlobalBuffer>,
kernel: <B::Kernels as Kernels>::AttentionKernel,
- gate_kernel: Option<<B::Kernels as Kernels>::SigmoidGateKernel>,
out_projection: Box<dyn Linear<B>>,
}
@@ -148,6 +148,7 @@ impl<B: Backend> Attention<B> {
num_groups,
num_q_heads,
has_sinks: sinks.is_some(),
+ gate_row_stride: config.has_gate.then_some(projection_dim),
is_kv_cache_ring,
is_causal,
sliding_window_size,
@@ -157,12 +158,6 @@ impl<B: Backend> Attention<B> {
)
.map_err(AttentionNewError::Backend)?;
- let gate_kernel = config
- .has_gate
- .then(|| <B::Kernels as Kernels>::SigmoidGateKernel::new(context, data_type))
- .transpose()
- .map_err(AttentionNewError::Backend)?;
-
let out_projection = <dyn Linear<B>>::new(
q_dim,
[hidden_dim],
@@ -180,6 +175,7 @@ impl<B: Backend> Attention<B> {
ring_capacity,
max_rope_length,
data_type,
+ has_gate: config.has_gate,
projection_dim,
projection: LinearProjection {
lin: projection,
@@ -188,7 +184,6 @@ impl<B: Backend> Attention<B> {
prepare,
sinks,
kernel,
- gate_kernel,
out_projection,
},
in_projection_input_hadamard_factors,
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
index 75b337ac1..d9cb25e90 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
@@ -1,7 +1,7 @@
use crate::{
backends::common::{
Backend, BufferMut, BufferRef, CommandBuffer, CommandBufferEncoding,
- kernel::{AttentionArguments, AttentionKernel, AttentionPrepareKernel, SigmoidGateKernel},
+ kernel::{AttentionArguments, AttentionKernel, AttentionPrepareKernel},
},
encodable_block::{
batch_topology::BatchTopology,
@@ -44,8 +44,13 @@ impl<B: Backend> Attention<B> {
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
let qkvg = self.projection.project(hidden, batch_dim.size(), command_buffer)?;
+ let gate = self.has_gate.then(|| {
+ qkvg.subrange(
+ (self.projection_dim - self.num_q_heads * self.head_dim) as usize * self.data_type.size_in_bytes()..,
+ )
+ });
- let mut attention_output = match state {
+ let attention_output = match state {
Some(MaybeMut::Mut(state)) => {
let prefix_len = state.view().prefix_len();
let queries = self.prepare_kv_and_queries(
@@ -58,12 +63,12 @@ impl<B: Backend> Attention<B> {
batch_dim.size(),
command_buffer,
)?;
- self.run_core(&queries, batch_dim, state, command_buffer)?
+ self.run_core(&queries, gate, batch_dim, state, command_buffer)?
},
Some(MaybeMut::Const(state)) => {
// KV sharing: QKVG contains queries and an optional gate only.
let queries = self.prepare_queries(&qkvg, precalculated_rope, batch_dim.size(), command_buffer)?;
- self.run_core(&queries, batch_dim, state, command_buffer)?
+ self.run_core(&queries, gate, batch_dim, state, command_buffer)?
},
None => {
let Some(num_kv_heads) = self.num_kv_heads else {
@@ -98,24 +103,13 @@ impl<B: Backend> Attention<B> {
trie: None::<&B::ConstantBuffer>,
sinks: self.sinks.as_ref(),
cache,
+ gate,
},
command_buffer,
)?
},
};
- if let Some(gate_kernel) = &self.gate_kernel {
- let gate_dim = self.num_q_heads * self.head_dim;
- let gate_offset = self.projection_dim - gate_dim;
- gate_kernel.encode(
- qkvg.subrange(gate_offset as usize * self.data_type.size_in_bytes()..),
- &mut attention_output,
- gate_dim,
- batch_dim.size(),
- self.projection_dim,
- command_buffer,
- );
- }
self.out_projection.encode(attention_output, batch_dim.size(), command_buffer)
}
@@ -148,6 +142,7 @@ impl<B: Backend> Attention<B> {
fn run_core(
&self,
queries: impl BufferRef<Backend = B>,
+ gate: Option<impl BufferRef<Backend = B>>,
batch_dim: &BatchTopology,
state: &AttentionState<B>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
@@ -167,6 +162,7 @@ impl<B: Backend> Attention<B> {
trie: trie.as_ref(),
sinks: self.sinks.as_ref(),
cache: state.view(),
+ gate,
},
command_buffer,
)
diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rs
index d446df08c..bb054c5cf 100644
--- a/crates/uzu-engine/src/encodable_block/weaver.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver.rs
@@ -343,6 +343,7 @@ impl<B: Backend> Weaver<B> {
trie: None::<&B::ConstantBuffer>,
sinks: None,
cache,
+ gate: None::<&B::ConstantBuffer>,
},
command_buffer,
)
diff --git a/crates/uzu-engine/src/encodable_block/weaver_layer.rs b/crates/uzu-engine/src/encodable_block/weaver_layer.rs
index 2b9080e6b..691cdf286 100644
--- a/crates/uzu-engine/src/encodable_block/weaver_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver_layer.rs
@@ -81,6 +81,7 @@ impl<B: Backend> WeaverLayer<B> {
num_groups: num_heads,
num_q_heads: num_heads,
has_sinks: false,
+ gate_row_stride: None,
is_kv_cache_ring: false,
is_causal: true,
sliding_window_size: None,
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs
index e820571a7..48a16d455 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs
@@ -98,6 +98,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
trie: None::<&B::GlobalBuffer>,
sinks: None,
cache: KVCacheView::full(segment_prefix_length as u32),
+ gate: None::<&B::GlobalBuffer>,
};
let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
@@ -132,6 +133,7 @@ fn get_gemm_output<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
trie: None::<&<Metal as Backend>::GlobalBuffer>,
sinks: None,
cache,
+ gate: None::<&<Metal as Backend>::GlobalBuffer>,
},
&mut command_buffer,
)
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
index 31e9f540e..9598b8344 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
@@ -3,11 +3,12 @@ use ndarray::{Array4, s};
use uzu_engine_macros::uzu_test;
use crate::{
+ array::ArrayElement,
backends::{
common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
+ Backend, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
gpu_types::trie::TrieNode as GpuTrieNode,
- kernel::{AttentionArguments, AttentionKernel},
+ kernel::{AttentionArguments, AttentionKernel, SigmoidGateKernel},
},
cpu::Cpu,
metal::Metal,
@@ -237,6 +238,7 @@ fn run_single_pass_attention(
trie: None::<&<Metal as Backend>::GlobalBuffer>,
sinks: sinks_buffer.as_ref(),
cache: KVCacheView::full(0),
+ gate: None::<&<Metal as Backend>::GlobalBuffer>,
},
&mut command_buffer,
)?;
@@ -306,6 +308,7 @@ fn run_gemm_attention(
trie: None::<&<Metal as Backend>::GlobalBuffer>,
sinks: sinks_buffer.as_ref(),
cache: KVCacheView::full(0),
+ gate: None::<&<Metal as Backend>::GlobalBuffer>,
};
let pooled_output = kernel.encode(args, &mut command_buffer)?;
@@ -489,6 +492,7 @@ fn run_two_pass_attention(
trie: None::<&<Metal as Backend>::GlobalBuffer>,
sinks: sinks_buffer.as_ref(),
cache: KVCacheView::full(0),
+ gate: None::<&<Metal as Backend>::GlobalBuffer>,
},
&mut command_buffer,
)?;
@@ -612,6 +616,7 @@ fn run_attention_with_kernel<B: Backend>(
trie: trie.as_ref(),
sinks: None,
cache,
+ gate: None::<&B::GlobalBuffer>,
};
let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");
let pooled = kernel.encode(arguments, &mut command_buffer).expect("encode");
@@ -744,3 +749,73 @@ fn attention_kernel_ring_matches_full_on_wrap() {
);
assert_eq_float::<bf16>(&full, &ring, 1e-2, "wrapped ring attention");
}
+
+fn check_attention_gate<B: Backend>(
+ context: &B::Context,
+ suffix_length: usize,
+ prefix_length: usize,
+) {
+ const HEAD_DIM: usize = 256;
+ const Q_HEADS: usize = 24;
+ const KV_HEADS: usize = 4;
+ const GATE_DIM: usize = Q_HEADS * HEAD_DIM;
+ const PROJECTION_DIM: usize = 2 * (Q_HEADS + KV_HEADS) * HEAD_DIM;
+ let mut config = super::default_attention_config::<bf16>(HEAD_DIM, Q_HEADS, KV_HEADS, true);
+ let plain = <B::Kernels as Kernels>::AttentionKernel::new(context, config).expect("attention kernel");
+ config.gate_row_stride = Some(PROJECTION_DIM as u32);
+ let gated = <B::Kernels as Kernels>::AttentionKernel::new(context, config).expect("gated attention kernel");
+ let sigmoid_gate =
+ <B::Kernels as Kernels>::SigmoidGateKernel::new(context, bf16::data_type()).expect("sigmoid gate kernel");
+ let shape = (HEAD_DIM, Q_HEADS, KV_HEADS, suffix_length, prefix_length, true);
+ let (keys, values) = attention_data(shape);
+ let queries = create_buffer_with_data::<B, bf16>(context, &fill_attention(Q_HEADS * suffix_length * HEAD_DIM, 0.5));
+ let keys = create_buffer_with_data::<B, bf16>(context, &keys);
+ let values = create_buffer_with_data::<B, bf16>(context, &values);
+ let qkvg = create_buffer_with_data::<B, bf16>(context, &fill_attention(suffix_length * PROJECTION_DIM, 3.0));
+ let arguments = |gate| AttentionArguments {
+ queries: &queries,
+ keys: &keys,
+ values: &values,
+ suffix_length: suffix_length as u32,
+ trie: None::<&B::GlobalBuffer>,
+ sinks: None,
+ cache: KVCacheView::full(prefix_length as u32),
+ gate,
+ };
+ let gate_segment = qkvg.subrange((PROJECTION_DIM - GATE_DIM) * size_of::<bf16>()..);
+ let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");
+ let mut expected = plain.encode(arguments(None), &mut command_buffer).expect("plain attention");
+ sigmoid_gate.encode(
+ gate_segment,
+ &mut expected,
+ GATE_DIM as u32,
+ suffix_length as u32,
+ PROJECTION_DIM as u32,
+ &mut command_buffer,
+ );
+ let fused = gated.encode(arguments(Some(gate_segment)), &mut command_buffer).expect("gated attention");
+ let mut expected_out = create_buffer::<B, bf16>(context, suffix_length * GATE_DIM);
+ let mut fused_out = create_buffer::<B, bf16>(context, suffix_length * GATE_DIM);
+ command_buffer.encode_copy(&expected, &mut expected_out);
+ command_buffer.encode_copy(&fused, &mut fused_out);
+ submit_command_buffer(command_buffer);
+ let expected = buffer_to_vec::<B, bf16>(&expected_out);
+ let fused = buffer_to_vec::<B, bf16>(&fused_out);
+ assert_eq!(expected, fused, "gated attention differs at S{suffix_length} L{prefix_length}");
+}
+
+#[uzu_test]
+fn attention_gate_matches_standalone_gate_on_cpu() {
+ let context = <Cpu as Backend>::Context::new().expect("CPU attention context");
+ check_attention_gate::<Cpu>(context.as_ref(), 3, 29);
+}
+
+#[uzu_test]
+fn attention_gate_matches_standalone_gate_on_metal() {
+ let context = <Metal as Backend>::Context::new().expect("Metal attention context");
+ for (suffix_length, prefix_length) in
+ [(1, 100), (1, 2000), (200, 800), (2, 1022), (16, 16368), (65, 959), (65, 16319), (1024, 0)]
+ {
+ check_attention_gate::<Metal>(context.as_ref(), suffix_length, prefix_length);
+ }
+}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs
index 888095db5..f13b664d7 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs
@@ -17,6 +17,7 @@ fn default_attention_config<T: ArrayElement>(
num_groups: num_groups as u32,
num_q_heads: num_q_heads as u32,
has_sinks: false,
+ gate_row_stride: None,
is_kv_cache_ring: false,
is_causal,
sliding_window_size: None,