Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

remove attention fallback (#1050)

CCYeh7eaf228ce5a8
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
index 085e22e6a..743b4bcb5 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
@@ -18,7 +18,6 @@ mod pooling;
mod radix_top_k_small;
mod sampling;
mod short_conv;
-mod softmax;
mod tensor_add_bias;
mod tensor_add_scale;
mod weaver;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/softmax/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/softmax/mod.rs
deleted file mode 100644
index c85937510..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/softmax/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub mod softmax;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/softmax/softmax.rs b/crates/uzu-engine/src/backends/cpu/kernel/softmax/softmax.rs
deleted file mode 100644
index 2c888f85a..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/softmax/softmax.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::data_type::DataTypeRepr;
-
-#[kernel(Softmax)]
-#[variants(T, f32, f16, bf16)]
-pub fn softmax<T: DataTypeRepr + Float>(
- values: *mut T,
- #[optional(has_sinks)] sinks: Option<*const T>,
- row_length: u32,
- outer_dim: u32,
- batch_dim: u32,
- #[specialize] has_sinks: bool,
-) {
- for outer_index in 0..outer_dim {
- for batch_index in 0..batch_dim {
- let row_offset = ((outer_index * batch_dim + batch_index) * row_length) as usize;
- let sink = if has_sinks {
- unsafe { *sinks.unwrap().add(outer_index as usize) }.to_f32().unwrap()
- } else {
- f32::NEG_INFINITY
- };
-
- let mut max_value = if has_sinks {
- sink
- } else {
- f32::NEG_INFINITY
- };
- for i in 0..row_length as usize {
- max_value = max_value.max(unsafe { *values.add(row_offset + i) }.to_f32().unwrap());
- }
-
- let mut sum_exp = if has_sinks {
- (sink - max_value).exp()
- } else {
- 0.0
- };
- for i in 0..row_length as usize {
- let p = (unsafe { *values.add(row_offset + i) }.to_f32().unwrap() - max_value).exp();
- sum_exp += p;
- unsafe {
- *values.add(row_offset + i) = T::from(p).unwrap();
- }
- }
-
- let inv = 1.0 / sum_exp;
- for i in 0..row_length as usize {
- unsafe {
- let p = *values.add(row_offset + i);
- *values.add(row_offset + i) = T::from(p.to_f32().unwrap() * inv).unwrap();
- }
- }
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal
deleted file mode 100644
index 4af0b3324..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal
+++ /dev/null
@@ -1,80 +0,0 @@
-#include <metal_stdlib>
-#include <metal_simdgroup>
-#include "../common/defines.h"
-#include "../common/dsl.h"
-#include "../common/thread_context.h"
-#include "../generated/ring.h"
-#include "../generated/trie.h"
-#include "mask.h"
-
-using namespace uzu::ring;
-using namespace uzu::trie;
-
-template <typename T>
-VARIANTS(T, bfloat)
-KERNEL(AttentionFallbackScatterScores)(
- const device T* group_scores,
- device T* scores,
- const constant RingParams& ring_params OPTIONAL(is_kv_cache_ring),
- const device TrieNode* trie OPTIONAL(is_trie),
- const constant uint& sliding_window_size OPTIONAL(is_sliding_window),
- const constant uint& group_index,
- const constant uint& gqa_factor,
- const constant uint& sequence_length,
- const constant uint& suffix_length,
- const constant uint& total_elements,
- const bool is_kv_cache_ring SPECIALIZE,
- const bool is_causal SPECIALIZE,
- const bool is_trie SPECIALIZE,
- const bool is_sliding_window SPECIALIZE,
- const uint group_score_index AXIS(total_elements, 256)
-) {
- const uint sequence_index = group_score_index % sequence_length;
- const uint row_index = group_score_index / sequence_length;
- const uint query_index = row_index % suffix_length;
- const uint head_in_group = row_index / suffix_length;
- const uint head_index = group_index * gqa_factor + head_in_group;
-
- const uint prefix_length = sequence_length - suffix_length;
- const uint suffix_position = is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;
- const uint query_position = is_trie ? suffix_position + trie[query_index].height : suffix_position + query_index;
-
- const bool use_key = should_use_key(
- ring_params,
- trie,
- sliding_window_size,
- query_index,
- prefix_length,
- suffix_position,
- query_position,
- sequence_index,
- is_kv_cache_ring,
- is_causal,
- is_trie,
- is_sliding_window
- );
-
- scores[(head_index * suffix_length + query_index) * sequence_length + sequence_index] =
- use_key ? group_scores[group_score_index] : T(-INFINITY);
-}
-
-template <typename T>
-VARIANTS(T, bfloat)
-KERNEL(AttentionFallbackScatterValues)(
- const device T* group_output,
- device T* out,
- const constant uint& group_index,
- const constant uint& gqa_factor,
- const constant uint& suffix_length,
- const constant uint& num_heads,
- const constant uint& head_dim,
- const constant uint& total_elements,
- const uint group_output_index AXIS(total_elements, 256)
-) {
- const uint dim_index = group_output_index % head_dim;
- const uint row_index = group_output_index / head_dim;
- const uint query_index = row_index % suffix_length;
- const uint head_in_group = row_index / suffix_length;
- const uint head_index = group_index * gqa_factor + head_in_group;
- out[(query_index * num_heads + head_index) * head_dim + dim_index] = group_output[group_output_index];
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/error.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/error.rs
index bc08b4091..c5d8a3a87 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/error.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/error.rs
@@ -1,6 +1,6 @@
use thiserror::Error;
-use crate::backends::metal::{error::MetalError, kernel::matmul::MatmulMetalError};
+use crate::backends::metal::error::MetalError;
#[derive(Debug, Error)]
pub enum AttentionMetalError {
@@ -8,6 +8,4 @@ pub enum AttentionMetalError {
UnsupportedInt8KV,
#[error("Backend error: {0}")]
Backend(#[from] MetalError),
- #[error("Matmul error: {0}")]
- Matmul(#[from] MatmulMetalError),
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
deleted file mode 100644
index df9b885cf..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
+++ /dev/null
@@ -1,194 +0,0 @@
-use parking_lot::Mutex;
-
-use super::FullPrecisionInput;
-use crate::{
- backends::{
- common::{
- Backend, BufferRef, CommandBufferEncoding, Kernels,
- kernel::{
- AttentionArguments, AttentionKernelConfig, SoftmaxKernel,
- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput},
- },
- },
- metal::{
- Metal,
- command_buffer::MetalCommandBufferEncoding,
- context::MetalContext,
- kernel::{
- AttentionFallbackScatterScoresMetalKernel, AttentionFallbackScatterValuesMetalKernel, MetalKernels,
- attention::AttentionMetalError,
- },
- },
- },
- data_type::DataType,
-};
-
-const HEAD_DIM: u32 = 512;
-
-pub struct AttentionFallback {
- head_dim: u32,
- num_groups: u32,
- num_q_heads: u32,
- sliding_window_size: Option<u32>,
- scale: Option<f32>,
- data_type: DataType,
- scatter_scores: AttentionFallbackScatterScoresMetalKernel,
- scatter_values: AttentionFallbackScatterValuesMetalKernel,
- softmax: <MetalKernels as Kernels>::SoftmaxKernel,
- matmul: Mutex<<MetalKernels as Kernels>::MatmulKernel>,
-}
-
-impl AttentionFallback {
- pub fn is_supported(config: &AttentionKernelConfig) -> bool {
- config.head_dim == HEAD_DIM && config.data_type == DataType::BF16
- }
-
- pub fn new(
- config: &AttentionKernelConfig,
- context: &MetalContext,
- ) -> Result<Self, AttentionMetalError> {
- let scatter_scores = AttentionFallbackScatterScoresMetalKernel::new(
- context,
- config.data_type,
- config.is_kv_cache_ring,
- config.is_causal,
- false,
- config.sliding_window_size.is_some(),
- )?;
- let scatter_values = AttentionFallbackScatterValuesMetalKernel::new(context, config.data_type)?;
- let softmax = <<MetalKernels as Kernels>::SoftmaxKernel as SoftmaxKernel>::new(
- context,
- config.data_type,
- config.has_sinks,
- )?;
- let matmul = Mutex::new(<<MetalKernels as Kernels>::MatmulKernel as MatmulKernel>::new(
- context,
- config.data_type,
- config.data_type,
- config.data_type,
- )?);
- Ok(Self {
- head_dim: config.head_dim,
- num_groups: config.num_groups,
- num_q_heads: config.num_q_heads,
- sliding_window_size: config.sliding_window_size,
- scale: config.scale,
- data_type: config.data_type,
- scatter_scores,
- scatter_values,
- softmax,
- matmul,
- })
- }
-
- pub fn encode(
- &self,
- input: FullPrecisionInput<
- impl BufferRef<Backend = Metal>,
- impl BufferRef<Backend = Metal>,
- impl BufferRef<Backend = Metal>,
- >,
- arguments: AttentionArguments<'_, Metal, impl BufferRef<Backend = Metal>, impl BufferRef<Backend = Metal>>,
- command_buffer: &mut MetalCommandBufferEncoding,
- ) -> Result<<Metal as Backend>::ScratchBuffer, AttentionMetalError> {
- assert!(arguments.trie.is_none(), "fallback does not support trie");
- let suffix_length = arguments.suffix_length;
- let sequence_length = arguments.cache.prefix_len() + suffix_length;
- let gqa_factor = self.num_q_heads / self.num_groups;
- let scale = self.scale.unwrap_or(1.0 / (self.head_dim as f32).sqrt());
- let dt_bytes = self.data_type.size_in_bytes();
- let head_dim_bytes = self.head_dim as usize * dt_bytes;
- let group_rows = (gqa_factor * suffix_length) as usize;
- let mut output = command_buffer
- .allocate_scratch_for_shape(&[suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;
- let mut scores = command_buffer
- .allocate_scratch_for_shape(&[self.num_q_heads, suffix_length, sequence_length], self.data_type)?;
- let mut group_scores = command_buffer
- .allocate_scratch_for_shape(&[gqa_factor * suffix_length, sequence_length], self.data_type)?;
-
- for group_index in 0..self.num_groups {
- self.matmul.lock().encode(
- MatmulArguments {
- a: MatmulA::FullPrecision {
- values: input.queries,
- offset: group_index as usize * group_rows * head_dim_bytes,
- },
- b: MatmulB::FullPrecision {
- b: input.keys.subrange(group_index as usize * head_dim_bytes..),
- },
- b_leading_dimension: Some(self.num_groups * self.head_dim),
- b_transpose: true,
- output: MatmulOutput::new(
- &mut group_scores,
- MatmulDOps {
- ab_scale: scale,
- ..MatmulDOps::none()
- },
- ),
- gather_indices: None::<&<Metal as Backend>::ScratchBuffer>,
- m: gqa_factor * suffix_length,
- n: sequence_length,
- k: self.head_dim,
- },
- command_buffer,
- )?;
- self.scatter_scores.encode(
- &group_scores,
- &mut scores,
- arguments.cache.ring_params(),
- None::<&<Metal as Backend>::ConstantBuffer>,
- self.sliding_window_size,
- group_index,
- gqa_factor,
- sequence_length,
- suffix_length,
- gqa_factor * suffix_length * sequence_length,
- command_buffer,
- );
- }
-
- self.softmax.encode(
- &mut scores,
- arguments.sinks,
- sequence_length,
- self.num_q_heads,
- suffix_length,
- command_buffer,
- );
- let mut group_output =
- command_buffer.allocate_scratch_for_shape(&[gqa_factor * suffix_length, self.head_dim], self.data_type)?;
- for group_index in 0..self.num_groups {
- self.matmul.lock().encode(
- MatmulArguments {
- a: MatmulA::FullPrecision {
- values: &scores,
- offset: group_index as usize * group_rows * sequence_length as usize * dt_bytes,
- },
- b: MatmulB::FullPrecision {
- b: input.values.subrange(group_index as usize * head_dim_bytes..),
- },
- b_leading_dimension: Some(self.num_groups * self.head_dim),
- b_transpose: false,
- output: MatmulOutput::new(&mut group_output, MatmulDOps::none()),
- gather_indices: None::<&<Metal as Backend>::ScratchBuffer>,
- m: gqa_factor * suffix_length,
- n: self.head_dim,
- k: sequence_length,
- },
- command_buffer,
- )?;
- self.scatter_values.encode(
- &group_output,
- &mut output,
- group_index,
- gqa_factor,
- suffix_length,
- self.num_q_heads,
- self.head_dim,
- gqa_factor * suffix_length * self.head_dim,
- command_buffer,
- );
- }
- Ok(output)
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
index 1c9270a93..bce8d8e70 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
@@ -1,7 +1,6 @@
use crate::backends::common::Backend;
mod error;
-mod fallback;
mod gemm;
mod gemm_grouped;
mod int8;
@@ -58,7 +57,6 @@ pub struct AttentionMetalKernel {
grouped: Option<gemm_grouped::AttentionGemmGrouped>,
int8: Option<int8::Int8Attention>,
gemm: Option<gemm::AttentionGemm>,
- fallback: Option<fallback::AttentionFallback>,
two_pass: two_pass::AttentionTwoPass,
single_pass: single_pass::AttentionSinglePass,
}
@@ -78,9 +76,6 @@ impl AttentionKernel for AttentionMetalKernel {
.transpose()
.map_err(AttentionMetalError::Backend)?;
let gemm = gemm::AttentionGemm::is_supported(&config).then(|| gemm::AttentionGemm::new(&config));
- let fallback = fallback::AttentionFallback::is_supported(&config)
- .then(|| fallback::AttentionFallback::new(&config, context))
- .transpose()?;
let gate = config
.gate_row_stride
.map(|stride| SigmoidGateMetalKernel::new(context, config.data_type).map(|kernel| (kernel, stride)))
@@ -91,7 +86,6 @@ impl AttentionKernel for AttentionMetalKernel {
grouped,
int8,
gemm,
- fallback,
two_pass: two_pass::AttentionTwoPass::new(&config),
single_pass: single_pass::AttentionSinglePass::new(&config),
})
@@ -169,13 +163,10 @@ impl AttentionMetalKernel {
return grouped.encode(mask, input, arguments, command_buffer).map_err(AttentionMetalError::Backend);
}
- if suffix_length > 8 {
- if let Some(gemm) = &self.gemm {
- return Ok((gemm.encode(input, arguments, command_buffer)?, false));
- }
- if !is_trie && let Some(fallback) = &self.fallback {
- return Ok((fallback.encode(input, arguments, command_buffer)?, false));
- }
+ if suffix_length > 8
+ && let Some(gemm) = &self.gemm
+ {
+ return Ok((gemm.encode(input, arguments, command_buffer)?, false));
}
if kv_length > SINGLE_PASS_KV_THRESHOLD {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/softmax/softmax.metal b/crates/uzu-engine/src/backends/metal/kernel/softmax/softmax.metal
deleted file mode 100644
index 3cb83c001..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/softmax/softmax.metal
+++ /dev/null
@@ -1,95 +0,0 @@
-#include <metal_stdlib>
-
-#include "../common/defines.h"
-#include "../common/dsl.h"
-
-// Online row-wise softmax (MLX-style: simd intrinsics + 2-pass online
-// algorithm). `has_sinks` seeds thread 0 with (sinks[outer_index], 1) as a
-// virtual element.
-
-#define SOFTMAX_THREADS 256
-// Number of consecutive elements each thread loads per iteration. Vectorizing
-// the per-thread loop this way improves memory coalescing on Apple Silicon.
-#define SOFTMAX_ELEMENTS_PER_THREAD 4
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(Softmax)(
- device T* values,
- const device T* sinks OPTIONAL(has_sinks),
- const constant uint& row_length,
- const constant uint& outer_dim,
- const constant uint& batch_dim,
- threadgroup float shared_max[METAL_SIMD_SIZE],
- threadgroup float shared_norm[METAL_SIMD_SIZE],
- const bool has_sinks SPECIALIZE,
- const uint outer_index GROUPS(outer_dim),
- const uint batch_index GROUPS(batch_dim),
- const uint tid THREADS(SOFTMAX_THREADS)
-) {
- const uint lane = tid & (METAL_SIMD_SIZE - 1);
- const uint simdgroup_id = tid / METAL_SIMD_SIZE;
- const uint stride = SOFTMAX_THREADS * SOFTMAX_ELEMENTS_PER_THREAD;
- device T* row = values + (outer_index * batch_dim + batch_index) * row_length;
-
- float maxval = (has_sinks && tid == 0) ? sinks[outer_index] : -FLT_MAX;
- float norm = (has_sinks && tid == 0) ? 1.0f : 0.0f;
-
- if (simdgroup_id == 0) {
- shared_max[lane] = -FLT_MAX;
- shared_norm[lane] = 0.0f;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Pass 1: online (max, normalizer) accumulation over the row.
- for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length; base += stride) {
- float vals[SOFTMAX_ELEMENTS_PER_THREAD];
- for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {
- vals[i] = (base + i < row_length) ? float(row[base + i]) : -FLT_MAX;
- }
- float prev_max = maxval;
- for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {
- maxval = max(maxval, vals[i]);
- }
- norm *= fast::exp(prev_max - maxval);
- for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {
- norm += fast::exp(vals[i] - maxval);
- }
- }
-
- // Combine across the simdgroup, then across simdgroups via shared memory.
- float prev_max = maxval;
- maxval = simd_max(maxval);
- norm *= fast::exp(prev_max - maxval);
- norm = simd_sum(norm);
-
- if (lane == 0) {
- shared_max[simdgroup_id] = maxval;
- shared_norm[simdgroup_id] = norm;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (simdgroup_id == 0) {
- float simdgroup_max = shared_max[lane];
- float simdgroup_norm = shared_norm[lane];
- float global_max = simd_max(simdgroup_max);
- simdgroup_norm *= fast::exp(simdgroup_max - global_max);
- float global_norm = simd_sum(simdgroup_norm);
- if (lane == 0) {
- shared_max[0] = global_max;
- shared_norm[0] = global_norm;
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const float final_max = shared_max[0];
- const float inv_norm = 1.0f / shared_norm[0];
-
- // Pass 2: normalize.
- for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length; base += stride) {
- for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {
- if (base + i < row_length) {
- row[base + i] = T(fast::exp(float(row[base + i]) - final_max) * inv_norm);
- }
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/slang/kernel/softmax/softmax.slang b/crates/uzu-engine/src/backends/slang/kernel/softmax/softmax.slang
deleted file mode 100644
index 0a422c4ca..000000000
--- a/crates/uzu-engine/src/backends/slang/kernel/softmax/softmax.slang
+++ /dev/null
@@ -1,17 +0,0 @@
-// Autogenerated stub
-import definitions;
-
-[[Variants("T", "float, half, BFloat16")]]
-[[Public]] [[Kernel]] func Softmax<T>(
- RWStructuredBuffer<T> values,
- [[Optional("has_sinks")]] StructuredBuffer<T> sinks,
- uint row_length,
- uint outer_dim,
- uint batch_dim,
- [[Specialize]] bool has_sinks,
- [[Groups("outer_dim")]] uint outer_index,
- [[Groups("batch_dim")]] uint batch_index,
- [[Threads("256")]] uint tid,
-) {
- // TODO
-}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
index 86b5f3d72..aa8679a27 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
@@ -112,7 +112,6 @@ fn attention_kernel_matches_cpu() {
let cpu_context = block_on(<Cpu as Backend>::Context::new()).expect("CPU attention context");
let metal_context = block_on(<Metal as Backend>::Context::new()).expect("Metal attention context");
for &(head_dim, num_q_heads, num_groups, suffix_length, prefix_length, causal, use_trie) in &[
- (512, 8, 8, 9, 0, false, false),
(128, 8, 2, 16, 1024, false, false),
(256, 6, 1, 2, 1024, false, false),
(256, 6, 1, 15, 2048, true, false),