A high-performance inference engine for AI models
remove attention fallback (#1050)
CCYeh7eaf228ce5a8
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/mod.rsindex 085e22e6a..743b4bcb5 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs@@ -18,7 +18,6 @@ mod pooling;mod radix_top_k_small;mod sampling;mod short_conv;-mod softmax;mod tensor_add_bias;mod tensor_add_scale;mod weaver;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/softmax/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/softmax/mod.rsdeleted file mode 100644index c85937510..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/softmax/mod.rs+++ /dev/null@@ -1 +0,0 @@-pub mod softmax;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/softmax/softmax.rs b/crates/uzu-engine/src/backends/cpu/kernel/softmax/softmax.rsdeleted file mode 100644index 2c888f85a..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/softmax/softmax.rs+++ /dev/null@@ -1,57 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::data_type::DataTypeRepr;--#[kernel(Softmax)]-#[variants(T, f32, f16, bf16)]-pub fn softmax<T: DataTypeRepr + Float>(- values: *mut T,- #[optional(has_sinks)] sinks: Option<*const T>,- row_length: u32,- outer_dim: u32,- batch_dim: u32,- #[specialize] has_sinks: bool,-) {- for outer_index in 0..outer_dim {- for batch_index in 0..batch_dim {- let row_offset = ((outer_index * batch_dim + batch_index) * row_length) as usize;- let sink = if has_sinks {- unsafe { *sinks.unwrap().add(outer_index as usize) }.to_f32().unwrap()- } else {- f32::NEG_INFINITY- };-- let mut max_value = if has_sinks {- sink- } else {- f32::NEG_INFINITY- };- for i in 0..row_length as usize {- max_value = max_value.max(unsafe { *values.add(row_offset + i) }.to_f32().unwrap());- }-- let mut sum_exp = if has_sinks {- (sink - max_value).exp()- } else {- 0.0- };- for i in 0..row_length as usize {- let p = (unsafe { *values.add(row_offset + i) }.to_f32().unwrap() - max_value).exp();- sum_exp += p;- unsafe {- *values.add(row_offset + i) = T::from(p).unwrap();- }- }-- let inv = 1.0 / sum_exp;- for i in 0..row_length as usize {- unsafe {- let p = *values.add(row_offset + i);- *values.add(row_offset + i) = T::from(p.to_f32().unwrap() * inv).unwrap();- }- }- }- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metaldeleted file mode 100644index 4af0b3324..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal+++ /dev/null@@ -1,80 +0,0 @@-#include <metal_stdlib>-#include <metal_simdgroup>-#include "../common/defines.h"-#include "../common/dsl.h"-#include "../common/thread_context.h"-#include "../generated/ring.h"-#include "../generated/trie.h"-#include "mask.h"--using namespace uzu::ring;-using namespace uzu::trie;--template <typename T>-VARIANTS(T, bfloat)-KERNEL(AttentionFallbackScatterScores)(- const device T* group_scores,- device T* scores,- const constant RingParams& ring_params OPTIONAL(is_kv_cache_ring),- const device TrieNode* trie OPTIONAL(is_trie),- const constant uint& sliding_window_size OPTIONAL(is_sliding_window),- const constant uint& group_index,- const constant uint& gqa_factor,- const constant uint& sequence_length,- const constant uint& suffix_length,- const constant uint& total_elements,- const bool is_kv_cache_ring SPECIALIZE,- const bool is_causal SPECIALIZE,- const bool is_trie SPECIALIZE,- const bool is_sliding_window SPECIALIZE,- const uint group_score_index AXIS(total_elements, 256)-) {- const uint sequence_index = group_score_index % sequence_length;- const uint row_index = group_score_index / sequence_length;- const uint query_index = row_index % suffix_length;- const uint head_in_group = row_index / suffix_length;- const uint head_index = group_index * gqa_factor + head_in_group;-- const uint prefix_length = sequence_length - suffix_length;- const uint suffix_position = is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;- const uint query_position = is_trie ? suffix_position + trie[query_index].height : suffix_position + query_index;-- const bool use_key = should_use_key(- ring_params,- trie,- sliding_window_size,- query_index,- prefix_length,- suffix_position,- query_position,- sequence_index,- is_kv_cache_ring,- is_causal,- is_trie,- is_sliding_window- );-- scores[(head_index * suffix_length + query_index) * sequence_length + sequence_index] =- use_key ? group_scores[group_score_index] : T(-INFINITY);-}--template <typename T>-VARIANTS(T, bfloat)-KERNEL(AttentionFallbackScatterValues)(- const device T* group_output,- device T* out,- const constant uint& group_index,- const constant uint& gqa_factor,- const constant uint& suffix_length,- const constant uint& num_heads,- const constant uint& head_dim,- const constant uint& total_elements,- const uint group_output_index AXIS(total_elements, 256)-) {- const uint dim_index = group_output_index % head_dim;- const uint row_index = group_output_index / head_dim;- const uint query_index = row_index % suffix_length;- const uint head_in_group = row_index / suffix_length;- const uint head_index = group_index * gqa_factor + head_in_group;- out[(query_index * num_heads + head_index) * head_dim + dim_index] = group_output[group_output_index];-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/error.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/error.rsindex bc08b4091..c5d8a3a87 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/error.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/error.rs@@ -1,6 +1,6 @@use thiserror::Error;-use crate::backends::metal::{error::MetalError, kernel::matmul::MatmulMetalError};+use crate::backends::metal::error::MetalError;#[derive(Debug, Error)]pub enum AttentionMetalError {@@ -8,6 +8,4 @@ pub enum AttentionMetalError {UnsupportedInt8KV,#[error("Backend error: {0}")]Backend(#[from] MetalError),- #[error("Matmul error: {0}")]- Matmul(#[from] MatmulMetalError),}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rsdeleted file mode 100644index df9b885cf..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs+++ /dev/null@@ -1,194 +0,0 @@-use parking_lot::Mutex;--use super::FullPrecisionInput;-use crate::{- backends::{- common::{- Backend, BufferRef, CommandBufferEncoding, Kernels,- kernel::{- AttentionArguments, AttentionKernelConfig, SoftmaxKernel,- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput},- },- },- metal::{- Metal,- command_buffer::MetalCommandBufferEncoding,- context::MetalContext,- kernel::{- AttentionFallbackScatterScoresMetalKernel, AttentionFallbackScatterValuesMetalKernel, MetalKernels,- attention::AttentionMetalError,- },- },- },- data_type::DataType,-};--const HEAD_DIM: u32 = 512;--pub struct AttentionFallback {- head_dim: u32,- num_groups: u32,- num_q_heads: u32,- sliding_window_size: Option<u32>,- scale: Option<f32>,- data_type: DataType,- scatter_scores: AttentionFallbackScatterScoresMetalKernel,- scatter_values: AttentionFallbackScatterValuesMetalKernel,- softmax: <MetalKernels as Kernels>::SoftmaxKernel,- matmul: Mutex<<MetalKernels as Kernels>::MatmulKernel>,-}--impl AttentionFallback {- pub fn is_supported(config: &AttentionKernelConfig) -> bool {- config.head_dim == HEAD_DIM && config.data_type == DataType::BF16- }-- pub fn new(- config: &AttentionKernelConfig,- context: &MetalContext,- ) -> Result<Self, AttentionMetalError> {- let scatter_scores = AttentionFallbackScatterScoresMetalKernel::new(- context,- config.data_type,- config.is_kv_cache_ring,- config.is_causal,- false,- config.sliding_window_size.is_some(),- )?;- let scatter_values = AttentionFallbackScatterValuesMetalKernel::new(context, config.data_type)?;- let softmax = <<MetalKernels as Kernels>::SoftmaxKernel as SoftmaxKernel>::new(- context,- config.data_type,- config.has_sinks,- )?;- let matmul = Mutex::new(<<MetalKernels as Kernels>::MatmulKernel as MatmulKernel>::new(- context,- config.data_type,- config.data_type,- config.data_type,- )?);- Ok(Self {- head_dim: config.head_dim,- num_groups: config.num_groups,- num_q_heads: config.num_q_heads,- sliding_window_size: config.sliding_window_size,- scale: config.scale,- data_type: config.data_type,- scatter_scores,- scatter_values,- softmax,- matmul,- })- }-- pub fn encode(- &self,- input: FullPrecisionInput<- impl BufferRef<Backend = Metal>,- impl BufferRef<Backend = Metal>,- impl BufferRef<Backend = Metal>,- >,- arguments: AttentionArguments<'_, Metal, impl BufferRef<Backend = Metal>, impl BufferRef<Backend = Metal>>,- command_buffer: &mut MetalCommandBufferEncoding,- ) -> Result<<Metal as Backend>::ScratchBuffer, AttentionMetalError> {- assert!(arguments.trie.is_none(), "fallback does not support trie");- let suffix_length = arguments.suffix_length;- let sequence_length = arguments.cache.prefix_len() + suffix_length;- let gqa_factor = self.num_q_heads / self.num_groups;- let scale = self.scale.unwrap_or(1.0 / (self.head_dim as f32).sqrt());- let dt_bytes = self.data_type.size_in_bytes();- let head_dim_bytes = self.head_dim as usize * dt_bytes;- let group_rows = (gqa_factor * suffix_length) as usize;- let mut output = command_buffer- .allocate_scratch_for_shape(&[suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;- let mut scores = command_buffer- .allocate_scratch_for_shape(&[self.num_q_heads, suffix_length, sequence_length], self.data_type)?;- let mut group_scores = command_buffer- .allocate_scratch_for_shape(&[gqa_factor * suffix_length, sequence_length], self.data_type)?;-- for group_index in 0..self.num_groups {- self.matmul.lock().encode(- MatmulArguments {- a: MatmulA::FullPrecision {- values: input.queries,- offset: group_index as usize * group_rows * head_dim_bytes,- },- b: MatmulB::FullPrecision {- b: input.keys.subrange(group_index as usize * head_dim_bytes..),- },- b_leading_dimension: Some(self.num_groups * self.head_dim),- b_transpose: true,- output: MatmulOutput::new(- &mut group_scores,- MatmulDOps {- ab_scale: scale,- ..MatmulDOps::none()- },- ),- gather_indices: None::<&<Metal as Backend>::ScratchBuffer>,- m: gqa_factor * suffix_length,- n: sequence_length,- k: self.head_dim,- },- command_buffer,- )?;- self.scatter_scores.encode(- &group_scores,- &mut scores,- arguments.cache.ring_params(),- None::<&<Metal as Backend>::ConstantBuffer>,- self.sliding_window_size,- group_index,- gqa_factor,- sequence_length,- suffix_length,- gqa_factor * suffix_length * sequence_length,- command_buffer,- );- }-- self.softmax.encode(- &mut scores,- arguments.sinks,- sequence_length,- self.num_q_heads,- suffix_length,- command_buffer,- );- let mut group_output =- command_buffer.allocate_scratch_for_shape(&[gqa_factor * suffix_length, self.head_dim], self.data_type)?;- for group_index in 0..self.num_groups {- self.matmul.lock().encode(- MatmulArguments {- a: MatmulA::FullPrecision {- values: &scores,- offset: group_index as usize * group_rows * sequence_length as usize * dt_bytes,- },- b: MatmulB::FullPrecision {- b: input.values.subrange(group_index as usize * head_dim_bytes..),- },- b_leading_dimension: Some(self.num_groups * self.head_dim),- b_transpose: false,- output: MatmulOutput::new(&mut group_output, MatmulDOps::none()),- gather_indices: None::<&<Metal as Backend>::ScratchBuffer>,- m: gqa_factor * suffix_length,- n: self.head_dim,- k: sequence_length,- },- command_buffer,- )?;- self.scatter_values.encode(- &group_output,- &mut output,- group_index,- gqa_factor,- suffix_length,- self.num_q_heads,- self.head_dim,- gqa_factor * suffix_length * self.head_dim,- command_buffer,- );- }- Ok(output)- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rsindex 1c9270a93..bce8d8e70 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs@@ -1,7 +1,6 @@use crate::backends::common::Backend;mod error;-mod fallback;mod gemm;mod gemm_grouped;mod int8;@@ -58,7 +57,6 @@ pub struct AttentionMetalKernel {grouped: Option<gemm_grouped::AttentionGemmGrouped>,int8: Option<int8::Int8Attention>,gemm: Option<gemm::AttentionGemm>,- fallback: Option<fallback::AttentionFallback>,two_pass: two_pass::AttentionTwoPass,single_pass: single_pass::AttentionSinglePass,}@@ -78,9 +76,6 @@ impl AttentionKernel for AttentionMetalKernel {.transpose().map_err(AttentionMetalError::Backend)?;let gemm = gemm::AttentionGemm::is_supported(&config).then(|| gemm::AttentionGemm::new(&config));- let fallback = fallback::AttentionFallback::is_supported(&config)- .then(|| fallback::AttentionFallback::new(&config, context))- .transpose()?;let gate = config.gate_row_stride.map(|stride| SigmoidGateMetalKernel::new(context, config.data_type).map(|kernel| (kernel, stride)))@@ -91,7 +86,6 @@ impl AttentionKernel for AttentionMetalKernel {grouped,int8,gemm,- fallback,two_pass: two_pass::AttentionTwoPass::new(&config),single_pass: single_pass::AttentionSinglePass::new(&config),})@@ -169,13 +163,10 @@ impl AttentionMetalKernel {return grouped.encode(mask, input, arguments, command_buffer).map_err(AttentionMetalError::Backend);}- if suffix_length > 8 {- if let Some(gemm) = &self.gemm {- return Ok((gemm.encode(input, arguments, command_buffer)?, false));- }- if !is_trie && let Some(fallback) = &self.fallback {- return Ok((fallback.encode(input, arguments, command_buffer)?, false));- }+ if suffix_length > 8+ && let Some(gemm) = &self.gemm+ {+ return Ok((gemm.encode(input, arguments, command_buffer)?, false));}if kv_length > SINGLE_PASS_KV_THRESHOLD {diff --git a/crates/uzu-engine/src/backends/metal/kernel/softmax/softmax.metal b/crates/uzu-engine/src/backends/metal/kernel/softmax/softmax.metaldeleted file mode 100644index 3cb83c001..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/softmax/softmax.metal+++ /dev/null@@ -1,95 +0,0 @@-#include <metal_stdlib>--#include "../common/defines.h"-#include "../common/dsl.h"--// Online row-wise softmax (MLX-style: simd intrinsics + 2-pass online-// algorithm). `has_sinks` seeds thread 0 with (sinks[outer_index], 1) as a-// virtual element.--#define SOFTMAX_THREADS 256-// Number of consecutive elements each thread loads per iteration. Vectorizing-// the per-thread loop this way improves memory coalescing on Apple Silicon.-#define SOFTMAX_ELEMENTS_PER_THREAD 4--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(Softmax)(- device T* values,- const device T* sinks OPTIONAL(has_sinks),- const constant uint& row_length,- const constant uint& outer_dim,- const constant uint& batch_dim,- threadgroup float shared_max[METAL_SIMD_SIZE],- threadgroup float shared_norm[METAL_SIMD_SIZE],- const bool has_sinks SPECIALIZE,- const uint outer_index GROUPS(outer_dim),- const uint batch_index GROUPS(batch_dim),- const uint tid THREADS(SOFTMAX_THREADS)-) {- const uint lane = tid & (METAL_SIMD_SIZE - 1);- const uint simdgroup_id = tid / METAL_SIMD_SIZE;- const uint stride = SOFTMAX_THREADS * SOFTMAX_ELEMENTS_PER_THREAD;- device T* row = values + (outer_index * batch_dim + batch_index) * row_length;-- float maxval = (has_sinks && tid == 0) ? sinks[outer_index] : -FLT_MAX;- float norm = (has_sinks && tid == 0) ? 1.0f : 0.0f;-- if (simdgroup_id == 0) {- shared_max[lane] = -FLT_MAX;- shared_norm[lane] = 0.0f;- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Pass 1: online (max, normalizer) accumulation over the row.- for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length; base += stride) {- float vals[SOFTMAX_ELEMENTS_PER_THREAD];- for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {- vals[i] = (base + i < row_length) ? float(row[base + i]) : -FLT_MAX;- }- float prev_max = maxval;- for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {- maxval = max(maxval, vals[i]);- }- norm *= fast::exp(prev_max - maxval);- for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {- norm += fast::exp(vals[i] - maxval);- }- }-- // Combine across the simdgroup, then across simdgroups via shared memory.- float prev_max = maxval;- maxval = simd_max(maxval);- norm *= fast::exp(prev_max - maxval);- norm = simd_sum(norm);-- if (lane == 0) {- shared_max[simdgroup_id] = maxval;- shared_norm[simdgroup_id] = norm;- }- threadgroup_barrier(mem_flags::mem_threadgroup);- if (simdgroup_id == 0) {- float simdgroup_max = shared_max[lane];- float simdgroup_norm = shared_norm[lane];- float global_max = simd_max(simdgroup_max);- simdgroup_norm *= fast::exp(simdgroup_max - global_max);- float global_norm = simd_sum(simdgroup_norm);- if (lane == 0) {- shared_max[0] = global_max;- shared_norm[0] = global_norm;- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- const float final_max = shared_max[0];- const float inv_norm = 1.0f / shared_norm[0];-- // Pass 2: normalize.- for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length; base += stride) {- for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {- if (base + i < row_length) {- row[base + i] = T(fast::exp(float(row[base + i]) - final_max) * inv_norm);- }- }- }-}diff --git a/crates/uzu-engine/src/backends/slang/kernel/softmax/softmax.slang b/crates/uzu-engine/src/backends/slang/kernel/softmax/softmax.slangdeleted file mode 100644index 0a422c4ca..000000000--- a/crates/uzu-engine/src/backends/slang/kernel/softmax/softmax.slang+++ /dev/null@@ -1,17 +0,0 @@-// Autogenerated stub-import definitions;--[[Variants("T", "float, half, BFloat16")]]-[[Public]] [[Kernel]] func Softmax<T>(- RWStructuredBuffer<T> values,- [[Optional("has_sinks")]] StructuredBuffer<T> sinks,- uint row_length,- uint outer_dim,- uint batch_dim,- [[Specialize]] bool has_sinks,- [[Groups("outer_dim")]] uint outer_index,- [[Groups("batch_dim")]] uint batch_index,- [[Threads("256")]] uint tid,-) {- // TODO-}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rsindex 86b5f3d72..aa8679a27 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs@@ -112,7 +112,6 @@ fn attention_kernel_matches_cpu() {let cpu_context = block_on(<Cpu as Backend>::Context::new()).expect("CPU attention context");let metal_context = block_on(<Metal as Backend>::Context::new()).expect("Metal attention context");for &(head_dim, num_q_heads, num_groups, suffix_length, prefix_length, causal, use_trie) in &[- (512, 8, 8, 9, 0, false, false),(128, 8, 2, 16, 1024, false, false),(256, 6, 1, 2, 1024, false, false),(256, 6, 1, 15, 2048, true, false),