A high-performance inference engine for AI models
support muse_glimmer int8 attention (#971)
CCYeh597e5e5013cb
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/prepare_int8.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/prepare_int8.rsindex ad7150121..180a1c1ac 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/prepare_int8.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/prepare_int8.rs@@ -14,7 +14,7 @@ use crate::{#[kernel(AttentionPrepareInt8)]#[variants(ElementT, bf16)]#[variants(RopeT, f32)]-#[variants(HEAD_DIM, 256)]+#[variants(HEAD_DIM, 128, 256)]pub fn attention_prepare_int8<ElementT: DataTypeRepr + Float, RopeT: DataTypeRepr + Float, const HEAD_DIM: u32>(qkvg: *const ElementT,#[optional(!quantize_queries)] queries: Option<*mut ElementT>,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_int8.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_int8.metalindex 8abe1be58..8fe859f61 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_int8.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_int8.metal@@ -19,7 +19,7 @@ struct Int8Scratch {UZU_CONST int STATS_OFFSET = Layout::SIMDGROUPS * Layout::SCORE_TILE_ELEMENTS / int(sizeof(float));UZU_CONST int SUMS_OFFSET = STATS_OFFSET + Layout::SIMDGROUPS * Layout::TILE_ROWS * 2;UZU_CONST int SLOT_FLOATS = SUMS_OFFSET + Layout::SIMDGROUPS * Layout::TILE_ROWS;- UZU_CONST int TRIE_MASKS = Layout::BLOCK_ROWS;+ UZU_CONST int ROW_MASKS = Layout::BLOCK_ROWS;UZU_CONST int QUERY_VECTORS_PER_BAND = int(HEAD_DIM) / (2 * uzu::kv_layout::TILE_DIMS) * METAL_SIMD_SIZE;UZU_CONST int QUERY_VECTORS = Layout::BANDS * QUERY_VECTORS_PER_BAND;@@ -114,7 +114,7 @@ struct Int8Softmax {template <typename ScaleType, uint HEAD_DIM>VARIANTS(ScaleType, bfloat)-VARIANTS(HEAD_DIM, 256)+VARIANTS(HEAD_DIM, 128, 256)KERNEL(AttentionGemmGroupedInt8)(const device int8_t* query_codes,const device ScaleType* query_scales,@@ -136,7 +136,7 @@ KERNEL(AttentionGemmGroupedInt8)(const bool is_trie SPECIALIZE,const uint32_t num_kv_heads SPECIALIZE,threadgroup float probability_exchange[2 * Int8Scratch<HEAD_DIM>::SLOT_FLOATS],- threadgroup ulong trie_masks[Int8Scratch<HEAD_DIM>::TRIE_MASKS],+ threadgroup ulong row_masks[Int8Scratch<HEAD_DIM>::ROW_MASKS],threadgroup uint4 query_stage[Int8Scratch<HEAD_DIM>::QUERY_VECTORS],const ThreadContext thread_context,const uint tile_split_index GROUPS(query_tiles * split_count),@@ -199,30 +199,30 @@ KERNEL(AttentionGemmGroupedInt8)(uint(score_position.y) + uint(Ops::THREAD_ELEMENT_ROW_STRIDE)};const uint query_band_offset = uint(query_band) * uint(Layout::TILE_ROWS);- uint query_positions[ROWS_PER_LANE];if (!mask_free) {+ ulong row_mask[ROWS_PER_LANE];+ uint query_positions[ROWS_PER_LANE];METAL_PRAGMA_UNROLLfor (ushort row = 0; row < ROWS_PER_LANE; ++row) {query_positions[row] = (query_row_base + lane_rows[row]) % params.q_len;+ row_mask[row] = is_trie ? 0ul : ulong(prefix_length + query_positions[row]);}if (is_trie) {- ulong row_trie_mask[ROWS_PER_LANE] = {};for (uint node_index = 0; node_index < trie_nodes; ++node_index) {const uzu::trie::TrieNode node = trie[node_index];METAL_PRAGMA_UNROLLfor (ushort row = 0; row < ROWS_PER_LANE; ++row) {if (query_positions[row] >= node.trie_start && query_positions[row] <= node.trie_end) {- row_trie_mask[row] |= (1ul << node_index);+ row_mask[row] |= (1ul << node_index);}}}- if (score_position.x == 0) {- METAL_PRAGMA_UNROLL- for (ushort row = 0; row < ROWS_PER_LANE; ++row) {- trie_masks[query_band_offset + lane_rows[row]] = row_trie_mask[row];- }+ }+ if (score_position.x == 0) {+ METAL_PRAGMA_UNROLL+ for (ushort row = 0; row < ROWS_PER_LANE; ++row) {+ row_masks[query_band_offset + lane_rows[row]] = row_mask[row];}- threadgroup_barrier(mem_flags::mem_threadgroup);}}@@ -348,21 +348,19 @@ KERNEL(AttentionGemmGroupedInt8)();if (!mask_free && (is_tail_block || (key_base + uint(uzu::kv_layout::TILE_KEYS) > prefix_length))) {- ulong trie_row_masks[ROWS_PER_LANE] = {0ul, 0ul};- if (is_trie) {- METAL_PRAGMA_UNROLL- for (ushort row = 0; row < ROWS_PER_LANE; ++row) {- trie_row_masks[row] = trie_masks[query_band_offset + lane_rows[row]];- }+ ulong row_mask[ROWS_PER_LANE];+ METAL_PRAGMA_UNROLL+ for (ushort row = 0; row < ROWS_PER_LANE; ++row) {+ row_mask[row] = row_masks[query_band_offset + lane_rows[row]];}map_lane_rows(scores, score_position, [&](ushort lane_row, short column, AccumType value) UZU_ALWAYS_INLINE {const uint key = key_base + uint(column);bool visible = key < key_length;if (is_trie) {- visible = visible &&- (key < prefix_length || ((trie_row_masks[lane_row] >> ulong(key - prefix_length)) & 1ul) != 0ul);+ visible =+ visible && (key < prefix_length || ((row_mask[lane_row] >> ulong(key - prefix_length)) & 1ul) != 0ul);} else {- visible = visible && key <= prefix_length + query_positions[lane_row];+ visible = visible && key <= uint(row_mask[lane_row]);}return visible ? value : -metal::numeric_limits<AccumType>::infinity();});diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine_int8.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine_int8.metalindex 7259b82d6..a97562335 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine_int8.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine_int8.metal@@ -9,7 +9,7 @@ using namespace metal;template <typename T, uint BD>VARIANTS(T, bfloat)-VARIANTS(BD, 256)+VARIANTS(BD, 128, 256)KERNEL(AttentionGemmGroupedInt8Combine)(const device bfloat* partials,const device float* partial_maxs,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rsindex 25ccfad65..c922132e4 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs@@ -29,6 +29,10 @@ const SLICE_COLS: u32 = 64;const TILE_ROWS: u32 = 16;const BLOCK_K: u32 = 32;+pub fn query_rows_per_threadgroup(head_dim: u32) -> u32 {+ SIMDGROUPS_PER_THREADGROUP / (head_dim / SLICE_COLS) * TILE_ROWS+}+struct AttentionGemmGroupedMetal {kernel: AttentionGemmGroupedMetalKernel,split_kernel: AttentionGemmGroupedMetalKernel,@@ -38,7 +42,7 @@ struct AttentionGemmGroupedMetal {num_q_heads: u32,scale: Option<f32>,gate_row_stride: Option<u32>,- block_rows: u32,+ query_rows_per_threadgroup: u32,mask: MaskKind,}@@ -55,7 +59,6 @@ impl AttentionGemmGroupedMetal {assert!(matches!(head_dim, 128 | 256), "head_dim must be 128 or 256");assert_eq!(num_q_heads % num_groups, 0, "num_q_heads must be divisible by num_groups");let slices = head_dim / SLICE_COLS;- let block_rows = SIMDGROUPS_PER_THREADGROUP / slices * TILE_ROWS;let new_kernel = |split| {AttentionGemmGroupedMetalKernel::new(context,@@ -79,7 +82,7 @@ impl AttentionGemmGroupedMetal {num_q_heads,scale,gate_row_stride,- block_rows,+ query_rows_per_threadgroup: query_rows_per_threadgroup(head_dim),mask,})}@@ -99,7 +102,7 @@ impl AttentionGemmGroupedMetal {) -> Result<bool, MetalError> {let params = self.params(suffix_length, kv_length);let grouped_rows = self.num_q_heads / self.num_groups * suffix_length;- let query_tiles = grouped_rows.div_ceil(self.block_rows);+ let query_tiles = grouped_rows.div_ceil(self.query_rows_per_threadgroup);assert_eq!(trie.is_some(), self.mask.is_trie(), "trie presence must match mask");if num_splits == 1 {@@ -121,7 +124,7 @@ impl AttentionGemmGroupedMetal {return Ok(false);}- let partial_rows = self.num_groups * query_tiles * num_splits * self.block_rows;+ let partial_rows = self.num_groups * query_tiles * num_splits * self.query_rows_per_threadgroup;let mut partials = command_buffer.allocate_scratch_for_shape(&[partial_rows, self.head_dim], DataType::F32)?;let mut partial_maxs = command_buffer.allocate_scratch_for_shape(&[partial_rows], DataType::F32)?;let mut partial_sums = command_buffer.allocate_scratch_for_shape(&[partial_rows], DataType::F32)?;@@ -150,7 +153,7 @@ impl AttentionGemmGroupedMetal {gate,params,grouped_rows,- query_tiles * self.block_rows,+ query_tiles * self.query_rows_per_threadgroup,self.num_groups,num_splits,self.gate_row_stride,@@ -277,7 +280,7 @@ impl AttentionGemmGrouped {head_dim: core.head_dim,num_q_heads: core.num_q_heads,num_groups: core.num_groups,- block_rows: core.block_rows,+ query_rows_per_threadgroup: core.query_rows_per_threadgroup,block_k: BLOCK_K,},suffix_length,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rsindex 58b76a6a8..4762d687f 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs@@ -78,7 +78,7 @@ pub struct SplitGeometry {pub head_dim: u32,pub num_q_heads: u32,pub num_groups: u32,- pub block_rows: u32,+ pub query_rows_per_threadgroup: u32,pub block_k: u32,}@@ -123,14 +123,14 @@ pub fn choose_splits(gpu_core_count: u32,) -> u32 {let threadgroups_per_kv_split = (geometry.num_q_heads / geometry.num_groups * suffix_length)- .div_ceil(geometry.block_rows)+ .div_ceil(geometry.query_rows_per_threadgroup)* geometry.num_groups;let max_splits = kv_length.div_ceil(geometry.block_k).max(1);if suffix_length > GEMM_GROUPED_DECODE_SUFFIX_MAX && kv_length >= PREFILL_SPLIT_MIN_KV {// Each padded row has head_dim partial values, one max, and one sum, all F32.let bytes_per_split = threadgroups_per_kv_split as u64- * geometry.block_rows as u64+ * geometry.query_rows_per_threadgroup as u64* (geometry.head_dim as u64 + PARTIAL_STATS_PER_ROW)* DataType::F32.size_in_bytes() as u64;let requested_splits = kv_length.div_ceil(PREFILL_SPLIT_KEYS).min(max_splits);diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rsindex e3f731421..47427026d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rs@@ -1,4 +1,7 @@-use super::gemm_grouped::policy::{MAX_TRIE_SUFFIX, int8_splits};+use super::gemm_grouped::{+ policy::{self, MAX_TRIE_SUFFIX, int8_splits},+ query_rows_per_threadgroup,+};use crate::{backends::{common::{@@ -18,8 +21,6 @@ use crate::{data_type::DataType,};-const QUERIES_PER_BLOCK: u32 = 2 * kv_layout::TILE_QUERIES;-pub struct Int8Attention {config: AttentionKernelConfig,mask_free: AttentionGemmGroupedInt8MetalKernel,@@ -33,12 +34,7 @@ impl Int8Attention {config: &AttentionKernelConfig,context: &MetalContext,) -> bool {- context.supports_mxu- && config.head_dim == 256- && config.is_causal- && config.data_type == DataType::BF16- && !config.has_sinks- && config.sliding_window_size.is_none()+ policy::is_supported(config, context) && config.is_causal}pub fn new(@@ -94,8 +90,9 @@ impl Int8Attention {let gqa_factor = num_q_heads / num_groups;let grouped_rows = gqa_factor * suffix_length;- let query_tiles = grouped_rows.div_ceil(QUERIES_PER_BLOCK);- let split_rows = num_groups * query_tiles * QUERIES_PER_BLOCK;+ let query_rows_per_threadgroup = query_rows_per_threadgroup(head_dim);+ let query_tiles = grouped_rows.div_ceil(query_rows_per_threadgroup);+ let split_rows = num_groups * query_tiles * query_rows_per_threadgroup;let num_splits = int8_splits(split_rows, head_dim, suffix_length, kv_length);let rows = split_rows * num_splits;@@ -160,7 +157,7 @@ impl Int8Attention {gate,params,grouped_rows,- query_tiles * QUERIES_PER_BLOCK,+ query_tiles * query_rows_per_threadgroup,num_groups,num_splits,config.gate_row_stride,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/prepare_int8.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/prepare_int8.metalindex 26724143f..f26252997 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/prepare_int8.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/prepare_int8.metal@@ -29,7 +29,7 @@ METAL_FUNC ElementT kv_quantize_row(const thread float* values, thread int* codetemplate <typename ElementT, typename RopeT, uint HEAD_DIM>VARIANTS(ElementT, bfloat)VARIANTS(RopeT, float)-VARIANTS(HEAD_DIM, 256)+VARIANTS(HEAD_DIM, 128, 256)PUBLIC KERNEL(AttentionPrepareInt8)(const device ElementT* qkvg,device ElementT* queries OPTIONAL(!quantize_queries),diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rsindex 1ca7955e6..a66352667 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs@@ -3,7 +3,7 @@ use uzu_engine_macros::uzu_test;use super::{MaskKind, SplitGeometry, choose_splits, int8_splits, should_encode};fn geometry(head_dim: u32) -> SplitGeometry {- let (num_q_heads, block_rows, num_groups) = if head_dim == 128 {+ let (num_q_heads, query_rows_per_threadgroup, num_groups) = if head_dim == 128 {(32, 64, 8)} else {(24, 32, 4)@@ -12,7 +12,7 @@ fn geometry(head_dim: u32) -> SplitGeometry {head_dim,num_q_heads,num_groups,- block_rows,+ query_rows_per_threadgroup,block_k: 32,}}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rsindex 627a1efdd..b736e5e54 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs@@ -272,8 +272,6 @@ fn attention_gate_matches_standalone_gate_on_metal() {#[uzu_test]fn int8_attention_matches_cpu() {- const DIM: usize = 256;- const KV_HEADS: usize = 4;let (metal, cpu) = (<Metal as Backend>::Context::new().unwrap(), <Cpu as Backend>::Context::new().unwrap());type Prepare = <<Metal as Backend>::Kernels as Kernels>::AttentionPrepareInt8Kernel;let trie: [GpuTrieNode; 3] = bytemuck::cast([0u32, 2, 0, 1, 1, 1, 2, 2, 1]); // (start, end, height) per token@@ -287,28 +285,43 @@ fn int8_attention_matches_cpu() {(24, 3, 16, true, true, NAN),(24, 8, 32, false, false, DENORMAL),(16, 3, 37, true, false, DENORMAL),+ (32, 1, 1024, false, false, NAN),+ (32, 1024, 24, false, false, bf16::MAX.to_bits()),+ (32, 3, 1024, true, false, bf16::INFINITY.to_bits()),] {- let (shape, nodes) = ((DIM, q_heads, KV_HEADS, suffix, prefix, true), use_trie.then_some(trie.as_slice()));- let config = super::default_attention_config::<bf16>(DIM, q_heads, KV_HEADS, true);+ let (head_dim, kv_heads) = if q_heads == 32 {+ (128, 2)+ } else {+ (256, 4)+ };+ let (shape, nodes) = ((head_dim, q_heads, kv_heads, suffix, prefix, true), use_trie.then_some(trie.as_slice()));+ let config = super::default_attention_config::<bf16>(head_dim, q_heads, kv_heads, true);let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionKernel::new(&metal, config).unwrap();if !kernel.supports_int8() {return;}// Fused QKV rows over the data `run_attention` uses; its queries are head-major.- let ((keys, values), stride) = (attention_data(shape), (q_heads + 2 * KV_HEADS) * DIM);+ let ((keys, values), stride) = (attention_data(shape), (q_heads + 2 * kv_heads) * head_dim);let mut rows = vec![bf16::ZERO; (prefix + suffix) * stride];- for (row, (k, v)) in rows.chunks_mut(stride).zip(keys.chunks(KV_HEADS * DIM).zip(values.chunks(KV_HEADS * DIM)))+ for (row, (k, v)) in+ rows.chunks_mut(stride).zip(keys.chunks(kv_heads * head_dim).zip(values.chunks(kv_heads * head_dim))){- let (key_row, value_row) = row[q_heads * DIM..].split_at_mut(KV_HEADS * DIM);+ let (key_row, value_row) = row[q_heads * head_dim..].split_at_mut(kv_heads * head_dim);key_row.copy_from_slice(k);value_row.copy_from_slice(v);}- for (index, query) in fill_attention(q_heads * suffix * DIM, 0.5).chunks(DIM).enumerate() {- rows[(prefix + index % suffix) * stride + index / suffix * DIM..][..DIM].copy_from_slice(query);+ for (index, query) in fill_attention(q_heads * suffix * head_dim, 0.5).chunks(head_dim).enumerate() {+ rows[(prefix + index % suffix) * stride + index / suffix * head_dim..][..head_dim].copy_from_slice(query);}- let mut cache =- KVStorage::<Metal>::create_empty(KVFormat::INT8, KV_HEADS as u32, DIM as u32, 2048, DataType::BF16, &metal)- .unwrap();+ let mut cache = KVStorage::<Metal>::create_empty(+ KVFormat::INT8,+ kv_heads as u32,+ head_dim as u32,+ 2048,+ DataType::BF16,+ &metal,+ )+ .unwrap();cache.prepare(2048).unwrap();let KVStorage::Int8(kv, _) = &mut cache else {unreachable!()@@ -325,20 +338,20 @@ fn int8_attention_matches_cpu() {submit_command_buffer(fill);let mut step = |rows: &[bf16], prefix: usize, trie: Option<&[GpuTrieNode]>, append_kv: bool| {let suffix = rows.len() / stride;- let query_rows = q_heads / KV_HEADS * suffix;+ let query_rows = q_heads / kv_heads * suffix;let prepare = Prepare::new(&metal,DataType::BF16,DataType::F32,- DIM as u32,+ head_dim as u32,false,true,- KV_HEADS as u32,+ kv_heads as u32,append_kv,).unwrap();- let mut query_codes = create_buffer::<Metal, i8>(&metal, KV_HEADS * query_rows * DIM);- let mut query_scales = create_buffer::<Metal, bf16>(&metal, KV_HEADS * query_rows);+ let mut query_codes = create_buffer::<Metal, i8>(&metal, kv_heads * query_rows * head_dim);+ let mut query_scales = create_buffer::<Metal, bf16>(&metal, kv_heads * query_rows);let mut command_buffer = metal.create_command_buffer(None, None).unwrap();let trie = trie.map(|nodes| command_buffer.allocate_constant_from_slice(nodes).unwrap());prepare.encode(@@ -376,7 +389,7 @@ fn int8_attention_matches_cpu() {gate: None::<&<Metal as Backend>::GlobalBuffer>,};let pooled = kernel.encode(input, arguments, &mut command_buffer).unwrap();- let mut output = create_buffer::<Metal, bf16>(&metal, suffix * q_heads * DIM);+ let mut output = create_buffer::<Metal, bf16>(&metal, suffix * q_heads * head_dim);command_buffer.encode_copy(&pooled, &mut output);drop((pooled, trie));submit_command_buffer(command_buffer);@@ -386,7 +399,7 @@ fn int8_attention_matches_cpu() {let mut actual = step(&rows[prefix * stride..], prefix, nodes, true);if reader {// The reader's own K/V rows are junk: appending them would change the output.- rows[prefix * stride..].chunks_mut(stride).for_each(|row| row[q_heads * DIM..].fill(bf16::ONE));+ rows[prefix * stride..].chunks_mut(stride).for_each(|row| row[q_heads * head_dim..].fill(bf16::ONE));actual = step(&rows[prefix * stride..], prefix, nodes, false);}let label = format!("int8 attention q{q_heads} S{suffix} L{prefix} trie={use_trie} reader={reader}");