A high-performance inference engine for AI models
Decode Trellis weights without tail padding (#1076)
CCYeh1866d189f91a
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rsindex adf77560c..d214d16d6 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs@@ -14,10 +14,6 @@ pub struct TrellisFormat {pub restart_columns: Option<u32>,}-/// Readable bytes past the last code row (the GEMM loads whole words).-#[cfg(backend = "metal")]-pub const TRELLIS_CODES_TAIL_BYTES: usize = 16;-impl TrellisFormat {/// Bytes per code row: restart rows pad to 16 bytes, connected rows to a byte.pub fn code_row_bytes(diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rsindex b611a716d..6bf75391f 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs@@ -10,9 +10,9 @@ pub use arguments::MatmulArguments;pub use d_ops::MatmulDOps;pub use kernel::MatmulKernel;pub use matmul_a::{Int8CodeLayout, MatmulA, packed_half_bytes};-pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection};#[cfg(backend = "metal")]-pub use matmul_b::{TRELLIS_CODES_TAIL_BYTES, TrellisFormat};+pub use matmul_b::TrellisFormat;+pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection};pub use output::MatmulOutput;#[cfg(any(backend = "cpu", backend = "metal"))] // TODO: webgpupub use output::MatmulOutputError;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/trellis.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/trellis.hindex f027c62a2..ab1da1f13 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/trellis.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/trellis.h@@ -126,13 +126,48 @@ static METAL_FUNC uint4 decode_biased_weight_quad(const uint first_bit =block * block_size + block_size - STATE_BITS - (state_in_block + window_states - 1) * transition_bits;const uint byte = row_start_byte + first_bit / BYTE_BITS;- const uint shift = byte % WORD_BYTES * BYTE_BITS + first_bit % BYTE_BITS;- const device uint* words = reinterpret_cast<const device uint*>(codes) + byte / WORD_BYTES;- const uint first = words[0], second = words[1];- const uint third = third_word_possible ? words[1 + uint(shift + bits_in_window > 2 * WORD_BITS)] : 0u;- const uint word0 = (first >> shift) | ((second << 1) << (WORD_BITS - 1 - shift));- const uint word1 = (second >> shift) | ((third << 1) << (WORD_BITS - 1 - shift));- const uint word2 = third >> shift;+ uint3 words(0u);+ uint bit_offset;+ if (format.restart_columns != 0) {+ bit_offset = byte % WORD_BYTES * BYTE_BITS + first_bit % BYTE_BITS;+ const device uint* aligned_words = reinterpret_cast<const device uint*>(codes) + byte / WORD_BYTES;+ const bool word_pair_fits = format.vector_width == 4 && transition_bits == 8 && format.restart_columns == 64;+ words.x = aligned_words[0];+ // Reuse an existing word to avoid conditional loads.+ const uint second_index = word_pair_fits || bit_offset + bits_in_window > WORD_BITS ? 1u : 0u;+ words.y = aligned_words[second_index];+ const uint third_index = bit_offset + bits_in_window > 2 * WORD_BITS ? 2u : (word_pair_fits ? 1u : 0u);+ words.z = third_word_possible ? aligned_words[third_index] : 0u;+ } else {+ const uint max_bit_offset = format.vector_width == 2 || transition_bits % 2 == 0 ? 0u : 4u;+ bit_offset = max_bit_offset == 0 ? 0u : first_bit % BYTE_BITS;+ const uint min_byte_count = div_ceil(bits_in_window, BYTE_BITS);+ const uint full_word_bytes = min_byte_count / WORD_BYTES * WORD_BYTES;+ const uint max_byte_count = div_ceil(bits_in_window + max_bit_offset, BYTE_BITS);+ const device packed_uchar4* packed_words = reinterpret_cast<const device packed_uchar4*>(codes + byte);+ if (full_word_bytes >= WORD_BYTES) {+ words.x = as_type<uint>(packed_words[0]);+ }+ if (full_word_bytes >= 2 * WORD_BYTES) {+ words.y = as_type<uint>(packed_words[1]);+ }+ METAL_PRAGMA_UNROLL+ for (uint byte_index = full_word_bytes; byte_index < max_byte_count; ++byte_index) {+ if (byte_index * BYTE_BITS < bit_offset + bits_in_window) {+ const uint value = uint(codes[byte + byte_index]) << (byte_index % WORD_BYTES * BYTE_BITS);+ if (byte_index < WORD_BYTES) {+ words.x |= value;+ } else if (byte_index < 2 * WORD_BYTES) {+ words.y |= value;+ } else {+ words.z |= value;+ }+ }+ }+ }+ const uint word0 = (words.x >> bit_offset) | ((words.y << 1) << (WORD_BITS - 1 - bit_offset));+ const uint word1 = (words.y >> bit_offset) | ((words.z << 1) << (WORD_BITS - 1 - bit_offset));+ const uint word2 = words.z >> bit_offset;METAL_PRAGMA_UNROLLfor (uint output = 0; output < window_states / states_per_weight_word; ++output) {const uint first_state_bit = (window_states - 1 - output * states_per_weight_word) * transition_bits;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rsindex 3b7c6ed60..f75074806 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs@@ -3,7 +3,7 @@ use std::collections::{HashMap, hash_map::Entry};use crate::backends::{common::{BufferMut, BufferRef, CommandBufferEncoding,- kernel::matmul::{MatmulA, MatmulArguments, MatmulB, TRELLIS_CODES_TAIL_BYTES, TrellisFormat},+ kernel::matmul::{MatmulA, MatmulArguments, MatmulB, TrellisFormat},},metal::{Metal,@@ -76,7 +76,7 @@ impl TrellisGemm {|| arguments.b_leading_dimension.is_some()|| output_stride < n|| (split_k > 1 && !n.is_multiple_of(4))- || codes.size() < (n * format.code_row_bytes(k)) as usize + TRELLIS_CODES_TAIL_BYTES+ || codes.size() < n as usize * format.code_row_bytes(k) as usize{return Err(unsupported());}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rsindex a47713008..a5f2a78d4 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs@@ -8,8 +8,7 @@ use crate::{common::{Backend, Context, Kernels,kernel::matmul::{- MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput, MatmulShape,- TRELLIS_CODES_TAIL_BYTES, TrellisFormat,+ MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput, MatmulShape, TrellisFormat,},},metal::{@@ -99,7 +98,14 @@ const PADDING: usize = 24;const SENTINEL: u16 = 0x7F7F;const RELATIVE_TOLERANCE: f32 = 1e-2;// Cases are ordered as (m, n, k).-const CASES: [(usize, usize, usize); 9] = [+const CASES: [(usize, usize, usize); 16] = [+ (1, 1, 64),+ (1, 1, 512),+ (1, 1, 2048),+ (1, 1, 320),+ (1, 85, 512),+ (1, 33, 2048),+ (1, 17, 5120),(1, 80, 5120),(7, 80, 5120),(17, 80, 5120),@@ -146,10 +152,7 @@ fn run_projection(let activations = create_buffer_with_data::<Metal, i8>(context, &values);let group_sums = create_buffer_with_data::<Metal, f32>(context, &group_sums_host);let activation_scales = create_buffer_with_data::<Metal, f32>(context, &activation_scales_host);- // The GEMM may read a word past the last row's codes; the tail must not change the result.- let mut codes_with_tail = stored.clone();- codes_with_tail.resize(stored.len() + TRELLIS_CODES_TAIL_BYTES, 0xA5);- let codes = create_buffer_with_data::<Metal, u8>(context, &codes_with_tail);+ let codes = create_buffer_with_data::<Metal, u8>(context, &stored);let row_scales_buffer = create_buffer_with_data::<Metal, f32>(context, &row_scales);let codebook = create_buffer_with_data::<Metal, f32>(context, &CODEBOOK);let mut output = create_buffer_with_data::<Metal, u16>(context, &vec![SENTINEL; m * (n + PADDING)]);@@ -236,6 +239,9 @@ fn run_projection(#[case::v4_t7(V4_T7)]#[case::v4_t6_r40(V4_T6_R40)]#[case::v4_t8_no_restart(V4_T8_NO_RESTART)]+#[case::v2_t1(TrellisFormat { transition_bits: 1, ..V2_T4 })]+#[case::v4_t1(TrellisFormat { transition_bits: 1, ..V4_T8_NO_RESTART })]+#[case::v4_t2(TrellisFormat { transition_bits: 2, ..V4_T8_NO_RESTART })]fn trellis_projection_matches_cpu_reference(#[case] format: TrellisFormat) {let context = shared_metal_context();// A projection's K is a whole number of restart blocks.@@ -249,6 +255,12 @@ fn trellis_projection_matches_cpu_reference(#[case] format: TrellisFormat) {}// The forced plans' K are whole 64-column blocks.if whole_blocks(64) {+ for k in [512, 2048] {+ run_projection(&context, format, (1, 1, k), Some((GemmTiling::Tile16x32x32_Simdgroups2x1, 1)));+ if context.supports_mxu {+ run_projection(&context, format, (1, 1, k), Some((GemmTiling::Tile16x32x256_Simdgroups1x1, 1)));+ }+ }if context.supports_mxu {run_projection(&context, format, (129, 132, 1024), Some((GemmTiling::Tile128x128x256_Simdgroups4x4, 1)));run_projection(&context, format, (129, 132, 64), Some((GemmTiling::Tile128x128x256_Simdgroups4x4, 1)));