Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Decode Trellis weights without tail padding (#1076)

CCYeh1866d189f91a
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
index adf77560c..d214d16d6 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
@@ -14,10 +14,6 @@ pub struct TrellisFormat {
pub restart_columns: Option<u32>,
}
-/// Readable bytes past the last code row (the GEMM loads whole words).
-#[cfg(backend = "metal")]
-pub const TRELLIS_CODES_TAIL_BYTES: usize = 16;
-
impl TrellisFormat {
/// Bytes per code row: restart rows pad to 16 bytes, connected rows to a byte.
pub fn code_row_bytes(
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
index b611a716d..6bf75391f 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
@@ -10,9 +10,9 @@ pub use arguments::MatmulArguments;
pub use d_ops::MatmulDOps;
pub use kernel::MatmulKernel;
pub use matmul_a::{Int8CodeLayout, MatmulA, packed_half_bytes};
-pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection};
#[cfg(backend = "metal")]
-pub use matmul_b::{TRELLIS_CODES_TAIL_BYTES, TrellisFormat};
+pub use matmul_b::TrellisFormat;
+pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection};
pub use output::MatmulOutput;
#[cfg(any(backend = "cpu", backend = "metal"))] // TODO: webgpu
pub use output::MatmulOutputError;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/trellis.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/trellis.h
index f027c62a2..ab1da1f13 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/trellis.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/trellis.h
@@ -126,13 +126,48 @@ static METAL_FUNC uint4 decode_biased_weight_quad(
const uint first_bit =
block * block_size + block_size - STATE_BITS - (state_in_block + window_states - 1) * transition_bits;
const uint byte = row_start_byte + first_bit / BYTE_BITS;
- const uint shift = byte % WORD_BYTES * BYTE_BITS + first_bit % BYTE_BITS;
- const device uint* words = reinterpret_cast<const device uint*>(codes) + byte / WORD_BYTES;
- const uint first = words[0], second = words[1];
- const uint third = third_word_possible ? words[1 + uint(shift + bits_in_window > 2 * WORD_BITS)] : 0u;
- const uint word0 = (first >> shift) | ((second << 1) << (WORD_BITS - 1 - shift));
- const uint word1 = (second >> shift) | ((third << 1) << (WORD_BITS - 1 - shift));
- const uint word2 = third >> shift;
+ uint3 words(0u);
+ uint bit_offset;
+ if (format.restart_columns != 0) {
+ bit_offset = byte % WORD_BYTES * BYTE_BITS + first_bit % BYTE_BITS;
+ const device uint* aligned_words = reinterpret_cast<const device uint*>(codes) + byte / WORD_BYTES;
+ const bool word_pair_fits = format.vector_width == 4 && transition_bits == 8 && format.restart_columns == 64;
+ words.x = aligned_words[0];
+ // Reuse an existing word to avoid conditional loads.
+ const uint second_index = word_pair_fits || bit_offset + bits_in_window > WORD_BITS ? 1u : 0u;
+ words.y = aligned_words[second_index];
+ const uint third_index = bit_offset + bits_in_window > 2 * WORD_BITS ? 2u : (word_pair_fits ? 1u : 0u);
+ words.z = third_word_possible ? aligned_words[third_index] : 0u;
+ } else {
+ const uint max_bit_offset = format.vector_width == 2 || transition_bits % 2 == 0 ? 0u : 4u;
+ bit_offset = max_bit_offset == 0 ? 0u : first_bit % BYTE_BITS;
+ const uint min_byte_count = div_ceil(bits_in_window, BYTE_BITS);
+ const uint full_word_bytes = min_byte_count / WORD_BYTES * WORD_BYTES;
+ const uint max_byte_count = div_ceil(bits_in_window + max_bit_offset, BYTE_BITS);
+ const device packed_uchar4* packed_words = reinterpret_cast<const device packed_uchar4*>(codes + byte);
+ if (full_word_bytes >= WORD_BYTES) {
+ words.x = as_type<uint>(packed_words[0]);
+ }
+ if (full_word_bytes >= 2 * WORD_BYTES) {
+ words.y = as_type<uint>(packed_words[1]);
+ }
+ METAL_PRAGMA_UNROLL
+ for (uint byte_index = full_word_bytes; byte_index < max_byte_count; ++byte_index) {
+ if (byte_index * BYTE_BITS < bit_offset + bits_in_window) {
+ const uint value = uint(codes[byte + byte_index]) << (byte_index % WORD_BYTES * BYTE_BITS);
+ if (byte_index < WORD_BYTES) {
+ words.x |= value;
+ } else if (byte_index < 2 * WORD_BYTES) {
+ words.y |= value;
+ } else {
+ words.z |= value;
+ }
+ }
+ }
+ }
+ const uint word0 = (words.x >> bit_offset) | ((words.y << 1) << (WORD_BITS - 1 - bit_offset));
+ const uint word1 = (words.y >> bit_offset) | ((words.z << 1) << (WORD_BITS - 1 - bit_offset));
+ const uint word2 = words.z >> bit_offset;
METAL_PRAGMA_UNROLL
for (uint output = 0; output < window_states / states_per_weight_word; ++output) {
const uint first_state_bit = (window_states - 1 - output * states_per_weight_word) * transition_bits;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs
index 3b7c6ed60..f75074806 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs
@@ -3,7 +3,7 @@ use std::collections::{HashMap, hash_map::Entry};
use crate::backends::{
common::{
BufferMut, BufferRef, CommandBufferEncoding,
- kernel::matmul::{MatmulA, MatmulArguments, MatmulB, TRELLIS_CODES_TAIL_BYTES, TrellisFormat},
+ kernel::matmul::{MatmulA, MatmulArguments, MatmulB, TrellisFormat},
},
metal::{
Metal,
@@ -76,7 +76,7 @@ impl TrellisGemm {
|| arguments.b_leading_dimension.is_some()
|| output_stride < n
|| (split_k > 1 && !n.is_multiple_of(4))
- || codes.size() < (n * format.code_row_bytes(k)) as usize + TRELLIS_CODES_TAIL_BYTES
+ || codes.size() < n as usize * format.code_row_bytes(k) as usize
{
return Err(unsupported());
}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs
index a47713008..a5f2a78d4 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs
@@ -8,8 +8,7 @@ use crate::{
common::{
Backend, Context, Kernels,
kernel::matmul::{
- MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput, MatmulShape,
- TRELLIS_CODES_TAIL_BYTES, TrellisFormat,
+ MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput, MatmulShape, TrellisFormat,
},
},
metal::{
@@ -99,7 +98,14 @@ const PADDING: usize = 24;
const SENTINEL: u16 = 0x7F7F;
const RELATIVE_TOLERANCE: f32 = 1e-2;
// Cases are ordered as (m, n, k).
-const CASES: [(usize, usize, usize); 9] = [
+const CASES: [(usize, usize, usize); 16] = [
+ (1, 1, 64),
+ (1, 1, 512),
+ (1, 1, 2048),
+ (1, 1, 320),
+ (1, 85, 512),
+ (1, 33, 2048),
+ (1, 17, 5120),
(1, 80, 5120),
(7, 80, 5120),
(17, 80, 5120),
@@ -146,10 +152,7 @@ fn run_projection(
let activations = create_buffer_with_data::<Metal, i8>(context, &values);
let group_sums = create_buffer_with_data::<Metal, f32>(context, &group_sums_host);
let activation_scales = create_buffer_with_data::<Metal, f32>(context, &activation_scales_host);
- // The GEMM may read a word past the last row's codes; the tail must not change the result.
- let mut codes_with_tail = stored.clone();
- codes_with_tail.resize(stored.len() + TRELLIS_CODES_TAIL_BYTES, 0xA5);
- let codes = create_buffer_with_data::<Metal, u8>(context, &codes_with_tail);
+ let codes = create_buffer_with_data::<Metal, u8>(context, &stored);
let row_scales_buffer = create_buffer_with_data::<Metal, f32>(context, &row_scales);
let codebook = create_buffer_with_data::<Metal, f32>(context, &CODEBOOK);
let mut output = create_buffer_with_data::<Metal, u16>(context, &vec![SENTINEL; m * (n + PADDING)]);
@@ -236,6 +239,9 @@ fn run_projection(
#[case::v4_t7(V4_T7)]
#[case::v4_t6_r40(V4_T6_R40)]
#[case::v4_t8_no_restart(V4_T8_NO_RESTART)]
+#[case::v2_t1(TrellisFormat { transition_bits: 1, ..V2_T4 })]
+#[case::v4_t1(TrellisFormat { transition_bits: 1, ..V4_T8_NO_RESTART })]
+#[case::v4_t2(TrellisFormat { transition_bits: 2, ..V4_T8_NO_RESTART })]
fn trellis_projection_matches_cpu_reference(#[case] format: TrellisFormat) {
let context = shared_metal_context();
// A projection's K is a whole number of restart blocks.
@@ -249,6 +255,12 @@ fn trellis_projection_matches_cpu_reference(#[case] format: TrellisFormat) {
}
// The forced plans' K are whole 64-column blocks.
if whole_blocks(64) {
+ for k in [512, 2048] {
+ run_projection(&context, format, (1, 1, k), Some((GemmTiling::Tile16x32x32_Simdgroups2x1, 1)));
+ if context.supports_mxu {
+ run_projection(&context, format, (1, 1, k), Some((GemmTiling::Tile16x32x256_Simdgroups1x1, 1)));
+ }
+ }
if context.supports_mxu {
run_projection(&context, format, (129, 132, 1024), Some((GemmTiling::Tile128x128x256_Simdgroups4x4, 1)));
run_projection(&context, format, (129, 132, 64), Some((GemmTiling::Tile128x128x256_Simdgroups4x4, 1)));