A high-performance inference engine for AI models
Support Trellis decode for pre M5s (#1071)
CCYeh23ac62825968
diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rsindex 55bca59a2..637e75054 100644--- a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs+++ b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs@@ -38,6 +38,7 @@ pub enum GemmTiling {}pub const MXU_SIMDGROUP_BLOCK_K: u32 = 32;+pub const TRELLIS_DECODE_MAX_BATCH: u32 = 8;impl GemmTiling {pub const fn is_packed_w4(self) -> bool {diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.hindex 16b699a07..59b475209 100644--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h@@ -56,6 +56,8 @@ enum class GemmTiling : uint32_t {static constant constexpr uint32_t MXU_SIMDGROUP_BLOCK_K = 32;+static constant constexpr uint32_t TRELLIS_DECODE_MAX_BATCH = 8;+struct GemmAlignment {uint32_t raw_value;constexpr GemmAlignment() thread : raw_value(0) {}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.hindex 9480e6012..15e4d921c 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h@@ -129,5 +129,15 @@ inline void dequantize(}}+// A byte ored with 0x6400 is the half 1024 plus the byte, so the subtraction is exact.+METAL_FUNC float4 bytes_to_float(const uint word, const float offset) {+ const half2 bias = half2(half(1024.0f + offset));+ const half2 even = as_type<half2>((word & 0x00FF00FFu) | 0x64006400u) - bias;+ const half2 odd = as_type<half2>(((word >> 8) & 0x00FF00FFu) | 0x64006400u) - bias;+ return float4(float(even.x), float(odd.x), float(even.y), float(odd.y));+}++METAL_FUNC float4 int8_to_float(const uint word) { return bytes_to_float(word ^ 0x80808080u, 128.0f); }+} // namespace gemm} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metalindex 6086c2013..2383d9fd6 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal@@ -100,16 +100,14 @@ KERNEL(GemmTrellis)(device bfloat* output,const constant uzu::matmul::GemmParams* params,threadgroup half stage[TRELLIS_STAGE_ELEMENTS],- const constant uint& group_count_x,- const constant uint& group_count_y,- const constant uint& group_count_z,+ const constant uint& partition_count,const GemmAlignment alignment SPECIALIZE,- const uint trellis_vector_width SPECIALIZE,- const uint trellis_transition_bits SPECIALIZE,- const uint trellis_restart_columns SPECIALIZE,- const uint group_x GROUPS(group_count_x),- const uint group_y GROUPS(group_count_y),- const uint group_z GROUPS(group_count_z),+ const uint vector_width SPECIALIZE,+ const uint transition_bits SPECIALIZE,+ const uint restart_columns SPECIALIZE,+ const uint group_x GROUPS(params[0].threadgroups_per_row),+ const uint group_y GROUPS(params[0].threadgroups_per_column),+ const uint group_z GROUPS(partition_count),const uint thread_x THREADS(METAL_SIMD_SIZE),const uint thread_y THREADS(gemm_tiling_simdgroups_per_column(GEMM_TILING)),const uint thread_z THREADS(gemm_tiling_simdgroups_per_row(GEMM_TILING)),@@ -128,11 +126,7 @@ KERNEL(GemmTrellis)(constexpr ushort TILES_M = SIMDGROUP_BLOCK_M / Ops::FRAGMENT_ROWS;constexpr ushort TILES_N = SIMDGROUP_BLOCK_N / Ops::FRAGMENT_COLS;using Products = uzu::matmul::Fragment<int, TILES_M, TILES_N, Ops>;- const trellis::GemmTrellisFormat trellis_format{- trellis_vector_width,- trellis_transition_bits,- trellis_restart_columns- };+ const trellis::GemmTrellisFormat trellis_format{vector_width, transition_bits, restart_columns};const uint simdgroup = thread_context.simdgroup_index;const uint token_base = group_y * BLOCK_M + SIMDGROUP_BLOCK_M * (simdgroup / SIMDGROUPS_PER_COLUMN);diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis_decode.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis_decode.metalnew file mode 100644index 000000000..aa779d936--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis_decode.metal@@ -0,0 +1,105 @@+#include "../../common/dsl.h"+#include "../common/quant_unpack.h"+#include "common/quantized/trellis.h"++using namespace metal;+using namespace uzu::gemm;++#define SIMDGROUPS 8+#define ROWS_PER_SIMDGROUP 4++UZU_CONST uint ROWS_PER_THREADGROUP = ROWS_PER_SIMDGROUP * SIMDGROUPS;+UZU_CONST uint MAX_BATCH = TRELLIS_DECODE_MAX_BATCH;++// Pre-M5 batch 1..8. Lane partials are exact integers in float, so the cross-lane sum is an int simd_sum.+KERNEL(GemmTrellisDecode)(+ const device uint8_t* activations,+ const device float4* column_group_sums,+ const device float* activation_scales,+ const device uint8_t* codes,+ const device float* row_scales,+ const device float* scale_and_offsets,+ device bfloat* output,+ const constant uint& k,+ const constant uint& n,+ const constant uint& output_stride,+ const uint batch SPECIALIZE,+ const uint vector_width SPECIALIZE,+ const uint transition_bits SPECIALIZE,+ const uint restart_columns SPECIALIZE,+ const uint row_block GROUPS(n.div_ceil(ROWS_PER_SIMDGROUP* SIMDGROUPS)),+ const uint lane THREADS(METAL_SIMD_SIZE),+ const uint simdgroup THREADS(SIMDGROUPS)+) {+ const trellis::GemmTrellisFormat format{vector_width, transition_bits, restart_columns};+ const uint row_bytes = trellis::row_bytes(format, k);+ const uint first_row = row_block * ROWS_PER_THREADGROUP + simdgroup * ROWS_PER_SIMDGROUP;+ // Rows past n re-read the last row.+ uint row_offsets[ROWS_PER_SIMDGROUP];+ if (batch <= 2) {+ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < ROWS_PER_SIMDGROUP; ++row) {+ row_offsets[row] = min(first_row + row, n - 1) * row_bytes;+ }+ }+ float sums[MAX_BATCH][ROWS_PER_SIMDGROUP] = {};++ for (uint column = lane * trellis::LANE_COLUMNS; column < k; column += METAL_SIMD_SIZE * trellis::LANE_COLUMNS) {+ uint4 words[ROWS_PER_SIMDGROUP];+ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < ROWS_PER_SIMDGROUP; ++row) {+ words[row] = trellis::decode_biased_weight_quad(+ format,+ codes,+ batch <= 2 ? row_offsets[row] : min(first_row + row, n - 1) * row_bytes,+ column,+ k+ );+ }+ uint4 inputs[MAX_BATCH];+ METAL_PRAGMA_UNROLL+ for (uint token = 0; token < MAX_BATCH; ++token) {+ if (token < batch) {+ inputs[token] = *reinterpret_cast<const device uint4*>(activations + size_t(token) * k + column);+ }+ }+ METAL_PRAGMA_UNROLL+ for (uint quad = 0; quad < 4; ++quad) {+ float4 weights[ROWS_PER_SIMDGROUP];+ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < ROWS_PER_SIMDGROUP; ++row) {+ weights[row] = bytes_to_float(words[row][quad], float(trellis::WEIGHT_BIAS));+ }+ METAL_PRAGMA_UNROLL+ for (uint token = 0; token < MAX_BATCH; ++token) {+ if (token < batch) {+ const float4 input = int8_to_float(inputs[token][quad]);+ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < ROWS_PER_SIMDGROUP; ++row) {+ METAL_PRAGMA_UNROLL+ for (uint lane_column = 0; lane_column < 4; ++lane_column) {+ sums[token][row] = fma(weights[row][lane_column], input[lane_column], sums[token][row]);+ }+ }+ }+ }+ }+ }++ const float4 offsets = float4(scale_and_offsets[1], scale_and_offsets[2], scale_and_offsets[3], scale_and_offsets[4]);+ METAL_PRAGMA_UNROLL+ for (uint token = 0; token < MAX_BATCH; ++token) {+ if (token < batch) {+ const float offsets_dot = dot(column_group_sums[token], offsets);+ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < ROWS_PER_SIMDGROUP; ++row) {+ const float level_dot = float(simd_sum(int(sums[token][row])));+ if (lane == 0 && first_row + row < n) {+ output[token * output_stride + first_row + row] = bfloat(+ (level_dot * scale_and_offsets[0] + offsets_dot) * row_scales[first_row + row] * activation_scales[token]+ );+ }+ }+ }+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rsindex d536dee8f..3b7c6ed60 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs@@ -9,11 +9,11 @@ use crate::backends::{Metal,command_buffer::MetalCommandBufferEncoding,gpu_types::{- gemm::{GemmAlignment, GemmTiling},+ gemm::{GemmAlignment, GemmTiling, TRELLIS_DECODE_MAX_BATCH},matmul::GemmParams,},kernel::{- GemmTrellisMetalKernel, GemmTrellisReduceMetalKernel,+ GemmTrellisDecodeMetalKernel, GemmTrellisMetalKernel, GemmTrellisReduceMetalKernel,matmul::{MatmulMetalError,gemm::{GemmEngine, GemmPlan, selection::TRELLIS_K_STEP as K_STEP},@@ -22,9 +22,16 @@ use crate::backends::{},};+const LANE_COLUMNS: u32 = 16;+const MAX_CENTERED_WEIGHT: u32 = 57;+const MAX_ACTIVATION: u32 = 128;+const DECODE_MAX_EXACT_K: u32 =+ ((1 << 24) - 1) / (LANE_COLUMNS * MAX_CENTERED_WEIGHT * MAX_ACTIVATION) * 32 * LANE_COLUMNS;+#[derive(Default)]pub(super) struct TrellisGemm {kernels: HashMap<(GemmTiling, TrellisFormat, GemmAlignment), GemmTrellisMetalKernel>,+ decode_kernels: HashMap<(u32, TrellisFormat), GemmTrellisDecodeMetalKernel>,reduce_kernel: Option<GemmTrellisReduceMetalKernel>,}@@ -74,6 +81,36 @@ impl TrellisGemm {return Err(unsupported());}+ if plan.engine == GemmEngine::Simdgroup+ && (1..=TRELLIS_DECODE_MAX_BATCH).contains(&m)+ && k <= DECODE_MAX_EXACT_K+ {+ let kernel = match self.decode_kernels.entry((m, format)) {+ Entry::Occupied(entry) => entry.into_mut(),+ Entry::Vacant(entry) => entry.insert(GemmTrellisDecodeMetalKernel::new(+ command_buffer.context(),+ m,+ format.vector_width,+ format.transition_bits,+ format.restart_columns.unwrap_or(0),+ )?),+ };+ kernel.encode(+ activations,+ column_group_sums,+ activation_scales,+ codes,+ row_scales,+ codebook,+ arguments.output.values,+ k,+ n,+ output_stride,+ command_buffer,+ );+ return Ok(());+ }+let params = GemmParams {M: m,N: n,@@ -111,8 +148,6 @@ impl TrellisGemm {codebook,$destination,std::slice::from_ref(¶ms),- params.threadgroups_per_row,- params.threadgroups_per_column,split_k,command_buffer,)diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rsindex d0926bbeb..a47713008 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs@@ -243,6 +243,10 @@ fn trellis_projection_matches_cpu_reference(#[case] format: TrellisFormat) {for dimensions in CASES.into_iter().filter(|&(_, _, k)| whole_blocks(k)) {run_projection(&context, format, dimensions, None);}+ // Simdgroup plans with m up to 8 run the decode kernel. K fits every restart size.+ for m in 1..=8 {+ run_projection(&context, format, (m, 37, 4160), Some((GemmTiling::Tile16x32x32_Simdgroups2x1, 1)));+ }// The forced plans' K are whole 64-column blocks.if whole_blocks(64) {if context.supports_mxu {@@ -252,7 +256,7 @@ fn trellis_projection_matches_cpu_reference(#[case] format: TrellisFormat) {run_projection(&context, format, (17, 80, 192), Some((GemmTiling::Tile16x32x256_Simdgroups1x1, 1)));run_projection(&context, format, (7, 80, 192), Some((GemmTiling::Tile16x32x256_Simdgroups1x1, 1)));}- run_projection(&context, format, (7, 68, 6144), Some((GemmTiling::Tile16x32x32_Simdgroups2x1, 4)));+ run_projection(&context, format, (9, 68, 6144), Some((GemmTiling::Tile16x32x32_Simdgroups2x1, 4)));run_projection(&context, format, (33, 68, 6144), Some((GemmTiling::Tile32x32x32_Simdgroups2x2, 4)));run_projection(&context, format, (65, 68, 6144), Some((GemmTiling::Tile64x64x32_Simdgroups2x2, 4)));}