A high-performance inference engine for AI models
2bit gemv mxutiled (#1089)
CCYeh3909d5b52124
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.hindex a33e91ead..56ace9003 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h@@ -1,5 +1,6 @@#pragma once+#include "../../gemm/common/mxu_tiled_layout.h"#include "arguments.h"#include "quant_chunk.h"#include "quant_metadata.h"@@ -39,18 +40,25 @@ private:static_assert(GROUP_SIZE % Tile::GROUP_LANES == 0, "group lanes must divide the quantization group");static_assert(VALUES_PER_LANE % CHUNK_VALUES == 0, "group lane slices must contain complete chunks");- static_assert(SLICE_VALUES <= MAX_SLICE_VALUES, "QMV slice exceeds its staged-value limit");static_assert(VALUES_PER_LANE % SLICE_VALUES == 0, "a lane loads complete group slices");uint4 weights[Tile::ROWS_PER_LANE][SLICE_WORDS];+ using Layout = MxuTiledLayout<BITS>;+ using Ops = typename Layout::Ops;+static METAL_FUNC uint mxu_tiled_offset(uint row, uint row_stride, uint byte) {- constexpr uint CHUNK_BYTES = BITS * 4;- const uint y = row % 8;- const uint x = byte % CHUNK_BYTES / BITS;+ constexpr uint CHUNK_BYTES = Ops::FRAGMENT_COLS * Layout::COL_FRAGMENTS;+ constexpr uint COLUMN_GROUP_BYTES = Ops::THREAD_ELEMENT_COLS * Layout::COL_FRAGMENTS;+ const uint y = row % Ops::THREAD_ELEMENT_ROW_STRIDE;+ const uint x = byte % CHUNK_BYTES / COLUMN_GROUP_BYTES;+ // Inverse of Ops::get_position.const uint lane = (x & 1) | ((y & 1) << 1) | ((y >> 1 & 1) << 2) | ((x >> 1) << 3) | ((y >> 2) << 4);- return row / 32 * 32 * row_stride + (byte / CHUNK_BYTES * 32 + lane) * CHUNK_BYTES + row % 32 / 8 * BITS +- byte % BITS;+ return row / Layout::BLOCK_ROWS * Layout::BLOCK_ROWS * row_stride ++ (byte / CHUNK_BYTES * METAL_SIMD_SIZE + lane) * Layout::LANE_BYTES ++ row % Layout::BLOCK_ROWS / Ops::THREAD_ELEMENT_ROW_STRIDE * (BITS == 2 ? 1 : Layout::COL_FRAGMENTS) *+ QuantChunk<BITS>::WORD_BYTES ++ byte % COLUMN_GROUP_BYTES;}METAL_FUNC void load_mxu_tiled_weights(@@ -91,6 +99,25 @@ private:}}+ METAL_FUNC void load_mxu_tiled_w2_weights(+ const device uint8_t* weights_base,+ const thread uint (&weight_row_indices)[Tile::ROWS_PER_LANE],+ uint row_stride,+ uint offset_bytes+ ) thread {+ static_assert(SLICE_BYTES == Ops::THREAD_ELEMENT_COLS * Layout::COL_FRAGMENTS, "2-bit slice is one column group");+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {+ constexpr uint R = decltype(output_index)::value;+ const device uint8_t* source = weights_base + mxu_tiled_offset(weight_row_indices[R], row_stride, offset_bytes);+ weights[R][0] = uint4(+ *reinterpret_cast<const device uint*>(source),+ *reinterpret_cast<const device uint*>(source + Ops::THREAD_ELEMENT_ROWS * QuantChunk<BITS>::WORD_BYTES),+ 0u,+ 0u+ );+ });+ }+public:static METAL_FUNC uint row_stride(const thread GemvParams& params) {return params.in_vec_size * BITS / QuantChunk<BITS>::BITS_PER_BYTE;@@ -106,11 +133,13 @@ public:) thread {const uint k = position.group * GROUP_SIZE + group_offset + position.slice * SLICE_VALUES;const uint offset_bytes = k * BITS / QuantChunk<BITS>::BITS_PER_BYTE;- if constexpr (BITS == 4 || BITS == 8) {- if (params.gemm_b_layout == GemmBLayout::MxuTiled) {+ if (params.gemm_b_layout == GemmBLayout::MxuTiled) {+ if constexpr (BITS == 2) {+ load_mxu_tiled_w2_weights(weights_base, weight_row_indices, row_stride, offset_bytes);+ } else {load_mxu_tiled_weights(weights_base, weight_row_indices, row_stride, offset_bytes, params.gathered);- return;}+ return;}Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {constexpr uint R = decltype(output_index)::value;@@ -158,7 +187,17 @@ public:position.group * GROUP_SIZE + group_offset + position.slice * SLICE_VALUES + chunk * CHUNK_VALUES;const uint input_row = tile.input_row + source_row;float input_values[CHUNK_VALUES];- if (BITS == 8 && params.gemm_b_layout == GemmBLayout::MxuTiled) {+ if (BITS == 2 && params.gemm_b_layout == GemmBLayout::MxuTiled) {+ const device AT* input =+ ops.a + input_row * params.in_vec_size + position.group * GROUP_SIZE ++ chunk / 2 * (CHUNK_VALUES / 2) * Ops::FRAGMENT_COLS ++ (group_offset + position.slice * SLICE_VALUES) / SLICE_VALUES * Ops::THREAD_ELEMENT_COLS + chunk % 2 * 2;+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < CHUNK_VALUES / 2; i++) {+ input_values[i] = float(input[Ops::FRAGMENT_COLS * i]);+ input_values[CHUNK_VALUES / 2 + i] = float(input[Ops::FRAGMENT_COLS * i + 1]);+ }+ } else if (BITS == 8 && params.gemm_b_layout == GemmBLayout::MxuTiled) {const device AT* input = ops.a + input_row * params.in_vec_size + k - k % 32 / 2;METAL_PRAGMA_UNROLLfor (uint i = 0; i < CHUNK_VALUES / 2; i++) {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rsindex ef08d5026..985d5fe46 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -74,9 +74,6 @@ impl GemvSpecialization {} else if is_quant {policy::quantized_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n, bf16_io)} else {- if shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH {- return None;- }let input_aligned = shape.k.is_multiple_of(FP_K_BLOCK);Some(policy::fp_tile(shape.m, shape.n, shape.k, input_aligned))};@@ -104,7 +101,7 @@ impl GemvSpecialization {return None;}let bits = shape.b_bits.unwrap_or(0);- if bits == 2 && gemm_b_layout == GemmBLayout::MxuTiled {+ if bits == 2 && gemm_b_layout == GemmBLayout::MxuTiled && !shape.gathered {return None;}if !is_quant && (shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH) {diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rsindex 9a9d38948..be35ca155 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs@@ -699,27 +699,34 @@ fn gemv_reads_mxu_tiled_layout_bf16(#[case::w4_zp_gs32(4u32, 32u32, ScaleZeroPoint)]#[case::w8_sym_gs64(8u32, 64u32, QuantizationMethod::ScaleSymmetric)]#[case::w2_bias_gs128(2u32, 128u32, QuantizationMethod::ScaleBias)]-fn gathered_gemv_reads_mxu_tiled_layout_bf16(+fn gathered_gemv_reads_weight_layout_bf16(#[case] bits: u32,#[case] group_size: u32,#[case] method: QuantizationMethod,+ #[values(GemmBLayout::RowMajor, GemmBLayout::MxuTiled)] gemm_b_layout: GemmBLayout,+ #[values(1, 3, 8)] m: u32,+ #[values(256, 4096)] k: u32,) {let context = block_on(MetalContext::new()).expect("Metal context");- if !context.supports_mxu {+ if gemm_b_layout == GemmBLayout::MxuTiled && !context.supports_mxu {return;}- let (k, vocab, gathered_rows) = (256u32, 256u32, 10u32);- let input = QuantInput::<bf16>::new(1, k, vocab, group_size, bits, method, 0).with_group_output();+ let (weight_rows, gathered_rows) = (256u32, 10u32);+ let input = QuantInput::<bf16>::new(m, k, weight_rows, group_size, bits, method, 0).with_group_output();let input = match bits {2 => input.with_one_group_per_row(),8 => input.with_signed_weight_codes(),_ => input,};- let ids: Vec<u32> = (0..gathered_rows).map(|row| (row * 37 + 11) % vocab).collect();- let dense = run_quant_cpu::<bf16>(&input);- let expected: Vec<bf16> = ids.iter().map(|&id| dense[id as usize]).collect();+ let gather_indices: Vec<u32> = (0..m * gathered_rows).map(|i| (i * 37 + 11) % weight_rows).collect();+ let dense_output = run_quant_cpu::<bf16>(&input);+ let expected: Vec<bf16> = gather_indices+ .chunks_exact(gathered_rows as usize)+ .zip(dense_output.chunks_exact(weight_rows as usize))+ .flat_map(|(indices, output_row)| indices.iter().map(move |&index| output_row[index as usize]))+ .collect();- let ids_buffer = crate::tests::helpers::create_buffer_with_data::<Metal, u32>(&context, &ids);+ let gather_indices_buffer = crate::tests::helpers::create_buffer_with_data::<Metal, u32>(&context, &gather_indices);let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(&context,@@ -728,19 +735,19 @@ fn gathered_gemv_reads_mxu_tiled_layout_bf16(bf16::data_type(),).expect("Metal matmul kernel");- if bits != 2 {+ if gemm_b_layout == GemmBLayout::MxuTiled {tile_weights(&matmul, &mut buffers, &input, &context);}let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");let mut arguments = quant_arguments(&mut buffers, &input);- arguments.gather_indices = Some(&ids_buffer);+ arguments.gather_indices = Some(&gather_indices_buffer);arguments.n = gathered_rows;matmul.encode(arguments, &mut command_buffer).expect("gathered GEMV");block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();- let actual = buffer_to_vec::<Metal, bf16>(&context, &buffers.y)[..gathered_rows as usize].to_vec();+ let actual = buffer_to_vec::<Metal, bf16>(&context, &buffers.y)[..(m * gathered_rows) as usize].to_vec();assert_parity::<bf16>(- &format!("gathered MxuTiled GEMV bits={bits} gs={group_size} method={method:?}"),+ &format!("gathered {gemm_b_layout:?} GEMV m={m} k={k} bits={bits} gs={group_size} method={method:?}"),&expected,&actual,0.05,