Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

2bit gemv mxutiled (#1089)

CCYeh3909d5b52124
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h
index a33e91ead..56ace9003 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h
@@ -1,5 +1,6 @@
#pragma once
+#include "../../gemm/common/mxu_tiled_layout.h"
#include "arguments.h"
#include "quant_chunk.h"
#include "quant_metadata.h"
@@ -39,18 +40,25 @@ private:
static_assert(GROUP_SIZE % Tile::GROUP_LANES == 0, "group lanes must divide the quantization group");
static_assert(VALUES_PER_LANE % CHUNK_VALUES == 0, "group lane slices must contain complete chunks");
- static_assert(SLICE_VALUES <= MAX_SLICE_VALUES, "QMV slice exceeds its staged-value limit");
static_assert(VALUES_PER_LANE % SLICE_VALUES == 0, "a lane loads complete group slices");
uint4 weights[Tile::ROWS_PER_LANE][SLICE_WORDS];
+ using Layout = MxuTiledLayout<BITS>;
+ using Ops = typename Layout::Ops;
+
static METAL_FUNC uint mxu_tiled_offset(uint row, uint row_stride, uint byte) {
- constexpr uint CHUNK_BYTES = BITS * 4;
- const uint y = row % 8;
- const uint x = byte % CHUNK_BYTES / BITS;
+ constexpr uint CHUNK_BYTES = Ops::FRAGMENT_COLS * Layout::COL_FRAGMENTS;
+ constexpr uint COLUMN_GROUP_BYTES = Ops::THREAD_ELEMENT_COLS * Layout::COL_FRAGMENTS;
+ const uint y = row % Ops::THREAD_ELEMENT_ROW_STRIDE;
+ const uint x = byte % CHUNK_BYTES / COLUMN_GROUP_BYTES;
+ // Inverse of Ops::get_position.
const uint lane = (x & 1) | ((y & 1) << 1) | ((y >> 1 & 1) << 2) | ((x >> 1) << 3) | ((y >> 2) << 4);
- return row / 32 * 32 * row_stride + (byte / CHUNK_BYTES * 32 + lane) * CHUNK_BYTES + row % 32 / 8 * BITS +
- byte % BITS;
+ return row / Layout::BLOCK_ROWS * Layout::BLOCK_ROWS * row_stride +
+ (byte / CHUNK_BYTES * METAL_SIMD_SIZE + lane) * Layout::LANE_BYTES +
+ row % Layout::BLOCK_ROWS / Ops::THREAD_ELEMENT_ROW_STRIDE * (BITS == 2 ? 1 : Layout::COL_FRAGMENTS) *
+ QuantChunk<BITS>::WORD_BYTES +
+ byte % COLUMN_GROUP_BYTES;
}
METAL_FUNC void load_mxu_tiled_weights(
@@ -91,6 +99,25 @@ private:
}
}
+ METAL_FUNC void load_mxu_tiled_w2_weights(
+ const device uint8_t* weights_base,
+ const thread uint (&weight_row_indices)[Tile::ROWS_PER_LANE],
+ uint row_stride,
+ uint offset_bytes
+ ) thread {
+ static_assert(SLICE_BYTES == Ops::THREAD_ELEMENT_COLS * Layout::COL_FRAGMENTS, "2-bit slice is one column group");
+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
+ constexpr uint R = decltype(output_index)::value;
+ const device uint8_t* source = weights_base + mxu_tiled_offset(weight_row_indices[R], row_stride, offset_bytes);
+ weights[R][0] = uint4(
+ *reinterpret_cast<const device uint*>(source),
+ *reinterpret_cast<const device uint*>(source + Ops::THREAD_ELEMENT_ROWS * QuantChunk<BITS>::WORD_BYTES),
+ 0u,
+ 0u
+ );
+ });
+ }
+
public:
static METAL_FUNC uint row_stride(const thread GemvParams& params) {
return params.in_vec_size * BITS / QuantChunk<BITS>::BITS_PER_BYTE;
@@ -106,11 +133,13 @@ public:
) thread {
const uint k = position.group * GROUP_SIZE + group_offset + position.slice * SLICE_VALUES;
const uint offset_bytes = k * BITS / QuantChunk<BITS>::BITS_PER_BYTE;
- if constexpr (BITS == 4 || BITS == 8) {
- if (params.gemm_b_layout == GemmBLayout::MxuTiled) {
+ if (params.gemm_b_layout == GemmBLayout::MxuTiled) {
+ if constexpr (BITS == 2) {
+ load_mxu_tiled_w2_weights(weights_base, weight_row_indices, row_stride, offset_bytes);
+ } else {
load_mxu_tiled_weights(weights_base, weight_row_indices, row_stride, offset_bytes, params.gathered);
- return;
}
+ return;
}
Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
constexpr uint R = decltype(output_index)::value;
@@ -158,7 +187,17 @@ public:
position.group * GROUP_SIZE + group_offset + position.slice * SLICE_VALUES + chunk * CHUNK_VALUES;
const uint input_row = tile.input_row + source_row;
float input_values[CHUNK_VALUES];
- if (BITS == 8 && params.gemm_b_layout == GemmBLayout::MxuTiled) {
+ if (BITS == 2 && params.gemm_b_layout == GemmBLayout::MxuTiled) {
+ const device AT* input =
+ ops.a + input_row * params.in_vec_size + position.group * GROUP_SIZE +
+ chunk / 2 * (CHUNK_VALUES / 2) * Ops::FRAGMENT_COLS +
+ (group_offset + position.slice * SLICE_VALUES) / SLICE_VALUES * Ops::THREAD_ELEMENT_COLS + chunk % 2 * 2;
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < CHUNK_VALUES / 2; i++) {
+ input_values[i] = float(input[Ops::FRAGMENT_COLS * i]);
+ input_values[CHUNK_VALUES / 2 + i] = float(input[Ops::FRAGMENT_COLS * i + 1]);
+ }
+ } else if (BITS == 8 && params.gemm_b_layout == GemmBLayout::MxuTiled) {
const device AT* input = ops.a + input_row * params.in_vec_size + k - k % 32 / 2;
METAL_PRAGMA_UNROLL
for (uint i = 0; i < CHUNK_VALUES / 2; i++) {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
index ef08d5026..985d5fe46 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -74,9 +74,6 @@ impl GemvSpecialization {
} else if is_quant {
policy::quantized_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n, bf16_io)
} else {
- if shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH {
- return None;
- }
let input_aligned = shape.k.is_multiple_of(FP_K_BLOCK);
Some(policy::fp_tile(shape.m, shape.n, shape.k, input_aligned))
};
@@ -104,7 +101,7 @@ impl GemvSpecialization {
return None;
}
let bits = shape.b_bits.unwrap_or(0);
- if bits == 2 && gemm_b_layout == GemmBLayout::MxuTiled {
+ if bits == 2 && gemm_b_layout == GemmBLayout::MxuTiled && !shape.gathered {
return None;
}
if !is_quant && (shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH) {
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
index 9a9d38948..be35ca155 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
@@ -699,27 +699,34 @@ fn gemv_reads_mxu_tiled_layout_bf16(
#[case::w4_zp_gs32(4u32, 32u32, ScaleZeroPoint)]
#[case::w8_sym_gs64(8u32, 64u32, QuantizationMethod::ScaleSymmetric)]
#[case::w2_bias_gs128(2u32, 128u32, QuantizationMethod::ScaleBias)]
-fn gathered_gemv_reads_mxu_tiled_layout_bf16(
+fn gathered_gemv_reads_weight_layout_bf16(
#[case] bits: u32,
#[case] group_size: u32,
#[case] method: QuantizationMethod,
+ #[values(GemmBLayout::RowMajor, GemmBLayout::MxuTiled)] gemm_b_layout: GemmBLayout,
+ #[values(1, 3, 8)] m: u32,
+ #[values(256, 4096)] k: u32,
) {
let context = block_on(MetalContext::new()).expect("Metal context");
- if !context.supports_mxu {
+ if gemm_b_layout == GemmBLayout::MxuTiled && !context.supports_mxu {
return;
}
- let (k, vocab, gathered_rows) = (256u32, 256u32, 10u32);
- let input = QuantInput::<bf16>::new(1, k, vocab, group_size, bits, method, 0).with_group_output();
+ let (weight_rows, gathered_rows) = (256u32, 10u32);
+ let input = QuantInput::<bf16>::new(m, k, weight_rows, group_size, bits, method, 0).with_group_output();
let input = match bits {
2 => input.with_one_group_per_row(),
8 => input.with_signed_weight_codes(),
_ => input,
};
- let ids: Vec<u32> = (0..gathered_rows).map(|row| (row * 37 + 11) % vocab).collect();
- let dense = run_quant_cpu::<bf16>(&input);
- let expected: Vec<bf16> = ids.iter().map(|&id| dense[id as usize]).collect();
+ let gather_indices: Vec<u32> = (0..m * gathered_rows).map(|i| (i * 37 + 11) % weight_rows).collect();
+ let dense_output = run_quant_cpu::<bf16>(&input);
+ let expected: Vec<bf16> = gather_indices
+ .chunks_exact(gathered_rows as usize)
+ .zip(dense_output.chunks_exact(weight_rows as usize))
+ .flat_map(|(indices, output_row)| indices.iter().map(move |&index| output_row[index as usize]))
+ .collect();
- let ids_buffer = crate::tests::helpers::create_buffer_with_data::<Metal, u32>(&context, &ids);
+ let gather_indices_buffer = crate::tests::helpers::create_buffer_with_data::<Metal, u32>(&context, &gather_indices);
let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
&context,
@@ -728,19 +735,19 @@ fn gathered_gemv_reads_mxu_tiled_layout_bf16(
bf16::data_type(),
)
.expect("Metal matmul kernel");
- if bits != 2 {
+ if gemm_b_layout == GemmBLayout::MxuTiled {
tile_weights(&matmul, &mut buffers, &input, &context);
}
let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");
let mut arguments = quant_arguments(&mut buffers, &input);
- arguments.gather_indices = Some(&ids_buffer);
+ arguments.gather_indices = Some(&gather_indices_buffer);
arguments.n = gathered_rows;
matmul.encode(arguments, &mut command_buffer).expect("gathered GEMV");
block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
- let actual = buffer_to_vec::<Metal, bf16>(&context, &buffers.y)[..gathered_rows as usize].to_vec();
+ let actual = buffer_to_vec::<Metal, bf16>(&context, &buffers.y)[..(m * gathered_rows) as usize].to_vec();
assert_parity::<bf16>(
- &format!("gathered MxuTiled GEMV bits={bits} gs={group_size} method={method:?}"),
+ &format!("gathered {gemm_b_layout:?} GEMV m={m} k={k} bits={bits} gs={group_size} method={method:?}"),
&expected,
&actual,
0.05,