Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Int8 mxu decode gemm (#951)

eugene86334b4c0558
diff --git a/Cargo.lock b/Cargo.lock
index bba00176d..0e52f3454 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5557,9 +5557,9 @@ dependencies = [
[[package]]
name = "rustls"
-version = "0.23.45"
+version = "0.23.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
+checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf"
dependencies = [
"aws-lc-rs",
"once_cell",
@@ -6444,7 +6444,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
- "getrandom 0.4.3",
+ "getrandom 0.3.4",
"once_cell",
"rustix",
"windows-sys 0.61.2",
diff --git a/apps/mirai-gui/desktop/Cargo.lock b/apps/mirai-gui/desktop/Cargo.lock
index c423a3ab2..a61abae00 100644
--- a/apps/mirai-gui/desktop/Cargo.lock
+++ b/apps/mirai-gui/desktop/Cargo.lock
@@ -5003,9 +5003,9 @@ dependencies = [
[[package]]
name = "rustls"
-version = "0.23.45"
+version = "0.23.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
+checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf"
dependencies = [
"aws-lc-rs",
"once_cell",
@@ -6242,7 +6242,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
- "getrandom 0.4.3",
+ "getrandom 0.3.4",
"once_cell",
"rustix",
"windows-sys 0.61.2",
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs
index f0ba1dd1e..a31674ea0 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs
@@ -17,6 +17,7 @@ use crate::{
pub trait MatmulKernel: Sized + Send + Sync {
type Backend: Backend<Kernels: Kernels<MatmulKernel = Self>>;
type Error: Error + 'static;
+ type WeightsLayout: Copy + Default + PartialEq + Send + Sync;
fn new(
context: &<Self::Backend as Backend>::Context,
@@ -49,8 +50,18 @@ pub trait MatmulKernel: Sized + Send + Sync {
fn select_activation_format(
&self,
_bf16_shape: &MatmulShape,
+ _weights_layout: Self::WeightsLayout,
_context: &<Self::Backend as Backend>::Context,
) -> ActivationFormat {
ActivationFormat::Bf16
}
+
+ fn optimize_weights_layout(
+ &self,
+ _a8_shape: &MatmulShape,
+ _weights: &mut <Self::Backend as Backend>::GlobalBuffer,
+ _context: &<Self::Backend as Backend>::Context,
+ ) -> Result<Self::WeightsLayout, <Self::Backend as Backend>::Error> {
+ Ok(Self::WeightsLayout::default())
+ }
}
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
index ca115c9b1..f0519ce3e 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
@@ -1,4 +1,4 @@
-use super::{QuantParams, QuantParamsLayout};
+use super::{QuantParams, QuantParamsLayout, WeightsLayout};
use crate::{
backends::common::{
BufferRef,
@@ -35,6 +35,7 @@ pub struct QuantizedB<TB: BufferRef> {
pub mode: QuantizationMode,
pub group_size: u32,
pub signed_codes: bool,
+ pub weights_layout: WeightsLayout<TB::Backend>,
}
#[derive(Clone, Copy)]
@@ -127,6 +128,10 @@ impl<TB: BufferRef> MatmulB<TB> {
self.quantized().is_some_and(|quantized| quantized.signed_codes)
}
+ pub fn weights_layout(&self) -> WeightsLayout<TB::Backend> {
+ self.quantized().map_or_else(Default::default, |quantized| quantized.weights_layout)
+ }
+
pub fn quant_params_layout(&self) -> Option<QuantParamsLayout> {
self.quantized().map(|quantized| quantized.params.layout())
}
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
index 314c0ef72..7d2640125 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
@@ -16,7 +16,12 @@ pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection};
pub use output::{MatmulOutput, MatmulOutputError};
pub use routing::{ActivationFormat, MatmulShape};
-use crate::backends::common::gpu_types::{matmul::QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT, quantization::QuantizationMode};
+use crate::backends::common::{
+ Backend, Kernels,
+ gpu_types::{matmul::QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT, quantization::QuantizationMode},
+};
+
+pub type WeightsLayout<B> = <<<B as Backend>::Kernels as Kernels>::MatmulKernel as MatmulKernel>::WeightsLayout;
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum QuantParamsLayout {
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
index 8a91010d9..f5cf904d2 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
@@ -34,6 +34,7 @@ pub struct MatmulCpuKernel {
impl MatmulKernel for MatmulCpuKernel {
type Backend = Cpu;
type Error = MatmulCpuError;
+ type WeightsLayout = ();
fn new(
context: &CpuContext,
diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
index 4a2a7e83a..c42e4d23c 100644
--- a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
+++ b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
@@ -8,6 +8,14 @@ pub enum GemmAPrologueKind {
Int8Symmetric,
}
+#[repr(C)]
+#[derive(Debug, Display, Default, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum GemmBLayout {
+ #[default]
+ RowMajor,
+ MxuTiled,
+}
+
#[repr(C)]
#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]
#[allow(non_camel_case_types)]
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal
index bef34b39f..a8741f1fc 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal
@@ -15,7 +15,7 @@ using namespace uzu::trie;
#define ROW_TILE 16u
#define COL_TILE 32u
-#define NUM_SIMDGROUPS 2u
+#define NUM_SIMDGROUPS 4u
#define INVALID_ROW 0xffffffffu
// kh0 = k @ h0[batch]^T for one row-tile, walking head_v_dim in COL_TILE
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
index 8f8b35954..b555f029c 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
@@ -31,6 +31,11 @@ enum class GemmAPrologueKind : uint32_t {
Int8Symmetric = 1,
};
+enum class GemmBLayout : uint32_t {
+ RowMajor = 0,
+ MxuTiled = 1,
+};
+
enum class GemmTiling : uint32_t {
Tile8x32x32_Simdgroups1x1 = 0,
Tile64x32x32_Simdgroups2x2 = 1,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h
index e2ba61926..681597107 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h
@@ -88,6 +88,7 @@ struct RightStorage {
const device typename Right::ScaleElement* biases;
const device uint8_t* zero_points;
bool signed_codes;
+ GemmBLayout gemm_b_layout;
METAL_FUNC const device typename Right::ScaleElement* bias() const thread {
static_assert(Right::SCHEME == GemmBPrologueKind::ScaleBiasDequant, "bias is only valid for ScaleBiasDequant");
@@ -123,10 +124,11 @@ METAL_FUNC RightStorage<Right> pack_right(
const device Element* scales,
const device Element* biases,
const device uint8_t* zero_points,
- const bool signed_codes
+ const bool signed_codes,
+ const GemmBLayout gemm_b_layout
) {
if constexpr (!Right::QUANTIZED) {
- return {dense, nullptr, nullptr, nullptr, nullptr, false};
+ return {dense, nullptr, nullptr, nullptr, nullptr, false, GemmBLayout::RowMajor};
} else {
return {
nullptr,
@@ -134,7 +136,8 @@ METAL_FUNC RightStorage<Right> pack_right(
scales,
Right::SCHEME == GemmBPrologueKind::ScaleBiasDequant ? biases : nullptr,
Right::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant ? zero_points : nullptr,
- signed_codes
+ signed_codes,
+ gemm_b_layout
};
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
index a9f0ff38d..c9c163c11 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
@@ -90,8 +90,24 @@ struct Int8Cursor {
int row_stride;
short simdgroup_limit;
ushort simd_lane_id;
+ GemmBLayout gemm_b_layout;
+ uint chunk_stride;
METAL_FUNC Fragment load(const uint) const thread {
+ if (gemm_b_layout == GemmBLayout::MxuTiled) {
+ Fragment tile;
+ const uint4 low = *reinterpret_cast<const device uint4*>(address);
+ const uint4 high = *reinterpret_cast<const device uint4*>(address + sizeof(uint4));
+ const uint words[8] = {low.x, low.y, low.z, low.w, high.x, high.y, high.z, high.w};
+ for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short) {
+ METAL_PRAGMA_UNROLL
+ for (ushort fragment_column = 0; fragment_column < Fragment::COL_FRAGMENTS; ++fragment_column) {
+ reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, fragment_column))[row_slot] =
+ words[(fragment_row * Ops::THREAD_ELEMENT_ROWS + row_slot) * Fragment::COL_FRAGMENTS + fragment_column];
+ }
+ });
+ return tile;
+ }
return load_int8_tile<Fragment, ALIGNED, CODES_GROUPED_BY_NIBBLE>(
address,
row_stride,
@@ -100,7 +116,7 @@ struct Int8Cursor {
);
}
- METAL_FUNC void advance() thread { address += BLOCK_K; }
+ METAL_FUNC void advance() thread { address += chunk_stride; }
METAL_FUNC void begin_k_group(const uint) thread {}
};
@@ -129,8 +145,14 @@ struct W4Cursor {
bool signed_codes;
const device uint8_t* zero_points;
uint zero_point_group_stride_bytes;
+ GemmBLayout gemm_b_layout;
+ uint chunk_stride;
METAL_FUNC PackedChunk fetch(const uint chunk_index) const thread {
+ if (gemm_b_layout == GemmBLayout::MxuTiled) {
+ const uint4 words = *reinterpret_cast<const device uint4*>(current + chunk_index * chunk_stride);
+ return {{words.x, words.y, words.z, words.w}};
+ }
// Padding decodes to zero after the optional sign flip and code-origin adjustment.
const uint padding_word = (Operand::CODE_ORIGIN == 0) == signed_codes ? W4_SIGN_MASK : 0u;
const device uint8_t* base =
@@ -177,7 +199,7 @@ struct W4Cursor {
METAL_FUNC Fragment load(const uint) const thread { return decode(fetch(0)); }
- METAL_FUNC void advance() thread { current += CHUNK_BYTES; }
+ METAL_FUNC void advance() thread { current += chunk_stride; }
METAL_FUNC void begin_k_group(const uint k_offset) thread {
if constexpr (HAS_ZERO_POINTS) {
@@ -201,7 +223,9 @@ static METAL_FUNC auto make_left_cursor(
origin,
int(params->leading_dimension_a),
tile.simdgroup_limit_m,
- ushort(thread_context.simd_lane_id)
+ ushort(thread_context.simd_lane_id),
+ GemmBLayout::RowMajor,
+ uint(Int8Cursor<Fragment, ALIGNED, LeftOperand::GROUPED_BY_NIBBLE>::BLOCK_K)
};
}
@@ -219,6 +243,17 @@ static METAL_FUNC auto make_right_cursor(
const device uint8_t* current =
source.codes + size_t(tile.block_col + tile.tile_col_offset) * size_t(row_stride_bytes) +
size_t(tile.k_offset) * size_t(get_bytes_per_pack<Operand::BITS>()) / size_t(get_pack_factor<Operand::BITS>());
+ static_assert(Core::SIMDGROUP_BLOCK_N == 2 * Ops::FRAGMENT_ROWS, "the MxuTiled layout holds 32-row blocks");
+ const uint mxu_tiled_lane_bytes = Operand::BITS == 4 ? 16 : 32;
+ const uint mxu_tiled_chunk_bytes = METAL_SIMD_SIZE * mxu_tiled_lane_bytes;
+ if (source.gemm_b_layout == GemmBLayout::MxuTiled) {
+ const uint chunk_count = uint(params->K) / uint(Core::SIMDGROUP_BLOCK_K);
+ const uint row_block = uint(tile.block_col + tile.tile_col_offset) / uint(Core::SIMDGROUP_BLOCK_N);
+ current =
+ source.codes +
+ (size_t(row_block) * chunk_count + tile.k_offset / uint(Core::SIMDGROUP_BLOCK_K)) * mxu_tiled_chunk_bytes +
+ thread_context.simd_lane_id * mxu_tiled_lane_bytes;
+ }
if constexpr (Operand::BITS == 4) {
static_assert(Core::TILES_K == get_pack_factor<Operand::BITS>(), "W4 requires two K fragments");
@@ -236,7 +271,9 @@ static METAL_FUNC auto make_right_cursor(
Ops::get_position(thread_context.simd_lane_id),
source.signed_codes,
zero_points,
- zero_point_group_stride_bytes
+ zero_point_group_stride_bytes,
+ source.gemm_b_layout,
+ source.gemm_b_layout == GemmBLayout::MxuTiled ? mxu_tiled_chunk_bytes : Cursor::CHUNK_BYTES
};
} else {
static_assert(Operand::BITS == 8, "integer tile cursors support 4-bit and 8-bit codes");
@@ -245,7 +282,10 @@ static METAL_FUNC auto make_right_cursor(
origin_int8,
row_stride_bytes,
tile.simdgroup_limit_n,
- ushort(thread_context.simd_lane_id)
+ ushort(thread_context.simd_lane_id),
+ source.gemm_b_layout,
+ source.gemm_b_layout == GemmBLayout::MxuTiled ? mxu_tiled_chunk_bytes
+ : uint(Int8Cursor<Fragment, ALIGNED, false>::BLOCK_K)
};
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
index 044de6e91..acd2b8e62 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
@@ -186,18 +186,6 @@ struct IntegerSchedule {
ALIGNED_N;
}
- template <typename RightCodes, int K_GROUP_COUNT, int CHUNKS_PER_K_GROUP>
- static METAL_FUNC void fetch_k_groups(
- const thread RightCodes& right_codes,
- thread typename RightCodes::PackedChunk (&packed_right_chunks)[K_GROUP_COUNT][CHUNKS_PER_K_GROUP]
- ) {
- for_each_static_index<K_GROUP_COUNT>([&](const ushort k_group) {
- for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {
- packed_right_chunks[k_group][chunk] = right_codes.fetch(uint(k_group * CHUNKS_PER_K_GROUP + chunk));
- });
- });
- }
-
template <typename Core>
static METAL_FUNC GroupProducts<Core> multiply_device_group(
const typename Core::LeftStorage left,
@@ -242,32 +230,32 @@ struct IntegerSchedule {
return product;
}
- template <typename Core, typename LeftCodes, typename RightCodes, int CHUNKS_PER_K_GROUP>
+ template <typename Core, typename LeftCursor, typename RightCursor, int CHUNKS_PER_K_GROUP>
static METAL_FUNC GroupProducts<Core> multiply_k_group(
- thread LeftCodes& left_codes,
- thread RightCodes& right_codes,
- const thread typename RightCodes::PackedChunk (&packed_right_chunks)[CHUNKS_PER_K_GROUP]
+ thread LeftCursor& left_cursor,
+ thread RightCursor& right_cursor,
+ const thread typename RightCursor::PackedChunk (&packed_right_chunks)[CHUNKS_PER_K_GROUP]
) {
GroupProducts<Core> group_product;
group_product.clear();
if constexpr (prefetches_int4_chunks<Core>()) {
for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {
- auto left_tile = left_codes.load(uint(chunk));
- auto right_tile = right_codes.decode(packed_right_chunks[chunk]);
+ auto left_tile = left_cursor.load(uint(chunk));
+ auto right_tile = right_cursor.decode(packed_right_chunks[chunk]);
uzu::matmul::fragment_mma(group_product, left_tile, right_tile);
- left_codes.advance();
- right_codes.advance();
+ left_cursor.advance();
+ right_cursor.advance();
});
} else {
- decltype(right_codes.load(0u)) right_tiles[CHUNKS_PER_K_GROUP];
+ decltype(right_cursor.load(0u)) right_tiles[CHUNKS_PER_K_GROUP];
for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {
- right_tiles[chunk] = right_codes.load(uint(chunk));
- right_codes.advance();
+ right_tiles[chunk] = right_cursor.load(uint(chunk));
+ right_cursor.advance();
});
for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {
- auto left_tile = left_codes.load(uint(chunk));
+ auto left_tile = left_cursor.load(uint(chunk));
uzu::matmul::fragment_mma(group_product, left_tile, right_tiles[chunk]);
- left_codes.advance();
+ left_cursor.advance();
});
}
return group_product;
@@ -331,9 +319,9 @@ struct IntegerSchedule {
}
}
- auto left_codes =
+ auto left_cursor =
quantized::make_left_cursor<Core, LeftOperand, ALIGNED_M>(left_storage, params, tile, thread_context);
- auto right_codes =
+ auto right_cursor =
quantized::make_right_cursor<Core, RightOperand, ALIGNED_N>(right_storage, params, tile, thread_context);
const short2 position = Core::FragmentOps::get_position(thread_context.simd_lane_id);
@@ -356,17 +344,18 @@ struct IntegerSchedule {
const int right_group_count = int(params->aligned_inner_iterations);
constexpr int CHUNKS_PER_K_GROUP = chunks_per_k_group<Core>();
constexpr bool PREFETCH_INT4_CHUNKS = prefetches_int4_chunks<Core>();
- constexpr bool IS_ALIGNED_M_INT4_TILE16X32 =
- PREFETCH_INT4_CHUNKS && ALIGNED_M && Core::THREADGROUP_BLOCK_M == 16 && Core::THREADGROUP_BLOCK_N == 32;
- constexpr int K_GROUPS_PER_FETCH = IS_ALIGNED_M_INT4_TILE16X32 ? 4 : 1;
- constexpr int MIN_K_GROUPS_PER_THREADGROUP_FOR_BATCHED_FETCH = 16;
- using PackedRightChunk = typename decltype(right_codes)::PackedChunk;
-
- auto multiply_group = [&](const int right_group_index,
- const thread PackedRightChunk(&packed_right_chunks)[CHUNKS_PER_K_GROUP]) {
+
+ METAL_PRAGMA_NO_UNROLL
+ for (int right_group_index = 0; right_group_index < right_group_count; ++right_group_index) {
+ typename decltype(right_cursor)::PackedChunk packed_right_chunks[CHUNKS_PER_K_GROUP];
+ if constexpr (PREFETCH_INT4_CHUNKS) {
+ for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {
+ packed_right_chunks[chunk] = right_cursor.fetch(uint(chunk));
+ });
+ }
const uint right_group_offset = uint(right_group_index) * RIGHT_GROUP_SIZE;
- left_codes.begin_k_group(right_group_offset);
- right_codes.begin_k_group(right_group_offset);
+ left_cursor.begin_k_group(right_group_offset);
+ right_cursor.begin_k_group(right_group_offset);
auto load_metadata = [&]() {
left_metadata.load(left_storage, metadata_context, tile.k_offset + right_group_offset);
right_metadata.load(right_storage, metadata_context, first_right_group + uint(right_group_index));
@@ -376,38 +365,24 @@ struct IntegerSchedule {
}
GroupProducts<Core> group_product;
if constexpr (multiplies_device_groups<Core, ALIGNED_M, ALIGNED_N>()) {
- group_product =
- multiply_device_group<Core>(left_storage, right_storage, params, tile, tile.k_offset + right_group_offset);
+ if (right_storage.gemm_b_layout == GemmBLayout::MxuTiled) {
+ group_product = multiply_k_group<Core>(left_cursor, right_cursor, packed_right_chunks);
+ } else {
+ group_product = multiply_device_group<Core>(
+ left_storage,
+ right_storage,
+ params,
+ tile,
+ tile.k_offset + right_group_offset
+ );
+ }
} else {
- group_product = multiply_k_group<Core>(left_codes, right_codes, packed_right_chunks);
+ group_product = multiply_k_group<Core>(left_cursor, right_cursor, packed_right_chunks);
}
if constexpr (PREFETCH_INT4_CHUNKS) {
load_metadata();
}
accumulate_group<Core, ALIGNED_M, ALIGNED_N>(accumulator, group_product, left_metadata, right_metadata);
- };
-
- int right_group_index = 0;
- if constexpr (K_GROUPS_PER_FETCH > 1) {
- const int batched_fetch_right_group_limit =
- right_group_count >= MIN_K_GROUPS_PER_THREADGROUP_FOR_BATCHED_FETCH ? right_group_count : 0;
- METAL_PRAGMA_NO_UNROLL
- for (; right_group_index + K_GROUPS_PER_FETCH <= batched_fetch_right_group_limit;
- right_group_index += K_GROUPS_PER_FETCH) {
- PackedRightChunk packed_right_chunks[K_GROUPS_PER_FETCH][CHUNKS_PER_K_GROUP];
- fetch_k_groups(right_codes, packed_right_chunks);
- for_each_static_index<K_GROUPS_PER_FETCH>([&](const ushort k_group) {
- multiply_group(right_group_index + int(k_group), packed_right_chunks[k_group]);
- });
- }
- }
- METAL_PRAGMA_NO_UNROLL
- for (; right_group_index < right_group_count; ++right_group_index) {
- PackedRightChunk packed_right_chunks[1][CHUNKS_PER_K_GROUP];
- if constexpr (PREFETCH_INT4_CHUNKS) {
- fetch_k_groups(right_codes, packed_right_chunks);
- }
- multiply_group(right_group_index, packed_right_chunks[0]);
}
return accumulator;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
index d1d234faa..bb29d29b5 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -137,6 +137,7 @@ KERNEL(Gemm)(
const GemmDTransform output_transform SPECIALIZE,
const GemmAlignment alignment SPECIALIZE,
const bool signed_codes SPECIALIZE,
+ const GemmBLayout gemm_b_layout SPECIALIZE,
threadgroup AT a_shared[GEMM_TGA_ELEMENTS],
threadgroup BT b_shared[GEMM_TGB_ELEMENTS],
const uint group_x GROUPS(group_count_x),
@@ -162,7 +163,8 @@ KERNEL(Gemm)(
"kernel bindings and operand correction policy must agree"
);
const auto left_storage = operands::pack_left<LeftOperand, AT>(a, a_int8, a_scales, a_group_sums);
- const auto right_storage = operands::pack_right<RightOperand, BT>(b, scales, biases, zero_points, signed_codes);
+ const auto right_storage =
+ operands::pack_right<RightOperand, BT>(b, scales, biases, zero_points, signed_codes, gemm_b_layout);
if constexpr (USE_MXU) {
using Core = MxuMmaCore<DT, GEMM_TILING, TRANSPOSE_B, LeftOperand, RightOperand>;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal
new file mode 100644
index 000000000..547f1d416
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal
@@ -0,0 +1,51 @@
+#include "../../common/dsl.h"
+#include "../../common/defines.h"
+#include "../common/mxu_fragment/ops.h"
+
+using namespace metal;
+
+#define PREPARE_B_LAYOUT_THREADS 1024u
+
+template <uint BITS, uint WORDS_PER_THREAD>
+VARIANTS(BITS, 4, 8)
+VARIANTS(WORDS_PER_THREAD, 32, 72, 160)
+KERNEL(GemmPrepareBLayout)(
+ device uint* weights,
+ const constant uint& row_words,
+ const constant uint& row_block_count,
+ const uint row_block GROUPS(row_block_count),
+ const uint thread_index THREADS(PREPARE_B_LAYOUT_THREADS)
+) {
+ using Ops = uzu::matmul::MxuFragmentOps<>;
+ constexpr uint CHUNK_BYTES = BITS == 4 ? 16 : 32;
+ constexpr uint COL_FRAGMENTS = BITS == 4 ? 1 : 2;
+ constexpr uint LANE_WORDS = 2 * Ops::THREAD_ELEMENT_ROWS * COL_FRAGMENTS;
+ const uint block_words = 2 * Ops::FRAGMENT_ROWS * row_words;
+ device uint* block = weights + size_t(row_block) * block_words;
+
+ uint words[WORDS_PER_THREAD];
+ METAL_PRAGMA_UNROLL
+ for (uint slot = 0; slot < WORDS_PER_THREAD; ++slot) {
+ const uint index = slot * PREPARE_B_LAYOUT_THREADS + thread_index;
+ if (index < block_words) {
+ const uint lane = (index / LANE_WORDS) % METAL_SIMD_SIZE;
+ const uint word = index % LANE_WORDS;
+ const uint fragment_row = word / (Ops::THREAD_ELEMENT_ROWS * COL_FRAGMENTS);
+ const uint row_slot = word / COL_FRAGMENTS % Ops::THREAD_ELEMENT_ROWS;
+ const uint fragment_column = word % COL_FRAGMENTS;
+ const short2 position = Ops::get_position(ushort(lane));
+ const uint row = uint(position.y) + fragment_row * Ops::FRAGMENT_ROWS + row_slot * Ops::THREAD_ELEMENT_ROW_STRIDE;
+ const uint byte = index / (METAL_SIMD_SIZE * LANE_WORDS) * CHUNK_BYTES + uint(position.x) +
+ fragment_column * Ops::FRAGMENT_COLS;
+ words[slot] = block[row * row_words + byte / 4];
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_device);
+ METAL_PRAGMA_UNROLL
+ for (uint slot = 0; slot < WORDS_PER_THREAD; ++slot) {
+ const uint index = slot * PREPARE_B_LAYOUT_THREADS + thread_index;
+ if (index < block_words) {
+ block[index] = words[slot];
+ }
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
index c2eb07de3..4ced1b076 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -3,7 +3,8 @@ use std::collections::{HashMap, hash_map::Entry};
use crate::{
backends::{
common::{
- Backend, BufferMut, BufferRef, CommandBufferEncoding,
+ Backend, BufferMut, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,
+ Context,
gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
kernel::matmul::{
Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulOutput, MatmulShape, QuantParamsLayout,
@@ -16,11 +17,11 @@ use crate::{
context::MetalContext,
error::MetalError,
gpu_types::{
- gemm::{GemmAPrologueKind, GemmAlignment},
+ gemm::{GemmAPrologueKind, GemmAlignment, GemmBLayout},
matmul::GemmParams,
},
kernel::{
- GemmMetalKernel, GemmSplitKReduceMetalKernel,
+ GemmMetalKernel, GemmPrepareBLayoutMetalKernel, GemmSplitKReduceMetalKernel,
matmul::{
MatmulMetalError, MatmulOutputWork,
gemm::{
@@ -91,12 +92,41 @@ impl GemmKernel {
specialization.output_transform,
specialization.alignment,
specialization.signed_codes,
+ specialization.gemm_b_layout,
)?;
Ok(entry.insert(kernel))
},
}
}
+ pub fn mxu_tiled_words_per_thread(
+ k: u32,
+ bits: u32,
+ ) -> Option<u32> {
+ let words_per_thread = (k * bits / 32).div_ceil(32);
+ [32, 72, 160].into_iter().find(|&variant| variant >= words_per_thread)
+ }
+
+ pub fn prepare_gemm_b_layout(
+ weights: &mut <Metal as Backend>::GlobalBuffer,
+ rows: u32,
+ k: u32,
+ bits: u32,
+ context: &MetalContext,
+ ) -> Result<(), MetalError> {
+ let row_words = k * bits / 32;
+ let variant = Self::mxu_tiled_words_per_thread(k, bits).expect("int8 eligibility admits only tileable weights");
+ let mut command_buffer = context.create_command_buffer(Some("prepare GEMM B layout"), None)?;
+ GemmPrepareBLayoutMetalKernel::new(context, bits, variant)?.encode(
+ weights,
+ row_words,
+ rows / 32,
+ &mut command_buffer,
+ );
+ command_buffer.end_encoding().submit().wait_until_completed()?;
+ Ok(())
+ }
+
fn get_or_create_split_k_reduce(
&mut self,
context: &MetalContext,
@@ -224,6 +254,13 @@ impl GemmKernel {
};
let use_mxu = plan.engine == GemmEngine::Mxu;
+ let gemm_b_layout = quantized.as_ref().map_or(GemmBLayout::RowMajor, |quantized| quantized.weights_layout);
+ if gemm_b_layout == GemmBLayout::MxuTiled && (!use_mxu || shape.a_full_precision) {
+ return Err(MatmulMetalError::IncompatibleA {
+ path: "Gemm",
+ reason: "MxuTiled GEMM B layout requires int8 activations on the MXU",
+ });
+ }
let is_quant = quantized.is_some();
if !is_quant && !matches!(&a, MatmulA::FullPrecision { .. }) {
@@ -299,6 +336,7 @@ impl GemmKernel {
scale_strides,
zero_point_strides,
shape,
+ gemm_b_layout,
plan,
output_transform,
output_bias,
@@ -346,6 +384,7 @@ impl GemmKernel {
let specialization = GemmSpecialization::from_plan(
plan,
shape,
+ gemm_b_layout,
self.weights_data_type,
output_transform,
alignment,
@@ -387,6 +426,7 @@ impl GemmKernel {
scale_strides: QuantParamsStrides,
zero_point_strides: QuantParamsStrides,
shape: MatmulShape,
+ gemm_b_layout: GemmBLayout,
plan: GemmPlan,
output_transform: GemmDTransform,
output_bias: Option<impl BufferRef<Backend = Metal>>,
@@ -427,6 +467,7 @@ impl GemmKernel {
let part_spec = GemmSpecialization::from_plan(
plan,
shape,
+ gemm_b_layout,
self.weights_data_type,
GemmDTransform::empty(),
alignment,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
index c80f269a3..84bd19da1 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
@@ -11,7 +11,6 @@ use crate::backends::metal::gpu_types::gemm::GemmTiling;
const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;
const MXU_SKINNY_M_MAX: u32 = 16;
-const MXU_K_SPLIT_FULL_BLOCK_N_MAX: u32 = 16384;
const SKINNY_SQUARE_K_MAX: u32 = 2560;
const READOUT_N_TO_K_RATIO: u32 = 32;
const WIDE_N_DEEP_K_MIN: u32 = 4096;
@@ -63,9 +62,7 @@ pub(super) fn mxu_int8_gemm_tile(
) -> GemmTiling {
match mxu_mn_tile(true, m, n) {
GemmTiling::Tile128x128x256_Simdgroups4x4 => MXU_DEFAULT_TILE,
- GemmTiling::Tile16x32x256_Simdgroups1x1 if m < MXU_SKINNY_M_MAX || n <= MXU_K_SPLIT_FULL_BLOCK_N_MAX => {
- GemmTiling::Tile16x32x256_Simdgroups1x1x4
- },
+ GemmTiling::Tile16x32x256_Simdgroups1x1 => GemmTiling::Tile16x32x256_Simdgroups1x1x4,
tiling => tiling,
}
}
@@ -150,6 +147,7 @@ pub(super) fn split_k_target_tiles(
(true, GemmTiling::Tile32x64x256_Simdgroups2x2, Some(4)) => SPLIT_K_TARGET_TILES_A8_TILE32_W4,
(true, GemmTiling::Tile32x64x256_Simdgroups2x2, _) => SPLIT_K_TARGET_TILES_A8_TILE32_W8,
(true, GemmTiling::Tile16x32x256_Simdgroups1x1, _) => SPLIT_K_TARGET_TILES_A8_TILE16X32,
+ (true, GemmTiling::Tile16x32x256_Simdgroups1x1x4, Some(4)) => SPLIT_K_TARGET_TILES_A8,
(true, GemmTiling::Tile16x32x256_Simdgroups1x1x4, _) => SPLIT_K_TARGET_TILES_A8_TILE16X32_K_SPLIT,
(true, _, _) => SPLIT_K_TARGET_TILES_A8,
(false, _, _) => SPLIT_K_TARGET_TILES_FP,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
index 25ea3639f..1b1c4b7f6 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -5,7 +5,7 @@ use crate::{
gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},
},
- metal::gpu_types::gemm::{GemmAPrologueKind, GemmAlignment, GemmTiling},
+ metal::gpu_types::gemm::{GemmAPrologueKind, GemmAlignment, GemmBLayout, GemmTiling},
},
data_type::DataType,
};
@@ -23,6 +23,7 @@ pub(super) struct GemmSpecialization {
pub(super) bits_per_b: Option<u32>,
pub(super) b_group_size: Option<u32>,
pub(super) signed_codes: bool,
+ pub(super) gemm_b_layout: GemmBLayout,
pub(super) a_group_size: Option<u32>,
}
@@ -30,6 +31,7 @@ impl GemmSpecialization {
pub(super) fn from_plan(
plan: GemmPlan,
shape: MatmulShape,
+ gemm_b_layout: GemmBLayout,
weights_data_type: DataType,
output_transform: GemmDTransform,
alignment: GemmAlignment,
@@ -48,6 +50,7 @@ impl GemmSpecialization {
bits_per_b: shape.b_bits,
b_group_size: shape.b_group_size,
signed_codes: shape.signed_codes,
+ gemm_b_layout,
a_group_size,
};
specialization.validate()?;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/arguments.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/arguments.h
index fef97899c..8605ef66b 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/arguments.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/arguments.h
@@ -36,6 +36,15 @@ struct GemvParams {
GemmDTransform output_transform;
bool gathered;
bool signed_codes;
+ GemmBLayout gemm_b_layout;
+
+ METAL_FUNC uint output_row(uint row) const thread {
+ if (gemm_b_layout != GemmBLayout::MxuTiled || gathered) {
+ return row;
+ }
+ const uint block_row = row % 32;
+ return row - block_row + block_row / 16 * 4 + block_row / 4 % 4 + block_row % 4 * 8;
+ }
};
} // namespace gemm
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h
index 042dcced7..314bb65c5 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h
@@ -75,18 +75,18 @@ private:
if (!(FULL_TILE || input_row < params.batch_size)) {
return;
}
- device DT* output = ops.d + input_row * params.out_vec_size + tile.row0;
+ device DT* output = ops.d + input_row * params.out_vec_size;
Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
constexpr uint R = decltype(output_index)::value;
- const uint global_row = tile.row0 + R;
+ const uint global_row = params.output_row(tile.row0 + R);
if (!OutputTile<Tile, FULL_TILE>::row_in_range(global_row, params.out_vec_size)) {
return;
}
if constexpr (STAGE_HADAMARD) {
- shared_results[I * Tile::OUTPUT_ROWS + tile.local_row + R] =
+ shared_results[I * Tile::OUTPUT_ROWS + params.output_row(tile.local_row + R)] =
transform<false>(result[I][R], global_row, ops, params);
} else {
- output[R] = static_cast<DT>(transform<true>(result[I][R], global_row, ops, params));
+ output[global_row] = static_cast<DT>(transform<true>(result[I][R], global_row, ops, params));
}
});
});
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h
index 220959e8e..9bdeefe12 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h
@@ -35,7 +35,7 @@ struct QuantBSource {
uint weight_row_indices[Tile::ROWS_PER_LANE];
Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
constexpr uint R = decltype(output_index)::value;
- const uint output_row = tile.row0 + R;
+ const uint output_row = params.output_row(tile.row0 + R);
weight_row_indices[R] =
params.gathered ? ops.gather_indices[tile.input_row * params.out_vec_size + output_row] : output_row;
});
@@ -50,7 +50,7 @@ struct QuantBSource {
Metadata metadata;
metadata.load(position.group, weight_row_indices, ops, params);
for (position.slice = 0; position.slice < Slice::SLICES_PER_LANE; position.slice++) {
- current.load_weights(position, weights, weight_row_indices, row_stride, group_offset);
+ current.load_weights(position, weights, weight_row_indices, row_stride, group_offset, params);
current.accumulate(result, position, ops, params, tile, group_offset, batch_remaining, metadata);
}
position.group += Tile::GROUPS_PER_STEP;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h
index fef42725d..c6f6a962c 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h
@@ -44,6 +44,53 @@ private:
uint4 weights[Tile::ROWS_PER_LANE][SLICE_WORDS];
+ static METAL_FUNC uint mxu_tiled_offset(uint row, uint row_stride, uint byte) {
+ constexpr uint CHUNK_BYTES = BITS * 4;
+ const uint y = row % 8;
+ const uint x = byte % CHUNK_BYTES / BITS;
+ const uint lane = (x & 1) | ((y & 1) << 1) | ((y >> 1 & 1) << 2) | ((x >> 1) << 3) | ((y >> 2) << 4);
+ return row / 32 * 32 * row_stride + (byte / CHUNK_BYTES * 32 + lane) * CHUNK_BYTES + row % 32 / 8 * BITS +
+ byte % BITS;
+ }
+
+ METAL_FUNC void load_mxu_tiled_weights(
+ const device uint8_t* weights_base,
+ const thread uint (&weight_row_indices)[Tile::ROWS_PER_LANE],
+ uint row_stride,
+ uint offset_bytes,
+ bool gathered
+ ) thread {
+ constexpr uint ROW_WORDS = BITS / 4;
+ constexpr uint SEGMENT_WORDS = Tile::ROWS_PER_LANE * ROW_WORDS;
+ constexpr uint VECTOR_WORDS = SEGMENT_WORDS < 4 ? SEGMENT_WORDS : 4;
+ constexpr bool VECTOR_LOADS = 4 % Tile::ROWS_PER_LANE == 0 && VECTOR_WORDS > 1;
+ METAL_PRAGMA_UNROLL
+ for (uint segment = 0; segment < SLICE_BYTES / BITS; ++segment) {
+ const uint byte = offset_bytes + segment * BITS;
+ if (VECTOR_LOADS && !gathered) {
+ const device vec<uint, VECTOR_WORDS>* source = reinterpret_cast<const device vec<uint, VECTOR_WORDS>*>(
+ weights_base + mxu_tiled_offset(weight_row_indices[0], row_stride, byte)
+ );
+ METAL_PRAGMA_UNROLL
+ for (uint word = 0; word < SEGMENT_WORDS; ++word) {
+ reinterpret_cast<thread uint*>(weights[word / ROW_WORDS])[segment * ROW_WORDS + word % ROW_WORDS] =
+ source[word / VECTOR_WORDS][word % VECTOR_WORDS];
+ }
+ } else {
+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
+ constexpr uint R = decltype(output_index)::value;
+ METAL_PRAGMA_UNROLL
+ for (uint word = 0; word < ROW_WORDS; ++word) {
+ reinterpret_cast<thread uint*>(weights[R])[segment * ROW_WORDS + word] =
+ *reinterpret_cast<const device uint*>(
+ weights_base + mxu_tiled_offset(weight_row_indices[R], row_stride, byte + word * 4)
+ );
+ }
+ });
+ }
+ }
+ }
+
public:
static METAL_FUNC uint row_stride(const thread GemvParams& params) {
return params.in_vec_size * BITS / QuantChunk<BITS>::BITS_PER_BYTE;
@@ -54,10 +101,15 @@ public:
const device uint8_t* weights_base,
const thread uint (&weight_row_indices)[Tile::ROWS_PER_LANE],
uint row_stride,
- uint group_offset
+ uint group_offset,
+ const thread GemvParams& params
) thread {
const uint k = position.group * GROUP_SIZE + group_offset + position.slice * SLICE_VALUES;
const uint offset_bytes = k * BITS / QuantChunk<BITS>::BITS_PER_BYTE;
+ if (params.gemm_b_layout == GemmBLayout::MxuTiled) {
+ load_mxu_tiled_weights(weights_base, weight_row_indices, row_stride, offset_bytes, params.gathered);
+ return;
+ }
Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
constexpr uint R = decltype(output_index)::value;
const uint row = weight_row_indices[R];
@@ -103,9 +155,18 @@ public:
const uint k =
position.group * GROUP_SIZE + group_offset + position.slice * SLICE_VALUES + chunk * CHUNK_VALUES;
const uint input_row = tile.input_row + source_row;
- const device AT* input = ops.a + input_row * params.in_vec_size + k;
float input_values[CHUNK_VALUES];
- QuantChunk<BITS>::template load<INPUT_ALIGNED>(input, input_values, k, params.in_vec_size, input_row);
+ if (BITS == 8 && params.gemm_b_layout == GemmBLayout::MxuTiled) {
+ const device AT* input = ops.a + input_row * params.in_vec_size + k - k % 32 / 2;
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < CHUNK_VALUES / 2; i++) {
+ input_values[i] = float(input[i]);
+ input_values[CHUNK_VALUES / 2 + i] = float(input[16 + i]);
+ }
+ } else {
+ const device AT* input = ops.a + input_row * params.in_vec_size + k;
+ QuantChunk<BITS>::template load<INPUT_ALIGNED>(input, input_values, k, params.in_vec_size, input_row);
+ }
if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
METAL_PRAGMA_UNROLL
for (uint i = 0; i < CHUNK_VALUES; i++) {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
index f953ee140..282ac4bcd 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -130,6 +130,7 @@ KERNEL(Gemv)(
const GemmDTransform output_transform SPECIALIZE,
const bool gathered SPECIALIZE,
const bool signed_codes SPECIALIZE,
+ const GemmBLayout gemm_b_layout SPECIALIZE,
const bool full_tile SPECIALIZE,
threadgroup float shared_results[INPUT_ROW_TILE * OUTPUT_ROW_TILE * K_SPLIT],
const uint input_tile_idx GROUPS(batch_size.div_ceil(INPUT_ROW_TILE)),
@@ -151,7 +152,8 @@ KERNEL(Gemv)(
soft_cap,
output_transform,
gathered,
- signed_codes
+ signed_codes,
+ gemm_b_layout
};
dispatch_bool(full_tile, [&](auto full_tile_constant) {
constexpr bool FullTile = decltype(full_tile_constant)::value;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv_mxu_tiled.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv_mxu_tiled.metal
new file mode 100644
index 000000000..f958d7b64
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv_mxu_tiled.metal
@@ -0,0 +1,158 @@
+#include "../../common/dsl.h"
+#include "../../common/soft_cap.h"
+#include "../../generated/gemm.h"
+#include "../../hadamard_transform/hadamard_transform.h"
+#include "common/arguments.h"
+#include "common/quant_metadata.h"
+#include "common/tile.h"
+
+using namespace metal;
+using namespace uzu;
+using namespace uzu::gemm;
+
+#define GEMV_MXU_TILED_SIMDGROUPS 8u
+
+template <uint GROUP_SIZE>
+VARIANTS(GROUP_SIZE, 32, 64)
+KERNEL(GemvMxuTiled)(
+ const device uint4* b,
+ const device bfloat* scales,
+ const device bfloat* a,
+ device bfloat* d,
+ const device bfloat* output_bias
+ OPTIONAL(output_transform.contains(GemmDTransform::BIAS)),
+ const device int32_t* hadamard_factors
+ OPTIONAL(output_transform.contains(GemmDTransform::RHT)),
+ const constant uint& in_vec_size,
+ const constant uint& out_vec_size,
+ const constant float& ab_scale,
+ const constant uint& scale_output_stride,
+ const constant uint& scale_group_stride,
+ const constant float& soft_cap
+ OPTIONAL(output_transform.contains(GemmDTransform::SOFT_CAP)),
+ const GemmDTransform output_transform SPECIALIZE,
+ const bool signed_codes SPECIALIZE,
+ threadgroup float shared_results[GEMV_MXU_TILED_SIMDGROUPS * 32],
+ const uint block GROUPS(out_vec_size / 32),
+ const uint lane THREADS(32),
+ const uint k_slice THREADS(GEMV_MXU_TILED_SIMDGROUPS)
+) {
+ constexpr uint K_SPLIT = GEMV_MXU_TILED_SIMDGROUPS;
+ using Tile = GemvTile<1, 32, 4, 1, K_SPLIT, K_SPLIT>;
+ using Ops = GemvOperands<bfloat, bfloat, bfloat>;
+ using Metadata = QuantMetadata<Tile, bfloat, bfloat, bfloat, GemmBPrologueKind::ScaleSymmetricDequant, 8>;
+ constexpr uint CHUNK_VALUES = 32;
+ constexpr uint SEGMENT_VECTORS = 2;
+ constexpr uint BLOCK_CHUNK_VECTORS = 32 * SEGMENT_VECTORS;
+ const Ops ops = {
+ reinterpret_cast<const device uint32_t*>(b),
+ scales,
+ nullptr,
+ nullptr,
+ a,
+ d,
+ output_bias,
+ hadamard_factors,
+ nullptr
+ };
+ const GemvParams params = {
+ in_vec_size,
+ out_vec_size,
+ scale_output_stride,
+ scale_group_stride,
+ 0,
+ 0,
+ 1,
+ ab_scale,
+ soft_cap,
+ output_transform,
+ false,
+ signed_codes,
+ GemmBLayout::MxuTiled
+ };
+
+ const uint pair = lane % 16;
+ const uint parity = lane / 16;
+ const uint y = (pair & 3) | (pair >> 3 << 2);
+ const uint half_index = pair >> 2 & 1;
+ uint rows[Tile::ROWS_PER_LANE];
+ METAL_PRAGMA_UNROLL
+ for (uint slot = 0; slot < Tile::ROWS_PER_LANE; ++slot) {
+ rows[slot] = block * 32 + y + 8 * slot;
+ }
+ const uint chunks = in_vec_size / CHUNK_VALUES;
+ const device uint4* block_weights = b + block * (in_vec_size / CHUNK_VALUES) * BLOCK_CHUNK_VECTORS;
+
+ float result[1][Tile::ROWS_PER_LANE] = {{0}};
+#pragma clang loop unroll_count(2)
+ for (uint chunk = 2 * k_slice + parity; chunk < chunks; chunk += 2 * K_SPLIT) {
+ const device uint4* source = block_weights + chunk * BLOCK_CHUNK_VECTORS + pair * 2 * SEGMENT_VECTORS;
+ uint4 packed[2 * SEGMENT_VECTORS];
+ METAL_PRAGMA_UNROLL
+ for (uint vector = 0; vector < 2 * SEGMENT_VECTORS; ++vector) {
+ packed[vector] = source[vector];
+ }
+ Metadata metadata;
+ metadata.load(chunk * CHUNK_VALUES / GROUP_SIZE, rows, ops, params);
+ float partial[1][Tile::ROWS_PER_LANE] = {{0}};
+ float input_sum[1] = {0};
+ const device bfloat* input = a + chunk * CHUNK_VALUES + 8 * half_index;
+ METAL_PRAGMA_UNROLL
+ for (uint step = 0; step < 2; ++step) {
+ float input_values[8];
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < 4; ++i) {
+ input_values[i] = float(input[4 * step + i]);
+ input_values[4 + i] = float(input[16 + 4 * step + i]);
+ }
+ METAL_PRAGMA_UNROLL
+ for (uint slot = 0; slot < Tile::ROWS_PER_LANE; ++slot) {
+ float weight_values[8];
+ metadata.decode(packed, step * 4 + slot, slot, weight_values);
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < 8; ++i) {
+ partial[0][slot] = fma(input_values[i], weight_values[i], partial[0][slot]);
+ }
+ }
+ }
+ metadata.fold(result, partial, input_sum);
+ }
+
+ METAL_PRAGMA_UNROLL
+ for (uint slot = 0; slot < Tile::ROWS_PER_LANE; ++slot) {
+ result[0][slot] += simd_shuffle_xor(result[0][slot], 4);
+ result[0][slot] += simd_shuffle_xor(result[0][slot], 16);
+ if ((lane & 20) == 0) {
+ shared_results[k_slice * 32 + y + 8 * slot] = result[0][slot];
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (k_slice != 0) {
+ return;
+ }
+
+ float value = 0.0f;
+ METAL_PRAGMA_UNROLL
+ for (uint slice = 0; slice < K_SPLIT; ++slice) {
+ value += shared_results[slice * 32 + lane];
+ }
+ const uint row = block * 32 + lane;
+ const bool fused_hadamard = output_transform.contains(GemmDTransform::RHT);
+ if (output_transform.contains(GemmDTransform::SCALE)) {
+ value *= ab_scale;
+ }
+ if (!fused_hadamard && output_transform.contains(GemmDTransform::BIAS)) {
+ value += float(output_bias[row]);
+ }
+ if (output_transform.contains(GemmDTransform::SOFT_CAP)) {
+ value = apply_soft_cap(value, soft_cap);
+ }
+ bfloat output = bfloat(value);
+ if (fused_hadamard) {
+ output = simdgroup_output_random_hadamard_transform(ushort(lane), output, hadamard_factors[row]);
+ if (output_transform.contains(GemmDTransform::BIAS)) {
+ output += output_bias[row];
+ }
+ }
+ d[row] = output;
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
index e3ec42f2a..0a29f270d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -14,8 +14,9 @@ use crate::{
command_buffer::MetalCommandBufferEncoding,
context::MetalContext,
error::MetalError,
+ gpu_types::gemm::GemmBLayout,
kernel::{
- GemvMetalKernel,
+ GemvMetalKernel, GemvMxuTiledMetalKernel,
matmul::{
MatmulOutputWork,
gemv::policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK},
@@ -27,6 +28,7 @@ use crate::{
};
const GEMV_MAX_BATCH: u32 = 8;
+const MXU_TILED_KERNEL_MIN_K: u32 = 2048;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct GemvSpecialization {
@@ -43,12 +45,15 @@ pub struct GemvSpecialization {
group_lanes: u32,
gathered: bool,
signed_codes: bool,
+ gemm_b_layout: GemmBLayout,
full_tile: bool,
+ mxu_tiled_kernel: bool,
}
impl GemvSpecialization {
pub fn select_shape(
shape: &MatmulShape,
+ gemm_b_layout: GemmBLayout,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
@@ -58,6 +63,14 @@ impl GemvSpecialization {
let is_quant = shape.is_quant();
let bits = shape.b_bits.unwrap_or(0);
let bf16_io = input_data_type == DataType::BF16 && output_data_type == DataType::BF16;
+ let mxu_tiled_kernel = gemm_b_layout == GemmBLayout::MxuTiled
+ && shape.b_prologue == GemmBPrologueKind::ScaleSymmetricDequant
+ && shape.m == 1
+ && !shape.gathered
+ && bf16_io
+ && weights_data_type == DataType::BF16
+ && shape.k >= MXU_TILED_KERNEL_MIN_K
+ && shape.n >= 32 * gpu_core_count;
let tile = if is_quant && shape.gathered {
policy::gathered_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n)
} else if is_quant {
@@ -71,6 +84,15 @@ impl GemvSpecialization {
shape.k,
bf16_io,
)
+ .map(|tile| match gemm_b_layout {
+ GemmBLayout::MxuTiled if mxu_tiled_kernel => {
+ policy::GemvTile::quantized_output_tile(8, 32, 1, 32, tile.group_lanes)
+ },
+ GemmBLayout::MxuTiled if tile.output_row_tile() < 16 => {
+ policy::GemvTile::quantized_output_tile(8, 16, 1, 32, tile.group_lanes)
+ },
+ _ => tile,
+ })
} else {
let mixed_precision = weights_data_type == DataType::F32
&& (input_data_type != DataType::F32 || output_data_type != DataType::F32);
@@ -80,11 +102,17 @@ impl GemvSpecialization {
let input_aligned = shape.k.is_multiple_of(FP_K_BLOCK);
Some(policy::fp_tile(gpu_core_count, apple_gpu_family, shape.m, shape.n, shape.k, input_aligned))
};
- Self::select_tile(shape, weights_data_type, input_data_type, output_data_type, tile?)
+ Self::select_tile(shape, gemm_b_layout, weights_data_type, input_data_type, output_data_type, tile?).map(
+ |specialization| Self {
+ mxu_tiled_kernel,
+ ..specialization
+ },
+ )
}
pub fn select_tile(
shape: &MatmulShape,
+ gemm_b_layout: GemmBLayout,
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
@@ -137,7 +165,9 @@ impl GemvSpecialization {
group_lanes: tile.group_lanes,
gathered: shape.gathered,
signed_codes: shape.signed_codes,
+ gemm_b_layout,
full_tile: full_tile(shape, tile),
+ mxu_tiled_kernel: false,
};
Some(specialization)
}
@@ -175,6 +205,7 @@ impl GemvSpecialization {
self.output_transform,
self.gathered,
self.signed_codes,
+ self.gemm_b_layout,
self.full_tile,
)
}
@@ -226,6 +257,7 @@ pub struct GemvKernel {
input_data_type: DataType,
output_data_type: DataType,
pipelines: HashMap<GemvSpecialization, GemvMetalKernel>,
+ mxu_tiled_pipelines: HashMap<GemvSpecialization, GemvMxuTiledMetalKernel>,
}
impl GemvKernel {
@@ -239,6 +271,7 @@ impl GemvKernel {
input_data_type,
output_data_type,
pipelines: HashMap::new(),
+ mxu_tiled_pipelines: HashMap::new(),
}
}
@@ -320,6 +353,38 @@ impl GemvKernel {
});
let output_group_count = n.div_ceil(specialization.output_row_tile());
let context = command_buffer.context();
+ if specialization.mxu_tiled_kernel {
+ let MatmulB::Quantized(quantized) = b else {
+ return Err(MatmulMetalError::UnsupportedLayout {
+ path: "Gemv",
+ });
+ };
+ let pipeline = match self.mxu_tiled_pipelines.entry(specialization) {
+ Entry::Occupied(entry) => entry.into_mut(),
+ Entry::Vacant(entry) => entry.insert(GemvMxuTiledMetalKernel::new(
+ context,
+ specialization.group_size,
+ specialization.output_transform,
+ specialization.signed_codes,
+ )?),
+ };
+ pipeline.encode(
+ quantized.codes,
+ quantized.scales,
+ a,
+ d.reborrow(),
+ gemv_bias,
+ gemv_rht_factors,
+ k,
+ n,
+ ab_scale,
+ scale_strides.output_stride,
+ scale_strides.group_stride,
+ soft_cap,
+ command_buffer,
+ );
+ return Ok(());
+ }
let pipeline = self.get_or_create(context, specialization)?;
match b {
MatmulB::FullPrecision {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
index deeb74426..ad2e00e1e 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
@@ -3,7 +3,7 @@ use metal::MTLGPUFamily;
use crate::{
backends::{
common::{
- BufferMut, BufferRef, CommandBufferEncoding,
+ Backend, BufferMut, BufferRef, CommandBufferEncoding,
kernel::{
ActivationQuantization,
activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
@@ -16,7 +16,8 @@ use crate::{
Metal,
command_buffer::MetalCommandBufferEncoding,
context::MetalContext,
- gpu_types::gemm::GemmTiling,
+ error::MetalError,
+ gpu_types::gemm::{GemmBLayout, GemmTiling},
kernel::matmul::{
GemmKernel, MatmulMetalError,
gemm::{self, GemmPlan, GemmProblem},
@@ -80,6 +81,7 @@ impl MatmulMetalKernel {
pub fn choose_dispatch(
shape: &MatmulShape,
+ gemm_b_layout: GemmBLayout,
device_name: &str,
gpu_core_count: u32,
apple_gpu_family: MTLGPUFamily,
@@ -98,14 +100,22 @@ impl MatmulMetalKernel {
if let Some(route) = qmv::route(device_name, apple_gpu_family, supports_mxu, shape, all_bf16) {
return match route {
QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) => MatmulDispatch::Gemv(
- GemvSpecialization::select_tile(shape, weights_data_type, input_data_type, output_data_type, tile)
- .expect("typed QMV route must contain a legal GEMV tile"),
+ GemvSpecialization::select_tile(
+ shape,
+ gemm_b_layout,
+ weights_data_type,
+ input_data_type,
+ output_data_type,
+ tile,
+ )
+ .expect("typed QMV route must contain a legal GEMV tile"),
),
QmvRoute::MainGemm(plan) => MatmulDispatch::Gemm(plan),
};
}
let gemv = GemvSpecialization::select_shape(
shape,
+ gemm_b_layout,
weights_data_type,
input_data_type,
output_data_type,
@@ -127,10 +137,12 @@ impl MatmulMetalKernel {
fn select_dispatch(
&self,
shape: &MatmulShape,
+ gemm_b_layout: GemmBLayout,
context: &MetalContext,
) -> MatmulDispatch {
Self::choose_dispatch(
shape,
+ gemm_b_layout,
&context.device_name,
context.gpu_core_count,
context.apple_gpu_family,
@@ -145,6 +157,7 @@ impl MatmulMetalKernel {
impl MatmulKernel for MatmulMetalKernel {
type Backend = Metal;
type Error = MatmulMetalError;
+ type WeightsLayout = GemmBLayout;
fn new(
context: &MetalContext,
@@ -193,6 +206,8 @@ impl MatmulKernel for MatmulMetalKernel {
|| shape.b_leading_dimension.is_some()
|| !shape.k.is_multiple_of(ACTIVATION_SCALE_GROUP_SIZE)
|| !shape.k.is_multiple_of(weight_group_size)
+ || !shape.n.is_multiple_of(32)
+ || shape.b_bits.and_then(|bits| GemmKernel::mxu_tiled_words_per_thread(shape.k, bits)).is_none()
{
return None;
}
@@ -203,25 +218,48 @@ impl MatmulKernel for MatmulMetalKernel {
fn select_activation_format(
&self,
bf16_shape: &MatmulShape,
+ weights_layout: GemmBLayout,
context: &MetalContext,
) -> ActivationFormat {
if bf16_shape.params_layout != Some(QuantParamsLayout::GroupOutput)
|| !supports_integer_right_operand(bf16_shape)
- || matches!(self.select_dispatch(bf16_shape, context), MatmulDispatch::Gemv(_))
+ || matches!(self.select_dispatch(bf16_shape, weights_layout, context), MatmulDispatch::Gemv(_))
{
return ActivationFormat::Bf16;
}
+ if weights_layout == GemmBLayout::MxuTiled {
+ return ActivationFormat::Int8;
+ }
let a8_shape = MatmulShape {
a_full_precision: false,
..*bf16_shape
};
- match self.select_dispatch(&a8_shape, context) {
+ match self.select_dispatch(&a8_shape, weights_layout, context) {
MatmulDispatch::Gemm(plan) if plan.engine == gemm::GemmEngine::Mxu => ActivationFormat::Int8,
_ => ActivationFormat::Bf16,
}
}
+ fn optimize_weights_layout(
+ &self,
+ a8_shape: &MatmulShape,
+ weights: &mut <Metal as Backend>::GlobalBuffer,
+ context: &MetalContext,
+ ) -> Result<GemmBLayout, MetalError> {
+ if self.select_activation_quantization(a8_shape, context).is_none() {
+ return Ok(GemmBLayout::RowMajor);
+ }
+ GemmKernel::prepare_gemm_b_layout(
+ weights,
+ a8_shape.n,
+ a8_shape.k,
+ a8_shape.b_bits.expect("quantized weights"),
+ context,
+ )?;
+ Ok(GemmBLayout::MxuTiled)
+ }
+
fn encode(
&mut self,
arguments: MatmulArguments<
@@ -235,7 +273,7 @@ impl MatmulKernel for MatmulMetalKernel {
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<(), Self::Error> {
let shape = MatmulShape::from_arguments(&arguments);
- let plan = match self.select_dispatch(&shape, command_buffer.context()) {
+ let plan = match self.select_dispatch(&shape, arguments.b.weights_layout(), command_buffer.context()) {
MatmulDispatch::Gemv(gemv) => {
return self.gemv.encode(arguments, gemv, &self.output_work, command_buffer);
},
diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
index ce609b594..172ecda61 100644
--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
@@ -118,6 +118,15 @@ impl<B: Backend> LinearMatmul<B> {
self.matrix.try_prepare_a8_storage().then_some(quantization)
}
+ pub fn optimize_weights_layout(
+ &mut self,
+ context: &B::Context,
+ ) -> Result<(), B::Error> {
+ let shape = self.matmul_shape(1, false);
+ let kernel = self.kernel.lock();
+ self.matrix.optimize_weights_layout(|weights| kernel.optimize_weights_layout(&shape, weights, context))
+ }
+
pub(super) fn encode_with_a(
&self,
a: MatmulA<impl BufferRef<Backend = B>>,
@@ -204,6 +213,6 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
context: &B::Context,
) -> ActivationFormat {
let bf16_shape = self.matmul_shape(batch_dim, true);
- self.kernel.lock().select_activation_format(&bf16_shape, context)
+ self.kernel.lock().select_activation_format(&bf16_shape, self.matmul_b().weights_layout(), context)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
index 4f8e5a9ef..4772b0f08 100644
--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
@@ -73,6 +73,9 @@ impl<B: Backend> RHTLinearWrapper<B> {
context,
)?;
let activation_quantization = inner_linear.prepare_a8(context);
+ if activation_quantization.is_some() {
+ inner_linear.optimize_weights_layout(context).map_err(RHTLinearWrapperError::BackendError)?;
+ }
Self::build_self_contained(
input_data_type,
LinearInputPreparation {
@@ -115,6 +118,9 @@ impl<B: Backend> RHTLinearWrapper<B> {
rht_signs,
activation_quantization: inner_linear.prepare_a8(context),
};
+ if input_preparation.activation_quantization.is_some() {
+ inner_linear.optimize_weights_layout(context).map_err(RHTLinearWrapperError::BackendError)?;
+ }
if input_preparation.activation_quantization.is_some() && !allow_prequantized_activation {
let wrapper = Self::build_self_contained(input_data_type, input_preparation, inner_linear, context)?;
Ok(Some((Box::new(wrapper), None)))
diff --git a/crates/uzu-engine/src/encodable_block/weight_matrix.rs b/crates/uzu-engine/src/encodable_block/weight_matrix.rs
index a80036620..0ad954d29 100644
--- a/crates/uzu-engine/src/encodable_block/weight_matrix.rs
+++ b/crates/uzu-engine/src/encodable_block/weight_matrix.rs
@@ -5,7 +5,7 @@ use crate::{
backends::common::{
Backend, BufferCpuAccessible, BufferMut,
gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},
- kernel::matmul::{MatmulB, QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection},
+ kernel::matmul::{MatmulB, QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection, WeightsLayout},
},
config::weight_matrix::{AnyWeightMatrixSpec, Layout},
data_type::DataType,
@@ -86,6 +86,7 @@ struct Quantized<B: Backend> {
params: QuantParams,
info: QuantizationInfo,
signed_codes: bool,
+ weights_layout: WeightsLayout<B>,
}
pub struct WeightMatrix<B: Backend> {
@@ -163,6 +164,7 @@ impl<B: Backend> WeightMatrix<B> {
params,
info,
signed_codes: false,
+ weights_layout: Default::default(),
}),
})
}
@@ -204,9 +206,23 @@ impl<B: Backend> WeightMatrix<B> {
mode,
group_size,
signed_codes,
+ weights_layout: quantized.weights_layout,
})
}
+ pub fn optimize_weights_layout<E>(
+ &mut self,
+ optimize: impl FnOnce(&mut B::GlobalBuffer) -> Result<WeightsLayout<B>, E>,
+ ) -> Result<(), E> {
+ let Some(quantized) =
+ self.quantized.as_mut().filter(|quantized| quantized.weights_layout == WeightsLayout::<B>::default())
+ else {
+ return Ok(());
+ };
+ quantized.weights_layout = optimize(&mut self.values)?;
+ Ok(())
+ }
+
pub fn try_prepare_a8_storage(&mut self) -> bool {
let Some(quantized) = self.quantized.as_mut() else {
return false;
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs
index abf3ca05f..e3fb0eded 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs
@@ -21,7 +21,7 @@ use crate::{
},
},
},
- metal::{Metal, context::MetalContext, kernel::matmul::gemm::GemmEngine},
+ metal::{Metal, context::MetalContext, gpu_types::gemm::GemmBLayout, kernel::matmul::gemm::GemmEngine},
},
data_type::DataType,
tests::{
@@ -146,6 +146,7 @@ impl BenchmarkData {
mode: self.mode,
group_size: self.group_size,
signed_codes: false,
+ weights_layout: GemmBLayout::RowMajor,
}),
b_leading_dimension: None,
b_transpose: true,
@@ -201,6 +202,7 @@ fn encode_step(
mode: data.mode,
group_size: data.group_size,
signed_codes: !matches!(data.mode, QuantizationMode::U4),
+ weights_layout: GemmBLayout::RowMajor,
}),
b_leading_dimension: None,
b_transpose: true,
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
index ac3233c8b..2dbc9e8bb 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
@@ -3,8 +3,12 @@ use uzu_engine_macros::uzu_test;
use super::*;
use crate::backends::{
- common::{gpu_types::gemm::GemmDTransform, kernel::matmul::QuantParamsLayout},
- metal::kernel::matmul::MatmulMetalKernel,
+ common::{
+ Context,
+ gpu_types::gemm::GemmDTransform,
+ kernel::matmul::{MatmulKernel, QuantParamsLayout},
+ },
+ metal::{context::MetalContext, kernel::matmul::MatmulMetalKernel},
};
fn shape(
@@ -118,10 +122,9 @@ fn selection_fallbacks_and_split_k_are_preserved() {
assert_eq!(problem(quant(shape(64, 4096, 4095)), DataType::BF16).select_plan().engine, Simdgroup);
for (m, n, k, expected_tiling, expected_split_k) in [
- (16, 4096, 4096, Tile16x32x256_Simdgroups1x1x4, 4),
+ (16, 4096, 4096, Tile16x32x256_Simdgroups1x1x4, 2),
(16, 1024, 4096, Tile16x32x256_Simdgroups1x1x4, 8),
- (16, 34816, 4096, Tile16x32x256_Simdgroups1x1, 1),
- (8, 34816, 4096, Tile16x32x256_Simdgroups1x1x4, 1),
+ (16, 34816, 4096, Tile16x32x256_Simdgroups1x1x4, 1),
(8, 4096, 384, Tile16x32x256_Simdgroups1x1, 1),
] {
let mut a8 = quant(shape(m, n, k));
@@ -197,3 +200,39 @@ fn gemv_gemm_route_boundaries_are_preserved() {
let plan = problem(gathered, DataType::BF16).select_plan();
assert!(!MatmulMetalKernel::prefer_gemm_over_gemv(gathered, plan, DataType::BF16, DataType::BF16, DataType::BF16,));
}
+
+#[uzu_test]
+fn int8_activations_require_mxu_tiled_weights() {
+ let context = MetalContext::new().expect("Metal context");
+ if !context.supports_mxu {
+ return;
+ }
+ let kernel =
+ MatmulMetalKernel::new(&context, DataType::BF16, DataType::BF16, DataType::BF16).expect("matmul kernel");
+ for (n, k, bits, expected) in [
+ (96, 256, 4, true),
+ (72, 256, 4, false),
+ (64, 40960, 4, true),
+ (64, 41088, 4, false),
+ (64, 20480, 8, true),
+ (64, 20608, 8, false),
+ ] {
+ let shape = MatmulShape {
+ b_prologue: if bits == 4 {
+ GemmBPrologueKind::ScaleZeroPointDequant
+ } else {
+ GemmBPrologueKind::ScaleSymmetricDequant
+ },
+ b_bits: Some(bits),
+ signed_codes: bits == 8,
+ a_full_precision: false,
+ params_layout: Some(QuantParamsLayout::GroupOutput),
+ ..quant(shape(16, n, k))
+ };
+ assert_eq!(
+ kernel.select_activation_quantization(&shape, &context).is_some(),
+ expected,
+ "n={n} k={k} bits={bits}"
+ );
+ }
+}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
index c4109b892..4ebc8fa76 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
@@ -3,9 +3,12 @@ use uzu_engine_macros::uzu_test;
use super::*;
use crate::{
- backends::common::{
- gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
- kernel::matmul::{MatmulShape, QuantParamsLayout},
+ backends::{
+ common::{
+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
+ kernel::matmul::{MatmulShape, QuantParamsLayout},
+ },
+ metal::gpu_types::gemm::GemmBLayout,
},
data_type::DataType,
};
@@ -151,6 +154,7 @@ fn block_unaligned_quantized_k_stays_on_gemv() {
assert!(
super::super::kernel::GemvSpecialization::select_shape(
&shape,
+ GemmBLayout::RowMajor,
DataType::BF16,
DataType::BF16,
DataType::BF16,
@@ -166,6 +170,7 @@ fn specialization_preserves_quantized_route() {
let select = |n, transform| {
super::super::kernel::GemvSpecialization::select_shape(
&quant_shape(4, n, transform),
+ GemmBLayout::RowMajor,
DataType::BF16,
DataType::BF16,
DataType::BF16,
@@ -186,6 +191,7 @@ fn gathered_group_major_is_a_gemv_route() {
assert!(
super::super::kernel::GemvSpecialization::select_tile(
&shape,
+ GemmBLayout::RowMajor,
DataType::BF16,
DataType::BF16,
DataType::BF16,
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
index 14d21ae85..c9e647499 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
@@ -9,7 +9,10 @@ use crate::{
gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
kernel::matmul::{MatmulShape, QuantParamsLayout},
},
- metal::kernel::matmul::{MatmulDispatch, MatmulMetalKernel, gemv::GemvSpecialization},
+ metal::{
+ gpu_types::gemm::GemmBLayout,
+ kernel::matmul::{MatmulDispatch, MatmulMetalKernel, gemv::GemvSpecialization},
+ },
},
data_type::DataType,
};
@@ -94,6 +97,7 @@ fn table_is_complete_and_fingerprint_is_stable() {
assert_eq!(route(device_name, apple_gpu_family, supports_mxu, &problem, true), Some(selected));
let runtime = MatmulMetalKernel::choose_dispatch(
&problem,
+ GemmBLayout::RowMajor,
device_name,
gpu_core_count,
apple_gpu_family,
@@ -105,6 +109,7 @@ fn table_is_complete_and_fingerprint_is_stable() {
if let QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) = selected {
let specialization = GemvSpecialization::select_tile(
&problem,
+ GemmBLayout::RowMajor,
DataType::BF16,
DataType::BF16,
DataType::BF16,
@@ -133,7 +138,17 @@ fn weaver_linears_select_the_tuned_tile() {
p.d_transform = GemmDTransform::RHT;
let tile = GemvTile::quantized_output_tile(2, 16, 4, 8, 1);
assert_eq!(route("Apple M5 Max", MTLGPUFamily::Apple10, true, &p, true), Some(QmvRoute::Tuned(tile)));
- assert!(GemvSpecialization::select_tile(&p, DataType::BF16, DataType::BF16, DataType::BF16, tile).is_some());
+ assert!(
+ GemvSpecialization::select_tile(
+ &p,
+ GemmBLayout::RowMajor,
+ DataType::BF16,
+ DataType::BF16,
+ DataType::BF16,
+ tile
+ )
+ .is_some()
+ );
}
}
@@ -161,11 +176,29 @@ fn exact_lookup_rejects_non_matrix_inputs() {
let QmvRoute::Tuned(tile) = selected else {
panic!("test anchor must use a tuned tile");
};
- let deferred = GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).unwrap();
+ let deferred = GemvSpecialization::select_tile(
+ &rht,
+ GemmBLayout::RowMajor,
+ DataType::BF16,
+ DataType::BF16,
+ DataType::BF16,
+ tile,
+ )
+ .unwrap();
assert_eq!(deferred.output_row_tile(), 16);
assert!(!deferred.fuses_rht());
rht.n -= 1;
- assert!(GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).is_none());
+ assert!(
+ GemvSpecialization::select_tile(
+ &rht,
+ GemmBLayout::RowMajor,
+ DataType::BF16,
+ DataType::BF16,
+ DataType::BF16,
+ tile
+ )
+ .is_none()
+ );
let p = problem(7, 5120, 17408, 8, 64, GemmBPrologueKind::ScaleSymmetricDequant);
assert!(matches!(route("Apple M5 Max", MTLGPUFamily::Apple10, true, &p, true), Some(QmvRoute::MainGemm(_))));
@@ -179,6 +212,7 @@ fn normal_routing_handles_inputs_outside_the_frozen_matrix() {
assert_eq!(route("Apple M1", MTLGPUFamily::Apple7, false, &problem, true), None);
let specialization = GemvSpecialization::select_shape(
&problem,
+ GemmBLayout::RowMajor,
DataType::BF16,
DataType::BF16,
DataType::BF16,
@@ -188,7 +222,7 @@ fn normal_routing_handles_inputs_outside_the_frozen_matrix() {
.expect("normal M1 policy should select GEMV for this anchor");
assert!(matches!(
MatmulMetalKernel::choose_dispatch(
- &problem,
+ &problem, GemmBLayout::RowMajor,
"Apple M1",
8,
MTLGPUFamily::Apple7,
@@ -208,12 +242,26 @@ fn normal_routing_handles_inputs_outside_the_frozen_matrix() {
let mut with_rht = fp;
with_rht.d_transform = GemmDTransform::RHT;
for (cores, family) in [(8, MTLGPUFamily::Apple7), (40, MTLGPUFamily::Apple10)] {
- let plain =
- GemvSpecialization::select_shape(&fp, DataType::BF16, DataType::BF16, DataType::BF16, cores, family)
- .expect("generic M=1 FP GEMV tile");
- let rht =
- GemvSpecialization::select_shape(&with_rht, DataType::BF16, DataType::BF16, DataType::BF16, cores, family)
- .expect("generic M=1 FP RHT GEMV tile");
+ let plain = GemvSpecialization::select_shape(
+ &fp,
+ GemmBLayout::RowMajor,
+ DataType::BF16,
+ DataType::BF16,
+ DataType::BF16,
+ cores,
+ family,
+ )
+ .expect("generic M=1 FP GEMV tile");
+ let rht = GemvSpecialization::select_shape(
+ &with_rht,
+ GemmBLayout::RowMajor,
+ DataType::BF16,
+ DataType::BF16,
+ DataType::BF16,
+ cores,
+ family,
+ )
+ .expect("generic M=1 FP RHT GEMV tile");
assert!(plain.output_row_tile() < 32);
assert_eq!(rht, plain);
}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
index 860ed111a..8b647b1a0 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
@@ -15,11 +15,19 @@ use crate::{
kernel::{
Kernels,
activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
- matmul::{MatmulDOps, MatmulKernel, MatmulOutput, QuantParams, QuantParamsLayout, QuantParamsStrides},
+ matmul::{
+ MatmulDOps, MatmulKernel, MatmulOutput, MatmulShape, QuantParams, QuantParamsLayout,
+ QuantParamsStrides,
+ },
},
},
cpu::Cpu,
- metal::{Metal, context::MetalContext, kernel::matmul::gemm::GemmEngine},
+ metal::{
+ Metal,
+ context::MetalContext,
+ gpu_types::gemm::GemmBLayout,
+ kernel::matmul::{MatmulMetalKernel, gemm::GemmEngine},
+ },
},
data_type::DataTypeRepr,
tests::{
@@ -28,9 +36,25 @@ use crate::{
},
};
+fn tile_weights<T: DataTypeRepr + Float>(
+ matmul: &MatmulMetalKernel,
+ buffers: &mut QuantBuffers<Metal, T>,
+ input: &QuantInput<T>,
+ context: &MetalContext,
+) {
+ let shape = MatmulShape {
+ a_full_precision: false,
+ ..MatmulShape::from_arguments(&quant_arguments(buffers, input))
+ };
+ buffers.weights_layout =
+ matmul.optimize_weights_layout(&shape, &mut buffers.w, context).expect("optimize weights layout");
+ assert_eq!(buffers.weights_layout, GemmBLayout::MxuTiled);
+}
+
fn run_quant_metal<T: DataTypeRepr + Float>(
context: &MetalContext,
input: &QuantInput<T>,
+ gemm_b_layout: GemmBLayout,
dispatch: Option<GemmEngine>,
) -> Vec<T> {
let mut buffers = QuantBuffers::<Metal, T>::allocate(context, input);
@@ -41,6 +65,9 @@ fn run_quant_metal<T: DataTypeRepr + Float>(
T::data_type(),
)
.expect("MatmulMetalKernel");
+ if gemm_b_layout == GemmBLayout::MxuTiled {
+ tile_weights(&matmul, &mut buffers, input, context);
+ }
let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");
let args = quant_arguments(&mut buffers, input);
if let Some(engine) = dispatch {
@@ -118,7 +145,7 @@ fn run_parity<T: DataTypeRepr + Float + Debug + Display>(
) {
let context = MetalContext::new().expect("Metal context");
let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 0);
- let unified = run_quant_metal::<T>(&context, &input, dispatch);
+ let unified = run_quant_metal::<T>(&context, &input, GemmBLayout::RowMajor, dispatch);
let reference = run_quant_cpu::<T>(&input);
assert_parity::<T>(
@@ -493,7 +520,7 @@ fn mxu_quant_parity_bf16(
return;
}
let input = QuantInput::<bf16>::new(m, k, n, gs, bits, method, 0);
- let actual = run_quant_metal::<bf16>(&context, &input, Some(GemmEngine::Mxu));
+ let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::RowMajor, Some(GemmEngine::Mxu));
let reference = run_quant_cpu::<bf16>(&input);
assert_parity::<bf16>(
&format!("MXU m={m} k={k} n={n} gs={gs} bits={bits} method={method:?}"),
@@ -506,30 +533,43 @@ fn mxu_quant_parity_bf16(
#[rstest]
#[test_attr(uzu_test)]
-#[case::sym_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]
-#[case::bias_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleBias)]
-#[case::zp_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleZeroPoint)]
-#[case::m1_bias_w4_gs64(1u32, 64u32, 4u32, QuantizationMethod::ScaleBias)]
+#[case::sym_w8_gs64(16u32, 256u32, 128u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]
+#[case::bias_w4_gs32(16u32, 256u32, 128u32, 32u32, 4u32, QuantizationMethod::ScaleBias)]
+#[case::zp_w4_gs32(16u32, 256u32, 128u32, 32u32, 4u32, QuantizationMethod::ScaleZeroPoint)]
+#[case::m1_bias_w4_gs64(1u32, 256u32, 128u32, 64u32, 4u32, QuantizationMethod::ScaleBias)]
+#[case::w8_m1(1u32, 256u32, 96u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]
+#[case::w8_m16_split_k(16u32, 4096u32, 1024u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]
+#[case::w8_m64(64u32, 512u32, 256u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]
+#[case::w4_m1(1u32, 256u32, 96u32, 64u32, 4u32, QuantizationMethod::ScaleZeroPoint)]
+#[case::w4_m16_split_k(16u32, 4096u32, 1024u32, 64u32, 4u32, QuantizationMethod::ScaleZeroPoint)]
+#[case::w4_m24(24u32, 512u32, 96u32, 64u32, 4u32, QuantizationMethod::ScaleZeroPoint)]
+#[case::w4_m128_bias(128u32, 512u32, 256u32, 64u32, 4u32, QuantizationMethod::ScaleBias)]
+#[case::w4_m16_k17408(16u32, 17408u32, 64u32, 64u32, 4u32, QuantizationMethod::ScaleZeroPoint)]
+#[case::w8_m16_k17408(16u32, 17408u32, 64u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]
fn a8w_mxu_parity_bf16(
#[case] m: u32,
+ #[case] k: u32,
+ #[case] n: u32,
#[case] weight_gs: u32,
#[case] bits: u32,
#[case] method: QuantizationMethod,
+ #[values(GemmBLayout::RowMajor, GemmBLayout::MxuTiled)] gemm_b_layout: GemmBLayout,
) {
let context = MetalContext::new().expect("Metal context");
if !context.supports_mxu {
return;
}
- let (k, n) = (256u32, 128u32);
let (input, reference_input) = QuantInput::<bf16>::new(m, k, n, weight_gs, bits, method, 0)
.with_prepared_a_and_reference(
ACTIVATION_SCALE_GROUP_SIZE,
(method != QuantizationMethod::ScaleSymmetric).then_some(weight_gs),
);
- let actual = run_quant_metal(&context, &input.with_group_output(), Some(GemmEngine::Mxu));
+ let actual = run_quant_metal(&context, &input.with_group_output(), gemm_b_layout, Some(GemmEngine::Mxu));
let reference = run_quant_cpu::<bf16>(&reference_input);
assert_parity::<bf16>(
- &format!("A8W{bits} MXU m={m} weight_gs={weight_gs} method={method:?}"),
+ &format!(
+ "A8W{bits} MXU m={m} k={k} n={n} weight_gs={weight_gs} method={method:?} gemm_b_layout={gemm_b_layout:?}"
+ ),
&reference,
&actual,
0.08,
@@ -558,7 +598,8 @@ fn a8w_independent_activation_group_parity_bf16(#[case] m: u32) {
(method != QuantizationMethod::ScaleSymmetric)
.then_some(activation_group_size.min(weight_group_size)),
);
- let actual = run_quant_metal(&context, &input.with_group_output(), Some(GemmEngine::Mxu));
+ let actual =
+ run_quant_metal(&context, &input.with_group_output(), GemmBLayout::RowMajor, Some(GemmEngine::Mxu));
let reference = run_quant_cpu::<bf16>(&reference_input);
assert_parity::<bf16>(
&format!("A8W{bits} act{activation_group_size} weight{weight_group_size} m={m} method={method:?}"),
@@ -584,7 +625,7 @@ fn a8w4_zero_point_tail_parity(#[case] m: u32) {
let input = QuantInput::<bf16>::new(m, 256, 72, 32, 4, QuantizationMethod::ScaleZeroPoint, 0)
.with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE, Some(32));
let reference = run_quant_cpu::<bf16>(&input);
- let actual = run_quant_metal(&context, &input.with_group_output(), Some(GemmEngine::Mxu));
+ let actual = run_quant_metal(&context, &input.with_group_output(), GemmBLayout::RowMajor, Some(GemmEngine::Mxu));
assert_parity::<bf16>(&format!("A8W4 ZP N-tail m={m}"), &reference, &actual, 0.08, 0.8);
}
@@ -605,7 +646,7 @@ fn signed_weights_full_precision_activations_parity_bf16(
let reference = run_quant_cpu::<bf16>(&reference_input);
for (label, dispatch) in [("gemv", None), ("simdgroup", Some(GemmEngine::Simdgroup))] {
- let actual = run_quant_metal::<bf16>(&context, &input, dispatch);
+ let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::RowMajor, dispatch);
assert_parity::<bf16>(
&format!("signed weights FP-A {label} bits={bits} method={method:?}"),
&reference,
@@ -766,3 +807,140 @@ fn parity_widened_f32_output(#[case] m: u32) {
let actual = run_widened_f32::<Metal>(&context, &input);
assert_parity::<f32>(&format!("widened f32 m={m}"), &reference, &actual, 0.05, 0.4);
}
+
+#[rstest]
+#[test_attr(uzu_test)]
+#[case::w4_zp_gs64(1024u32, 1024u32, 64u32, 4u32, ScaleZeroPoint)]
+#[case::w4_zp_gs32_wide(1024u32, 3584u32, 32u32, 4u32, ScaleZeroPoint)]
+#[case::w4_zp_gs64_deep(3584u32, 1024u32, 64u32, 4u32, ScaleZeroPoint)]
+#[case::w4_bias_gs64(2048u32, 2048u32, 64u32, 4u32, ScaleBias)]
+#[case::w8_sym_gs64(1024u32, 2048u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]
+#[case::w8_sym_gs64_blocks(2048u32, 4096u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]
+#[case::w8_sym_gs32_blocks(2048u32, 4096u32, 32u32, 8u32, QuantizationMethod::ScaleSymmetric)]
+fn gemv_reads_mxu_tiled_layout_bf16(
+ #[case] k: u32,
+ #[case] n: u32,
+ #[case] group_size: u32,
+ #[case] bits: u32,
+ #[case] method: QuantizationMethod,
+ #[values(false, true)] output_hadamard: bool,
+) {
+ let context = MetalContext::new().expect("Metal context");
+ if !context.supports_mxu {
+ return;
+ }
+ let input = QuantInput::<bf16>::new(1, k, n, group_size, bits, method, 0).with_group_output();
+ let input = if bits == 8 {
+ input.with_signed_weight_codes()
+ } else {
+ input
+ };
+ let factors: Vec<i32> = (0..n)
+ .map(|row| {
+ if row.is_multiple_of(3) {
+ -1
+ } else {
+ 1
+ }
+ })
+ .collect();
+
+ let cpu_context = <Cpu as Backend>::Context::new().expect("CPU context");
+ let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &input);
+ let cpu_factors = crate::tests::helpers::create_buffer_with_data::<Cpu, i32>(&cpu_context, &factors);
+ let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(
+ &cpu_context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
+ .expect("CPU matmul kernel");
+ let mut cpu_command_buffer = cpu_context.create_command_buffer(None, None).expect("CPU command buffer");
+ let mut cpu_arguments = quant_arguments(&mut cpu_buffers, &input);
+ cpu_arguments.output.ops = MatmulDOps {
+ rht_factors: output_hadamard.then_some(&cpu_factors),
+ ..MatmulDOps::none()
+ };
+ cpu_matmul.encode(cpu_arguments, &mut cpu_command_buffer).expect("CPU matmul");
+ cpu_command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let reference = buffer_to_vec::<Cpu, bf16>(&cpu_buffers.y);
+
+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
+ let metal_factors = crate::tests::helpers::create_buffer_with_data::<Metal, i32>(&context, &factors);
+ let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
+ &context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
+ .expect("Metal matmul kernel");
+ tile_weights(&matmul, &mut buffers, &input, &context);
+ let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");
+ let mut arguments = quant_arguments(&mut buffers, &input);
+ arguments.output.ops = MatmulDOps {
+ rht_factors: output_hadamard.then_some(&metal_factors),
+ ..MatmulDOps::none()
+ };
+ matmul.encode(arguments, &mut command_buffer).expect("Metal matmul");
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let actual = buffer_to_vec::<Metal, bf16>(&buffers.y);
+
+ assert_parity::<bf16>(
+ &format!("MxuTiled GEMV k={k} n={n} gs={group_size} bits={bits} method={method:?} hadamard={output_hadamard}"),
+ &reference,
+ &actual,
+ 0.05,
+ 0.5,
+ );
+}
+
+#[rstest]
+#[test_attr(uzu_test)]
+#[case::w4_zp_gs32(4u32, 32u32, ScaleZeroPoint)]
+#[case::w8_sym_gs64(8u32, 64u32, QuantizationMethod::ScaleSymmetric)]
+fn gathered_gemv_reads_mxu_tiled_layout_bf16(
+ #[case] bits: u32,
+ #[case] group_size: u32,
+ #[case] method: QuantizationMethod,
+) {
+ let context = MetalContext::new().expect("Metal context");
+ if !context.supports_mxu {
+ return;
+ }
+ let (k, vocab, gathered_rows) = (256u32, 256u32, 10u32);
+ let input = QuantInput::<bf16>::new(1, k, vocab, group_size, bits, method, 0).with_group_output();
+ let input = if bits == 8 {
+ input.with_signed_weight_codes()
+ } else {
+ input
+ };
+ let ids: Vec<u32> = (0..gathered_rows).map(|row| (row * 37 + 11) % vocab).collect();
+ let dense = run_quant_cpu::<bf16>(&input);
+ let expected: Vec<bf16> = ids.iter().map(|&id| dense[id as usize]).collect();
+
+ let ids_buffer = crate::tests::helpers::create_buffer_with_data::<Metal, u32>(&context, &ids);
+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
+ let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
+ &context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
+ .expect("Metal matmul kernel");
+ tile_weights(&matmul, &mut buffers, &input, &context);
+ let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");
+ let mut arguments = quant_arguments(&mut buffers, &input);
+ arguments.gather_indices = Some(&ids_buffer);
+ arguments.n = gathered_rows;
+ matmul.encode(arguments, &mut command_buffer).expect("gathered GEMV");
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let actual = buffer_to_vec::<Metal, bf16>(&buffers.y)[..gathered_rows as usize].to_vec();
+
+ assert_parity::<bf16>(
+ &format!("gathered MxuTiled GEMV bits={bits} gs={group_size} method={method:?}"),
+ &expected,
+ &actual,
+ 0.05,
+ 0.5,
+ );
+}
diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rs
index 083b34269..eb9e8c9e4 100644
--- a/crates/uzu-engine/unit/common/matmul/quant.rs
+++ b/crates/uzu-engine/unit/common/matmul/quant.rs
@@ -13,7 +13,7 @@ use crate::{
ActivationQuantization, ActivationTransform, Kernels,
matmul::{
Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput,
- QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection,
+ QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection, WeightsLayout,
},
},
},
@@ -293,6 +293,7 @@ pub struct QuantBuffers<B: Backend, T: DataTypeRepr + Float> {
pub prepared_a_scales: Option<B::GlobalBuffer>,
pub prepared_a_group_sums: Option<B::GlobalBuffer>,
pub y: B::GlobalBuffer,
+ pub weights_layout: WeightsLayout<B>,
_t: std::marker::PhantomData<T>,
}
@@ -332,6 +333,7 @@ impl<B: Backend, T: DataTypeRepr + Float> QuantBuffers<B, T> {
.filter(|prepared| !prepared.group_sums.is_empty())
.map(|prepared| create_buffer_with_data::<B, i32>(context, &prepared.group_sums)),
y: create_buffer::<B, T>(context, (input.m as usize) * (input.n as usize)),
+ weights_layout: Default::default(),
_t: std::marker::PhantomData,
};
if params_layout == QuantParamsLayout::GroupOutput {
@@ -344,7 +346,15 @@ impl<B: Backend, T: DataTypeRepr + Float> QuantBuffers<B, T> {
&'a self,
input: &QuantInput<T>,
) -> MatmulB<&'a B::GlobalBuffer> {
- quant_b_variant(&self.w, &self.scales, self.zp.as_ref(), self.bias.as_ref(), input.params_layout, input)
+ quant_b_variant(
+ &self.w,
+ &self.scales,
+ self.zp.as_ref(),
+ self.bias.as_ref(),
+ input.params_layout,
+ self.weights_layout,
+ input,
+ )
}
fn transpose_quant_params(
@@ -384,6 +394,7 @@ fn quant_b_variant<TB: BufferRef, T: DataTypeRepr + Float>(
zero_points: Option<TB>,
biases: Option<TB>,
params_layout: QuantParamsLayout,
+ weights_layout: WeightsLayout<TB::Backend>,
input: &QuantInput<T>,
) -> MatmulB<TB> {
let params = QuantParams::new(params_layout, input.n, input.k.div_ceil(input.group_size));
@@ -400,6 +411,7 @@ fn quant_b_variant<TB: BufferRef, T: DataTypeRepr + Float>(
mode: input.mode,
group_size: input.group_size,
signed_codes: input.signed_codes,
+ weights_layout,
})
}
@@ -417,9 +429,10 @@ pub fn quant_arguments<'a, B: Backend, T: DataTypeRepr + Float>(
prepared_a_scales,
prepared_a_group_sums,
y,
+ weights_layout,
..
} = buffers;
- let b = quant_b_variant(&*w, &*scales, zp.as_ref(), bias.as_ref(), input.params_layout, input);
+ let b = quant_b_variant(&*w, &*scales, zp.as_ref(), bias.as_ref(), input.params_layout, *weights_layout, input);
let a = match &input.prepared_a {
Some(prepared) => MatmulA::Int8Symmetric {
values: prepared_a.as_ref().expect("prepared activation buffer"),