A high-performance inference engine for AI models
Int8 mxu decode gemm (#951)
eugene86334b4c0558
diff --git a/Cargo.lock b/Cargo.lockindex bba00176d..0e52f3454 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5557,9 +5557,9 @@ dependencies = [[[package]]name = "rustls"-version = "0.23.45"+version = "0.23.46"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"+checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf"dependencies = ["aws-lc-rs","once_cell",@@ -6444,7 +6444,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"dependencies = ["fastrand",- "getrandom 0.4.3",+ "getrandom 0.3.4","once_cell","rustix","windows-sys 0.61.2",diff --git a/apps/mirai-gui/desktop/Cargo.lock b/apps/mirai-gui/desktop/Cargo.lockindex c423a3ab2..a61abae00 100644--- a/apps/mirai-gui/desktop/Cargo.lock+++ b/apps/mirai-gui/desktop/Cargo.lock@@ -5003,9 +5003,9 @@ dependencies = [[[package]]name = "rustls"-version = "0.23.45"+version = "0.23.46"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"+checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf"dependencies = ["aws-lc-rs","once_cell",@@ -6242,7 +6242,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"dependencies = ["fastrand",- "getrandom 0.4.3",+ "getrandom 0.3.4","once_cell","rustix","windows-sys 0.61.2",diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rsindex f0ba1dd1e..a31674ea0 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs@@ -17,6 +17,7 @@ use crate::{pub trait MatmulKernel: Sized + Send + Sync {type Backend: Backend<Kernels: Kernels<MatmulKernel = Self>>;type Error: Error + 'static;+ type WeightsLayout: Copy + Default + PartialEq + Send + Sync;fn new(context: &<Self::Backend as Backend>::Context,@@ -49,8 +50,18 @@ pub trait MatmulKernel: Sized + Send + Sync {fn select_activation_format(&self,_bf16_shape: &MatmulShape,+ _weights_layout: Self::WeightsLayout,_context: &<Self::Backend as Backend>::Context,) -> ActivationFormat {ActivationFormat::Bf16}++ fn optimize_weights_layout(+ &self,+ _a8_shape: &MatmulShape,+ _weights: &mut <Self::Backend as Backend>::GlobalBuffer,+ _context: &<Self::Backend as Backend>::Context,+ ) -> Result<Self::WeightsLayout, <Self::Backend as Backend>::Error> {+ Ok(Self::WeightsLayout::default())+ }}diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rsindex ca115c9b1..f0519ce3e 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs@@ -1,4 +1,4 @@-use super::{QuantParams, QuantParamsLayout};+use super::{QuantParams, QuantParamsLayout, WeightsLayout};use crate::{backends::common::{BufferRef,@@ -35,6 +35,7 @@ pub struct QuantizedB<TB: BufferRef> {pub mode: QuantizationMode,pub group_size: u32,pub signed_codes: bool,+ pub weights_layout: WeightsLayout<TB::Backend>,}#[derive(Clone, Copy)]@@ -127,6 +128,10 @@ impl<TB: BufferRef> MatmulB<TB> {self.quantized().is_some_and(|quantized| quantized.signed_codes)}+ pub fn weights_layout(&self) -> WeightsLayout<TB::Backend> {+ self.quantized().map_or_else(Default::default, |quantized| quantized.weights_layout)+ }+pub fn quant_params_layout(&self) -> Option<QuantParamsLayout> {self.quantized().map(|quantized| quantized.params.layout())}diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rsindex 314c0ef72..7d2640125 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs@@ -16,7 +16,12 @@ pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection};pub use output::{MatmulOutput, MatmulOutputError};pub use routing::{ActivationFormat, MatmulShape};-use crate::backends::common::gpu_types::{matmul::QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT, quantization::QuantizationMode};+use crate::backends::common::{+ Backend, Kernels,+ gpu_types::{matmul::QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT, quantization::QuantizationMode},+};++pub type WeightsLayout<B> = <<<B as Backend>::Kernels as Kernels>::MatmulKernel as MatmulKernel>::WeightsLayout;#[derive(Clone, Copy, Debug, PartialEq)]pub enum QuantParamsLayout {diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rsindex 8a91010d9..f5cf904d2 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs@@ -34,6 +34,7 @@ pub struct MatmulCpuKernel {impl MatmulKernel for MatmulCpuKernel {type Backend = Cpu;type Error = MatmulCpuError;+ type WeightsLayout = ();fn new(context: &CpuContext,diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rsindex 4a2a7e83a..c42e4d23c 100644--- a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs+++ b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs@@ -8,6 +8,14 @@ pub enum GemmAPrologueKind {Int8Symmetric,}+#[repr(C)]+#[derive(Debug, Display, Default, Clone, Copy, PartialEq, Eq, Hash)]+pub enum GemmBLayout {+ #[default]+ RowMajor,+ MxuTiled,+}+#[repr(C)]#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]#[allow(non_camel_case_types)]diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metalindex bef34b39f..a8741f1fc 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal@@ -15,7 +15,7 @@ using namespace uzu::trie;#define ROW_TILE 16u#define COL_TILE 32u-#define NUM_SIMDGROUPS 2u+#define NUM_SIMDGROUPS 4u#define INVALID_ROW 0xffffffffu// kh0 = k @ h0[batch]^T for one row-tile, walking head_v_dim in COL_TILEdiff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.hindex 8f8b35954..b555f029c 100644--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h@@ -31,6 +31,11 @@ enum class GemmAPrologueKind : uint32_t {Int8Symmetric = 1,};+enum class GemmBLayout : uint32_t {+ RowMajor = 0,+ MxuTiled = 1,+};+enum class GemmTiling : uint32_t {Tile8x32x32_Simdgroups1x1 = 0,Tile64x32x32_Simdgroups2x2 = 1,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.hindex e2ba61926..681597107 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h@@ -88,6 +88,7 @@ struct RightStorage {const device typename Right::ScaleElement* biases;const device uint8_t* zero_points;bool signed_codes;+ GemmBLayout gemm_b_layout;METAL_FUNC const device typename Right::ScaleElement* bias() const thread {static_assert(Right::SCHEME == GemmBPrologueKind::ScaleBiasDequant, "bias is only valid for ScaleBiasDequant");@@ -123,10 +124,11 @@ METAL_FUNC RightStorage<Right> pack_right(const device Element* scales,const device Element* biases,const device uint8_t* zero_points,- const bool signed_codes+ const bool signed_codes,+ const GemmBLayout gemm_b_layout) {if constexpr (!Right::QUANTIZED) {- return {dense, nullptr, nullptr, nullptr, nullptr, false};+ return {dense, nullptr, nullptr, nullptr, nullptr, false, GemmBLayout::RowMajor};} else {return {nullptr,@@ -134,7 +136,8 @@ METAL_FUNC RightStorage<Right> pack_right(scales,Right::SCHEME == GemmBPrologueKind::ScaleBiasDequant ? biases : nullptr,Right::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant ? zero_points : nullptr,- signed_codes+ signed_codes,+ gemm_b_layout};}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.hindex a9f0ff38d..c9c163c11 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h@@ -90,8 +90,24 @@ struct Int8Cursor {int row_stride;short simdgroup_limit;ushort simd_lane_id;+ GemmBLayout gemm_b_layout;+ uint chunk_stride;METAL_FUNC Fragment load(const uint) const thread {+ if (gemm_b_layout == GemmBLayout::MxuTiled) {+ Fragment tile;+ const uint4 low = *reinterpret_cast<const device uint4*>(address);+ const uint4 high = *reinterpret_cast<const device uint4*>(address + sizeof(uint4));+ const uint words[8] = {low.x, low.y, low.z, low.w, high.x, high.y, high.z, high.w};+ for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short) {+ METAL_PRAGMA_UNROLL+ for (ushort fragment_column = 0; fragment_column < Fragment::COL_FRAGMENTS; ++fragment_column) {+ reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, fragment_column))[row_slot] =+ words[(fragment_row * Ops::THREAD_ELEMENT_ROWS + row_slot) * Fragment::COL_FRAGMENTS + fragment_column];+ }+ });+ return tile;+ }return load_int8_tile<Fragment, ALIGNED, CODES_GROUPED_BY_NIBBLE>(address,row_stride,@@ -100,7 +116,7 @@ struct Int8Cursor {);}- METAL_FUNC void advance() thread { address += BLOCK_K; }+ METAL_FUNC void advance() thread { address += chunk_stride; }METAL_FUNC void begin_k_group(const uint) thread {}};@@ -129,8 +145,14 @@ struct W4Cursor {bool signed_codes;const device uint8_t* zero_points;uint zero_point_group_stride_bytes;+ GemmBLayout gemm_b_layout;+ uint chunk_stride;METAL_FUNC PackedChunk fetch(const uint chunk_index) const thread {+ if (gemm_b_layout == GemmBLayout::MxuTiled) {+ const uint4 words = *reinterpret_cast<const device uint4*>(current + chunk_index * chunk_stride);+ return {{words.x, words.y, words.z, words.w}};+ }// Padding decodes to zero after the optional sign flip and code-origin adjustment.const uint padding_word = (Operand::CODE_ORIGIN == 0) == signed_codes ? W4_SIGN_MASK : 0u;const device uint8_t* base =@@ -177,7 +199,7 @@ struct W4Cursor {METAL_FUNC Fragment load(const uint) const thread { return decode(fetch(0)); }- METAL_FUNC void advance() thread { current += CHUNK_BYTES; }+ METAL_FUNC void advance() thread { current += chunk_stride; }METAL_FUNC void begin_k_group(const uint k_offset) thread {if constexpr (HAS_ZERO_POINTS) {@@ -201,7 +223,9 @@ static METAL_FUNC auto make_left_cursor(origin,int(params->leading_dimension_a),tile.simdgroup_limit_m,- ushort(thread_context.simd_lane_id)+ ushort(thread_context.simd_lane_id),+ GemmBLayout::RowMajor,+ uint(Int8Cursor<Fragment, ALIGNED, LeftOperand::GROUPED_BY_NIBBLE>::BLOCK_K)};}@@ -219,6 +243,17 @@ static METAL_FUNC auto make_right_cursor(const device uint8_t* current =source.codes + size_t(tile.block_col + tile.tile_col_offset) * size_t(row_stride_bytes) +size_t(tile.k_offset) * size_t(get_bytes_per_pack<Operand::BITS>()) / size_t(get_pack_factor<Operand::BITS>());+ static_assert(Core::SIMDGROUP_BLOCK_N == 2 * Ops::FRAGMENT_ROWS, "the MxuTiled layout holds 32-row blocks");+ const uint mxu_tiled_lane_bytes = Operand::BITS == 4 ? 16 : 32;+ const uint mxu_tiled_chunk_bytes = METAL_SIMD_SIZE * mxu_tiled_lane_bytes;+ if (source.gemm_b_layout == GemmBLayout::MxuTiled) {+ const uint chunk_count = uint(params->K) / uint(Core::SIMDGROUP_BLOCK_K);+ const uint row_block = uint(tile.block_col + tile.tile_col_offset) / uint(Core::SIMDGROUP_BLOCK_N);+ current =+ source.codes ++ (size_t(row_block) * chunk_count + tile.k_offset / uint(Core::SIMDGROUP_BLOCK_K)) * mxu_tiled_chunk_bytes ++ thread_context.simd_lane_id * mxu_tiled_lane_bytes;+ }if constexpr (Operand::BITS == 4) {static_assert(Core::TILES_K == get_pack_factor<Operand::BITS>(), "W4 requires two K fragments");@@ -236,7 +271,9 @@ static METAL_FUNC auto make_right_cursor(Ops::get_position(thread_context.simd_lane_id),source.signed_codes,zero_points,- zero_point_group_stride_bytes+ zero_point_group_stride_bytes,+ source.gemm_b_layout,+ source.gemm_b_layout == GemmBLayout::MxuTiled ? mxu_tiled_chunk_bytes : Cursor::CHUNK_BYTES};} else {static_assert(Operand::BITS == 8, "integer tile cursors support 4-bit and 8-bit codes");@@ -245,7 +282,10 @@ static METAL_FUNC auto make_right_cursor(origin_int8,row_stride_bytes,tile.simdgroup_limit_n,- ushort(thread_context.simd_lane_id)+ ushort(thread_context.simd_lane_id),+ source.gemm_b_layout,+ source.gemm_b_layout == GemmBLayout::MxuTiled ? mxu_tiled_chunk_bytes+ : uint(Int8Cursor<Fragment, ALIGNED, false>::BLOCK_K)};}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.hindex 044de6e91..acd2b8e62 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h@@ -186,18 +186,6 @@ struct IntegerSchedule {ALIGNED_N;}- template <typename RightCodes, int K_GROUP_COUNT, int CHUNKS_PER_K_GROUP>- static METAL_FUNC void fetch_k_groups(- const thread RightCodes& right_codes,- thread typename RightCodes::PackedChunk (&packed_right_chunks)[K_GROUP_COUNT][CHUNKS_PER_K_GROUP]- ) {- for_each_static_index<K_GROUP_COUNT>([&](const ushort k_group) {- for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {- packed_right_chunks[k_group][chunk] = right_codes.fetch(uint(k_group * CHUNKS_PER_K_GROUP + chunk));- });- });- }-template <typename Core>static METAL_FUNC GroupProducts<Core> multiply_device_group(const typename Core::LeftStorage left,@@ -242,32 +230,32 @@ struct IntegerSchedule {return product;}- template <typename Core, typename LeftCodes, typename RightCodes, int CHUNKS_PER_K_GROUP>+ template <typename Core, typename LeftCursor, typename RightCursor, int CHUNKS_PER_K_GROUP>static METAL_FUNC GroupProducts<Core> multiply_k_group(- thread LeftCodes& left_codes,- thread RightCodes& right_codes,- const thread typename RightCodes::PackedChunk (&packed_right_chunks)[CHUNKS_PER_K_GROUP]+ thread LeftCursor& left_cursor,+ thread RightCursor& right_cursor,+ const thread typename RightCursor::PackedChunk (&packed_right_chunks)[CHUNKS_PER_K_GROUP]) {GroupProducts<Core> group_product;group_product.clear();if constexpr (prefetches_int4_chunks<Core>()) {for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {- auto left_tile = left_codes.load(uint(chunk));- auto right_tile = right_codes.decode(packed_right_chunks[chunk]);+ auto left_tile = left_cursor.load(uint(chunk));+ auto right_tile = right_cursor.decode(packed_right_chunks[chunk]);uzu::matmul::fragment_mma(group_product, left_tile, right_tile);- left_codes.advance();- right_codes.advance();+ left_cursor.advance();+ right_cursor.advance();});} else {- decltype(right_codes.load(0u)) right_tiles[CHUNKS_PER_K_GROUP];+ decltype(right_cursor.load(0u)) right_tiles[CHUNKS_PER_K_GROUP];for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {- right_tiles[chunk] = right_codes.load(uint(chunk));- right_codes.advance();+ right_tiles[chunk] = right_cursor.load(uint(chunk));+ right_cursor.advance();});for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {- auto left_tile = left_codes.load(uint(chunk));+ auto left_tile = left_cursor.load(uint(chunk));uzu::matmul::fragment_mma(group_product, left_tile, right_tiles[chunk]);- left_codes.advance();+ left_cursor.advance();});}return group_product;@@ -331,9 +319,9 @@ struct IntegerSchedule {}}- auto left_codes =+ auto left_cursor =quantized::make_left_cursor<Core, LeftOperand, ALIGNED_M>(left_storage, params, tile, thread_context);- auto right_codes =+ auto right_cursor =quantized::make_right_cursor<Core, RightOperand, ALIGNED_N>(right_storage, params, tile, thread_context);const short2 position = Core::FragmentOps::get_position(thread_context.simd_lane_id);@@ -356,17 +344,18 @@ struct IntegerSchedule {const int right_group_count = int(params->aligned_inner_iterations);constexpr int CHUNKS_PER_K_GROUP = chunks_per_k_group<Core>();constexpr bool PREFETCH_INT4_CHUNKS = prefetches_int4_chunks<Core>();- constexpr bool IS_ALIGNED_M_INT4_TILE16X32 =- PREFETCH_INT4_CHUNKS && ALIGNED_M && Core::THREADGROUP_BLOCK_M == 16 && Core::THREADGROUP_BLOCK_N == 32;- constexpr int K_GROUPS_PER_FETCH = IS_ALIGNED_M_INT4_TILE16X32 ? 4 : 1;- constexpr int MIN_K_GROUPS_PER_THREADGROUP_FOR_BATCHED_FETCH = 16;- using PackedRightChunk = typename decltype(right_codes)::PackedChunk;-- auto multiply_group = [&](const int right_group_index,- const thread PackedRightChunk(&packed_right_chunks)[CHUNKS_PER_K_GROUP]) {++ METAL_PRAGMA_NO_UNROLL+ for (int right_group_index = 0; right_group_index < right_group_count; ++right_group_index) {+ typename decltype(right_cursor)::PackedChunk packed_right_chunks[CHUNKS_PER_K_GROUP];+ if constexpr (PREFETCH_INT4_CHUNKS) {+ for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {+ packed_right_chunks[chunk] = right_cursor.fetch(uint(chunk));+ });+ }const uint right_group_offset = uint(right_group_index) * RIGHT_GROUP_SIZE;- left_codes.begin_k_group(right_group_offset);- right_codes.begin_k_group(right_group_offset);+ left_cursor.begin_k_group(right_group_offset);+ right_cursor.begin_k_group(right_group_offset);auto load_metadata = [&]() {left_metadata.load(left_storage, metadata_context, tile.k_offset + right_group_offset);right_metadata.load(right_storage, metadata_context, first_right_group + uint(right_group_index));@@ -376,38 +365,24 @@ struct IntegerSchedule {}GroupProducts<Core> group_product;if constexpr (multiplies_device_groups<Core, ALIGNED_M, ALIGNED_N>()) {- group_product =- multiply_device_group<Core>(left_storage, right_storage, params, tile, tile.k_offset + right_group_offset);+ if (right_storage.gemm_b_layout == GemmBLayout::MxuTiled) {+ group_product = multiply_k_group<Core>(left_cursor, right_cursor, packed_right_chunks);+ } else {+ group_product = multiply_device_group<Core>(+ left_storage,+ right_storage,+ params,+ tile,+ tile.k_offset + right_group_offset+ );+ }} else {- group_product = multiply_k_group<Core>(left_codes, right_codes, packed_right_chunks);+ group_product = multiply_k_group<Core>(left_cursor, right_cursor, packed_right_chunks);}if constexpr (PREFETCH_INT4_CHUNKS) {load_metadata();}accumulate_group<Core, ALIGNED_M, ALIGNED_N>(accumulator, group_product, left_metadata, right_metadata);- };-- int right_group_index = 0;- if constexpr (K_GROUPS_PER_FETCH > 1) {- const int batched_fetch_right_group_limit =- right_group_count >= MIN_K_GROUPS_PER_THREADGROUP_FOR_BATCHED_FETCH ? right_group_count : 0;- METAL_PRAGMA_NO_UNROLL- for (; right_group_index + K_GROUPS_PER_FETCH <= batched_fetch_right_group_limit;- right_group_index += K_GROUPS_PER_FETCH) {- PackedRightChunk packed_right_chunks[K_GROUPS_PER_FETCH][CHUNKS_PER_K_GROUP];- fetch_k_groups(right_codes, packed_right_chunks);- for_each_static_index<K_GROUPS_PER_FETCH>([&](const ushort k_group) {- multiply_group(right_group_index + int(k_group), packed_right_chunks[k_group]);- });- }- }- METAL_PRAGMA_NO_UNROLL- for (; right_group_index < right_group_count; ++right_group_index) {- PackedRightChunk packed_right_chunks[1][CHUNKS_PER_K_GROUP];- if constexpr (PREFETCH_INT4_CHUNKS) {- fetch_k_groups(right_codes, packed_right_chunks);- }- multiply_group(right_group_index, packed_right_chunks[0]);}return accumulator;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metalindex d1d234faa..bb29d29b5 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -137,6 +137,7 @@ KERNEL(Gemm)(const GemmDTransform output_transform SPECIALIZE,const GemmAlignment alignment SPECIALIZE,const bool signed_codes SPECIALIZE,+ const GemmBLayout gemm_b_layout SPECIALIZE,threadgroup AT a_shared[GEMM_TGA_ELEMENTS],threadgroup BT b_shared[GEMM_TGB_ELEMENTS],const uint group_x GROUPS(group_count_x),@@ -162,7 +163,8 @@ KERNEL(Gemm)("kernel bindings and operand correction policy must agree");const auto left_storage = operands::pack_left<LeftOperand, AT>(a, a_int8, a_scales, a_group_sums);- const auto right_storage = operands::pack_right<RightOperand, BT>(b, scales, biases, zero_points, signed_codes);+ const auto right_storage =+ operands::pack_right<RightOperand, BT>(b, scales, biases, zero_points, signed_codes, gemm_b_layout);if constexpr (USE_MXU) {using Core = MxuMmaCore<DT, GEMM_TILING, TRANSPOSE_B, LeftOperand, RightOperand>;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metalnew file mode 100644index 000000000..547f1d416--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal@@ -0,0 +1,51 @@+#include "../../common/dsl.h"+#include "../../common/defines.h"+#include "../common/mxu_fragment/ops.h"++using namespace metal;++#define PREPARE_B_LAYOUT_THREADS 1024u++template <uint BITS, uint WORDS_PER_THREAD>+VARIANTS(BITS, 4, 8)+VARIANTS(WORDS_PER_THREAD, 32, 72, 160)+KERNEL(GemmPrepareBLayout)(+ device uint* weights,+ const constant uint& row_words,+ const constant uint& row_block_count,+ const uint row_block GROUPS(row_block_count),+ const uint thread_index THREADS(PREPARE_B_LAYOUT_THREADS)+) {+ using Ops = uzu::matmul::MxuFragmentOps<>;+ constexpr uint CHUNK_BYTES = BITS == 4 ? 16 : 32;+ constexpr uint COL_FRAGMENTS = BITS == 4 ? 1 : 2;+ constexpr uint LANE_WORDS = 2 * Ops::THREAD_ELEMENT_ROWS * COL_FRAGMENTS;+ const uint block_words = 2 * Ops::FRAGMENT_ROWS * row_words;+ device uint* block = weights + size_t(row_block) * block_words;++ uint words[WORDS_PER_THREAD];+ METAL_PRAGMA_UNROLL+ for (uint slot = 0; slot < WORDS_PER_THREAD; ++slot) {+ const uint index = slot * PREPARE_B_LAYOUT_THREADS + thread_index;+ if (index < block_words) {+ const uint lane = (index / LANE_WORDS) % METAL_SIMD_SIZE;+ const uint word = index % LANE_WORDS;+ const uint fragment_row = word / (Ops::THREAD_ELEMENT_ROWS * COL_FRAGMENTS);+ const uint row_slot = word / COL_FRAGMENTS % Ops::THREAD_ELEMENT_ROWS;+ const uint fragment_column = word % COL_FRAGMENTS;+ const short2 position = Ops::get_position(ushort(lane));+ const uint row = uint(position.y) + fragment_row * Ops::FRAGMENT_ROWS + row_slot * Ops::THREAD_ELEMENT_ROW_STRIDE;+ const uint byte = index / (METAL_SIMD_SIZE * LANE_WORDS) * CHUNK_BYTES + uint(position.x) ++ fragment_column * Ops::FRAGMENT_COLS;+ words[slot] = block[row * row_words + byte / 4];+ }+ }+ threadgroup_barrier(mem_flags::mem_device);+ METAL_PRAGMA_UNROLL+ for (uint slot = 0; slot < WORDS_PER_THREAD; ++slot) {+ const uint index = slot * PREPARE_B_LAYOUT_THREADS + thread_index;+ if (index < block_words) {+ block[index] = words[slot];+ }+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rsindex c2eb07de3..4ced1b076 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -3,7 +3,8 @@ use std::collections::{HashMap, hash_map::Entry};use crate::{backends::{common::{- Backend, BufferMut, BufferRef, CommandBufferEncoding,+ Backend, BufferMut, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,+ Context,gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},kernel::matmul::{Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulOutput, MatmulShape, QuantParamsLayout,@@ -16,11 +17,11 @@ use crate::{context::MetalContext,error::MetalError,gpu_types::{- gemm::{GemmAPrologueKind, GemmAlignment},+ gemm::{GemmAPrologueKind, GemmAlignment, GemmBLayout},matmul::GemmParams,},kernel::{- GemmMetalKernel, GemmSplitKReduceMetalKernel,+ GemmMetalKernel, GemmPrepareBLayoutMetalKernel, GemmSplitKReduceMetalKernel,matmul::{MatmulMetalError, MatmulOutputWork,gemm::{@@ -91,12 +92,41 @@ impl GemmKernel {specialization.output_transform,specialization.alignment,specialization.signed_codes,+ specialization.gemm_b_layout,)?;Ok(entry.insert(kernel))},}}+ pub fn mxu_tiled_words_per_thread(+ k: u32,+ bits: u32,+ ) -> Option<u32> {+ let words_per_thread = (k * bits / 32).div_ceil(32);+ [32, 72, 160].into_iter().find(|&variant| variant >= words_per_thread)+ }++ pub fn prepare_gemm_b_layout(+ weights: &mut <Metal as Backend>::GlobalBuffer,+ rows: u32,+ k: u32,+ bits: u32,+ context: &MetalContext,+ ) -> Result<(), MetalError> {+ let row_words = k * bits / 32;+ let variant = Self::mxu_tiled_words_per_thread(k, bits).expect("int8 eligibility admits only tileable weights");+ let mut command_buffer = context.create_command_buffer(Some("prepare GEMM B layout"), None)?;+ GemmPrepareBLayoutMetalKernel::new(context, bits, variant)?.encode(+ weights,+ row_words,+ rows / 32,+ &mut command_buffer,+ );+ command_buffer.end_encoding().submit().wait_until_completed()?;+ Ok(())+ }+fn get_or_create_split_k_reduce(&mut self,context: &MetalContext,@@ -224,6 +254,13 @@ impl GemmKernel {};let use_mxu = plan.engine == GemmEngine::Mxu;+ let gemm_b_layout = quantized.as_ref().map_or(GemmBLayout::RowMajor, |quantized| quantized.weights_layout);+ if gemm_b_layout == GemmBLayout::MxuTiled && (!use_mxu || shape.a_full_precision) {+ return Err(MatmulMetalError::IncompatibleA {+ path: "Gemm",+ reason: "MxuTiled GEMM B layout requires int8 activations on the MXU",+ });+ }let is_quant = quantized.is_some();if !is_quant && !matches!(&a, MatmulA::FullPrecision { .. }) {@@ -299,6 +336,7 @@ impl GemmKernel {scale_strides,zero_point_strides,shape,+ gemm_b_layout,plan,output_transform,output_bias,@@ -346,6 +384,7 @@ impl GemmKernel {let specialization = GemmSpecialization::from_plan(plan,shape,+ gemm_b_layout,self.weights_data_type,output_transform,alignment,@@ -387,6 +426,7 @@ impl GemmKernel {scale_strides: QuantParamsStrides,zero_point_strides: QuantParamsStrides,shape: MatmulShape,+ gemm_b_layout: GemmBLayout,plan: GemmPlan,output_transform: GemmDTransform,output_bias: Option<impl BufferRef<Backend = Metal>>,@@ -427,6 +467,7 @@ impl GemmKernel {let part_spec = GemmSpecialization::from_plan(plan,shape,+ gemm_b_layout,self.weights_data_type,GemmDTransform::empty(),alignment,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rsindex c80f269a3..84bd19da1 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs@@ -11,7 +11,6 @@ use crate::backends::metal::gpu_types::gemm::GemmTiling;const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;const MXU_SKINNY_M_MAX: u32 = 16;-const MXU_K_SPLIT_FULL_BLOCK_N_MAX: u32 = 16384;const SKINNY_SQUARE_K_MAX: u32 = 2560;const READOUT_N_TO_K_RATIO: u32 = 32;const WIDE_N_DEEP_K_MIN: u32 = 4096;@@ -63,9 +62,7 @@ pub(super) fn mxu_int8_gemm_tile() -> GemmTiling {match mxu_mn_tile(true, m, n) {GemmTiling::Tile128x128x256_Simdgroups4x4 => MXU_DEFAULT_TILE,- GemmTiling::Tile16x32x256_Simdgroups1x1 if m < MXU_SKINNY_M_MAX || n <= MXU_K_SPLIT_FULL_BLOCK_N_MAX => {- GemmTiling::Tile16x32x256_Simdgroups1x1x4- },+ GemmTiling::Tile16x32x256_Simdgroups1x1 => GemmTiling::Tile16x32x256_Simdgroups1x1x4,tiling => tiling,}}@@ -150,6 +147,7 @@ pub(super) fn split_k_target_tiles((true, GemmTiling::Tile32x64x256_Simdgroups2x2, Some(4)) => SPLIT_K_TARGET_TILES_A8_TILE32_W4,(true, GemmTiling::Tile32x64x256_Simdgroups2x2, _) => SPLIT_K_TARGET_TILES_A8_TILE32_W8,(true, GemmTiling::Tile16x32x256_Simdgroups1x1, _) => SPLIT_K_TARGET_TILES_A8_TILE16X32,+ (true, GemmTiling::Tile16x32x256_Simdgroups1x1x4, Some(4)) => SPLIT_K_TARGET_TILES_A8,(true, GemmTiling::Tile16x32x256_Simdgroups1x1x4, _) => SPLIT_K_TARGET_TILES_A8_TILE16X32_K_SPLIT,(true, _, _) => SPLIT_K_TARGET_TILES_A8,(false, _, _) => SPLIT_K_TARGET_TILES_FP,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rsindex 25ea3639f..1b1c4b7f6 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -5,7 +5,7 @@ use crate::{gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},},- metal::gpu_types::gemm::{GemmAPrologueKind, GemmAlignment, GemmTiling},+ metal::gpu_types::gemm::{GemmAPrologueKind, GemmAlignment, GemmBLayout, GemmTiling},},data_type::DataType,};@@ -23,6 +23,7 @@ pub(super) struct GemmSpecialization {pub(super) bits_per_b: Option<u32>,pub(super) b_group_size: Option<u32>,pub(super) signed_codes: bool,+ pub(super) gemm_b_layout: GemmBLayout,pub(super) a_group_size: Option<u32>,}@@ -30,6 +31,7 @@ impl GemmSpecialization {pub(super) fn from_plan(plan: GemmPlan,shape: MatmulShape,+ gemm_b_layout: GemmBLayout,weights_data_type: DataType,output_transform: GemmDTransform,alignment: GemmAlignment,@@ -48,6 +50,7 @@ impl GemmSpecialization {bits_per_b: shape.b_bits,b_group_size: shape.b_group_size,signed_codes: shape.signed_codes,+ gemm_b_layout,a_group_size,};specialization.validate()?;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/arguments.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/arguments.hindex fef97899c..8605ef66b 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/arguments.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/arguments.h@@ -36,6 +36,15 @@ struct GemvParams {GemmDTransform output_transform;bool gathered;bool signed_codes;+ GemmBLayout gemm_b_layout;++ METAL_FUNC uint output_row(uint row) const thread {+ if (gemm_b_layout != GemmBLayout::MxuTiled || gathered) {+ return row;+ }+ const uint block_row = row % 32;+ return row - block_row + block_row / 16 * 4 + block_row / 4 % 4 + block_row % 4 * 8;+ }};} // namespace gemmdiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.hindex 042dcced7..314bb65c5 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h@@ -75,18 +75,18 @@ private:if (!(FULL_TILE || input_row < params.batch_size)) {return;}- device DT* output = ops.d + input_row * params.out_vec_size + tile.row0;+ device DT* output = ops.d + input_row * params.out_vec_size;Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {constexpr uint R = decltype(output_index)::value;- const uint global_row = tile.row0 + R;+ const uint global_row = params.output_row(tile.row0 + R);if (!OutputTile<Tile, FULL_TILE>::row_in_range(global_row, params.out_vec_size)) {return;}if constexpr (STAGE_HADAMARD) {- shared_results[I * Tile::OUTPUT_ROWS + tile.local_row + R] =+ shared_results[I * Tile::OUTPUT_ROWS + params.output_row(tile.local_row + R)] =transform<false>(result[I][R], global_row, ops, params);} else {- output[R] = static_cast<DT>(transform<true>(result[I][R], global_row, ops, params));+ output[global_row] = static_cast<DT>(transform<true>(result[I][R], global_row, ops, params));}});});diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.hindex 220959e8e..9bdeefe12 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h@@ -35,7 +35,7 @@ struct QuantBSource {uint weight_row_indices[Tile::ROWS_PER_LANE];Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {constexpr uint R = decltype(output_index)::value;- const uint output_row = tile.row0 + R;+ const uint output_row = params.output_row(tile.row0 + R);weight_row_indices[R] =params.gathered ? ops.gather_indices[tile.input_row * params.out_vec_size + output_row] : output_row;});@@ -50,7 +50,7 @@ struct QuantBSource {Metadata metadata;metadata.load(position.group, weight_row_indices, ops, params);for (position.slice = 0; position.slice < Slice::SLICES_PER_LANE; position.slice++) {- current.load_weights(position, weights, weight_row_indices, row_stride, group_offset);+ current.load_weights(position, weights, weight_row_indices, row_stride, group_offset, params);current.accumulate(result, position, ops, params, tile, group_offset, batch_remaining, metadata);}position.group += Tile::GROUPS_PER_STEP;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.hindex fef42725d..c6f6a962c 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h@@ -44,6 +44,53 @@ private:uint4 weights[Tile::ROWS_PER_LANE][SLICE_WORDS];+ static METAL_FUNC uint mxu_tiled_offset(uint row, uint row_stride, uint byte) {+ constexpr uint CHUNK_BYTES = BITS * 4;+ const uint y = row % 8;+ const uint x = byte % CHUNK_BYTES / BITS;+ const uint lane = (x & 1) | ((y & 1) << 1) | ((y >> 1 & 1) << 2) | ((x >> 1) << 3) | ((y >> 2) << 4);+ return row / 32 * 32 * row_stride + (byte / CHUNK_BYTES * 32 + lane) * CHUNK_BYTES + row % 32 / 8 * BITS ++ byte % BITS;+ }++ METAL_FUNC void load_mxu_tiled_weights(+ const device uint8_t* weights_base,+ const thread uint (&weight_row_indices)[Tile::ROWS_PER_LANE],+ uint row_stride,+ uint offset_bytes,+ bool gathered+ ) thread {+ constexpr uint ROW_WORDS = BITS / 4;+ constexpr uint SEGMENT_WORDS = Tile::ROWS_PER_LANE * ROW_WORDS;+ constexpr uint VECTOR_WORDS = SEGMENT_WORDS < 4 ? SEGMENT_WORDS : 4;+ constexpr bool VECTOR_LOADS = 4 % Tile::ROWS_PER_LANE == 0 && VECTOR_WORDS > 1;+ METAL_PRAGMA_UNROLL+ for (uint segment = 0; segment < SLICE_BYTES / BITS; ++segment) {+ const uint byte = offset_bytes + segment * BITS;+ if (VECTOR_LOADS && !gathered) {+ const device vec<uint, VECTOR_WORDS>* source = reinterpret_cast<const device vec<uint, VECTOR_WORDS>*>(+ weights_base + mxu_tiled_offset(weight_row_indices[0], row_stride, byte)+ );+ METAL_PRAGMA_UNROLL+ for (uint word = 0; word < SEGMENT_WORDS; ++word) {+ reinterpret_cast<thread uint*>(weights[word / ROW_WORDS])[segment * ROW_WORDS + word % ROW_WORDS] =+ source[word / VECTOR_WORDS][word % VECTOR_WORDS];+ }+ } else {+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {+ constexpr uint R = decltype(output_index)::value;+ METAL_PRAGMA_UNROLL+ for (uint word = 0; word < ROW_WORDS; ++word) {+ reinterpret_cast<thread uint*>(weights[R])[segment * ROW_WORDS + word] =+ *reinterpret_cast<const device uint*>(+ weights_base + mxu_tiled_offset(weight_row_indices[R], row_stride, byte + word * 4)+ );+ }+ });+ }+ }+ }+public:static METAL_FUNC uint row_stride(const thread GemvParams& params) {return params.in_vec_size * BITS / QuantChunk<BITS>::BITS_PER_BYTE;@@ -54,10 +101,15 @@ public:const device uint8_t* weights_base,const thread uint (&weight_row_indices)[Tile::ROWS_PER_LANE],uint row_stride,- uint group_offset+ uint group_offset,+ const thread GemvParams& params) thread {const uint k = position.group * GROUP_SIZE + group_offset + position.slice * SLICE_VALUES;const uint offset_bytes = k * BITS / QuantChunk<BITS>::BITS_PER_BYTE;+ if (params.gemm_b_layout == GemmBLayout::MxuTiled) {+ load_mxu_tiled_weights(weights_base, weight_row_indices, row_stride, offset_bytes, params.gathered);+ return;+ }Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {constexpr uint R = decltype(output_index)::value;const uint row = weight_row_indices[R];@@ -103,9 +155,18 @@ public:const uint k =position.group * GROUP_SIZE + group_offset + position.slice * SLICE_VALUES + chunk * CHUNK_VALUES;const uint input_row = tile.input_row + source_row;- const device AT* input = ops.a + input_row * params.in_vec_size + k;float input_values[CHUNK_VALUES];- QuantChunk<BITS>::template load<INPUT_ALIGNED>(input, input_values, k, params.in_vec_size, input_row);+ if (BITS == 8 && params.gemm_b_layout == GemmBLayout::MxuTiled) {+ const device AT* input = ops.a + input_row * params.in_vec_size + k - k % 32 / 2;+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < CHUNK_VALUES / 2; i++) {+ input_values[i] = float(input[i]);+ input_values[CHUNK_VALUES / 2 + i] = float(input[16 + i]);+ }+ } else {+ const device AT* input = ops.a + input_row * params.in_vec_size + k;+ QuantChunk<BITS>::template load<INPUT_ALIGNED>(input, input_values, k, params.in_vec_size, input_row);+ }if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {METAL_PRAGMA_UNROLLfor (uint i = 0; i < CHUNK_VALUES; i++) {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metalindex f953ee140..282ac4bcd 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -130,6 +130,7 @@ KERNEL(Gemv)(const GemmDTransform output_transform SPECIALIZE,const bool gathered SPECIALIZE,const bool signed_codes SPECIALIZE,+ const GemmBLayout gemm_b_layout SPECIALIZE,const bool full_tile SPECIALIZE,threadgroup float shared_results[INPUT_ROW_TILE * OUTPUT_ROW_TILE * K_SPLIT],const uint input_tile_idx GROUPS(batch_size.div_ceil(INPUT_ROW_TILE)),@@ -151,7 +152,8 @@ KERNEL(Gemv)(soft_cap,output_transform,gathered,- signed_codes+ signed_codes,+ gemm_b_layout};dispatch_bool(full_tile, [&](auto full_tile_constant) {constexpr bool FullTile = decltype(full_tile_constant)::value;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv_mxu_tiled.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv_mxu_tiled.metalnew file mode 100644index 000000000..f958d7b64--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv_mxu_tiled.metal@@ -0,0 +1,158 @@+#include "../../common/dsl.h"+#include "../../common/soft_cap.h"+#include "../../generated/gemm.h"+#include "../../hadamard_transform/hadamard_transform.h"+#include "common/arguments.h"+#include "common/quant_metadata.h"+#include "common/tile.h"++using namespace metal;+using namespace uzu;+using namespace uzu::gemm;++#define GEMV_MXU_TILED_SIMDGROUPS 8u++template <uint GROUP_SIZE>+VARIANTS(GROUP_SIZE, 32, 64)+KERNEL(GemvMxuTiled)(+ const device uint4* b,+ const device bfloat* scales,+ const device bfloat* a,+ device bfloat* d,+ const device bfloat* output_bias+ OPTIONAL(output_transform.contains(GemmDTransform::BIAS)),+ const device int32_t* hadamard_factors+ OPTIONAL(output_transform.contains(GemmDTransform::RHT)),+ const constant uint& in_vec_size,+ const constant uint& out_vec_size,+ const constant float& ab_scale,+ const constant uint& scale_output_stride,+ const constant uint& scale_group_stride,+ const constant float& soft_cap+ OPTIONAL(output_transform.contains(GemmDTransform::SOFT_CAP)),+ const GemmDTransform output_transform SPECIALIZE,+ const bool signed_codes SPECIALIZE,+ threadgroup float shared_results[GEMV_MXU_TILED_SIMDGROUPS * 32],+ const uint block GROUPS(out_vec_size / 32),+ const uint lane THREADS(32),+ const uint k_slice THREADS(GEMV_MXU_TILED_SIMDGROUPS)+) {+ constexpr uint K_SPLIT = GEMV_MXU_TILED_SIMDGROUPS;+ using Tile = GemvTile<1, 32, 4, 1, K_SPLIT, K_SPLIT>;+ using Ops = GemvOperands<bfloat, bfloat, bfloat>;+ using Metadata = QuantMetadata<Tile, bfloat, bfloat, bfloat, GemmBPrologueKind::ScaleSymmetricDequant, 8>;+ constexpr uint CHUNK_VALUES = 32;+ constexpr uint SEGMENT_VECTORS = 2;+ constexpr uint BLOCK_CHUNK_VECTORS = 32 * SEGMENT_VECTORS;+ const Ops ops = {+ reinterpret_cast<const device uint32_t*>(b),+ scales,+ nullptr,+ nullptr,+ a,+ d,+ output_bias,+ hadamard_factors,+ nullptr+ };+ const GemvParams params = {+ in_vec_size,+ out_vec_size,+ scale_output_stride,+ scale_group_stride,+ 0,+ 0,+ 1,+ ab_scale,+ soft_cap,+ output_transform,+ false,+ signed_codes,+ GemmBLayout::MxuTiled+ };++ const uint pair = lane % 16;+ const uint parity = lane / 16;+ const uint y = (pair & 3) | (pair >> 3 << 2);+ const uint half_index = pair >> 2 & 1;+ uint rows[Tile::ROWS_PER_LANE];+ METAL_PRAGMA_UNROLL+ for (uint slot = 0; slot < Tile::ROWS_PER_LANE; ++slot) {+ rows[slot] = block * 32 + y + 8 * slot;+ }+ const uint chunks = in_vec_size / CHUNK_VALUES;+ const device uint4* block_weights = b + block * (in_vec_size / CHUNK_VALUES) * BLOCK_CHUNK_VECTORS;++ float result[1][Tile::ROWS_PER_LANE] = {{0}};+#pragma clang loop unroll_count(2)+ for (uint chunk = 2 * k_slice + parity; chunk < chunks; chunk += 2 * K_SPLIT) {+ const device uint4* source = block_weights + chunk * BLOCK_CHUNK_VECTORS + pair * 2 * SEGMENT_VECTORS;+ uint4 packed[2 * SEGMENT_VECTORS];+ METAL_PRAGMA_UNROLL+ for (uint vector = 0; vector < 2 * SEGMENT_VECTORS; ++vector) {+ packed[vector] = source[vector];+ }+ Metadata metadata;+ metadata.load(chunk * CHUNK_VALUES / GROUP_SIZE, rows, ops, params);+ float partial[1][Tile::ROWS_PER_LANE] = {{0}};+ float input_sum[1] = {0};+ const device bfloat* input = a + chunk * CHUNK_VALUES + 8 * half_index;+ METAL_PRAGMA_UNROLL+ for (uint step = 0; step < 2; ++step) {+ float input_values[8];+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < 4; ++i) {+ input_values[i] = float(input[4 * step + i]);+ input_values[4 + i] = float(input[16 + 4 * step + i]);+ }+ METAL_PRAGMA_UNROLL+ for (uint slot = 0; slot < Tile::ROWS_PER_LANE; ++slot) {+ float weight_values[8];+ metadata.decode(packed, step * 4 + slot, slot, weight_values);+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < 8; ++i) {+ partial[0][slot] = fma(input_values[i], weight_values[i], partial[0][slot]);+ }+ }+ }+ metadata.fold(result, partial, input_sum);+ }++ METAL_PRAGMA_UNROLL+ for (uint slot = 0; slot < Tile::ROWS_PER_LANE; ++slot) {+ result[0][slot] += simd_shuffle_xor(result[0][slot], 4);+ result[0][slot] += simd_shuffle_xor(result[0][slot], 16);+ if ((lane & 20) == 0) {+ shared_results[k_slice * 32 + y + 8 * slot] = result[0][slot];+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (k_slice != 0) {+ return;+ }++ float value = 0.0f;+ METAL_PRAGMA_UNROLL+ for (uint slice = 0; slice < K_SPLIT; ++slice) {+ value += shared_results[slice * 32 + lane];+ }+ const uint row = block * 32 + lane;+ const bool fused_hadamard = output_transform.contains(GemmDTransform::RHT);+ if (output_transform.contains(GemmDTransform::SCALE)) {+ value *= ab_scale;+ }+ if (!fused_hadamard && output_transform.contains(GemmDTransform::BIAS)) {+ value += float(output_bias[row]);+ }+ if (output_transform.contains(GemmDTransform::SOFT_CAP)) {+ value = apply_soft_cap(value, soft_cap);+ }+ bfloat output = bfloat(value);+ if (fused_hadamard) {+ output = simdgroup_output_random_hadamard_transform(ushort(lane), output, hadamard_factors[row]);+ if (output_transform.contains(GemmDTransform::BIAS)) {+ output += output_bias[row];+ }+ }+ d[row] = output;+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rsindex e3ec42f2a..0a29f270d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -14,8 +14,9 @@ use crate::{command_buffer::MetalCommandBufferEncoding,context::MetalContext,error::MetalError,+ gpu_types::gemm::GemmBLayout,kernel::{- GemvMetalKernel,+ GemvMetalKernel, GemvMxuTiledMetalKernel,matmul::{MatmulOutputWork,gemv::policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK},@@ -27,6 +28,7 @@ use crate::{};const GEMV_MAX_BATCH: u32 = 8;+const MXU_TILED_KERNEL_MIN_K: u32 = 2048;#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct GemvSpecialization {@@ -43,12 +45,15 @@ pub struct GemvSpecialization {group_lanes: u32,gathered: bool,signed_codes: bool,+ gemm_b_layout: GemmBLayout,full_tile: bool,+ mxu_tiled_kernel: bool,}impl GemvSpecialization {pub fn select_shape(shape: &MatmulShape,+ gemm_b_layout: GemmBLayout,weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,@@ -58,6 +63,14 @@ impl GemvSpecialization {let is_quant = shape.is_quant();let bits = shape.b_bits.unwrap_or(0);let bf16_io = input_data_type == DataType::BF16 && output_data_type == DataType::BF16;+ let mxu_tiled_kernel = gemm_b_layout == GemmBLayout::MxuTiled+ && shape.b_prologue == GemmBPrologueKind::ScaleSymmetricDequant+ && shape.m == 1+ && !shape.gathered+ && bf16_io+ && weights_data_type == DataType::BF16+ && shape.k >= MXU_TILED_KERNEL_MIN_K+ && shape.n >= 32 * gpu_core_count;let tile = if is_quant && shape.gathered {policy::gathered_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n)} else if is_quant {@@ -71,6 +84,15 @@ impl GemvSpecialization {shape.k,bf16_io,)+ .map(|tile| match gemm_b_layout {+ GemmBLayout::MxuTiled if mxu_tiled_kernel => {+ policy::GemvTile::quantized_output_tile(8, 32, 1, 32, tile.group_lanes)+ },+ GemmBLayout::MxuTiled if tile.output_row_tile() < 16 => {+ policy::GemvTile::quantized_output_tile(8, 16, 1, 32, tile.group_lanes)+ },+ _ => tile,+ })} else {let mixed_precision = weights_data_type == DataType::F32&& (input_data_type != DataType::F32 || output_data_type != DataType::F32);@@ -80,11 +102,17 @@ impl GemvSpecialization {let input_aligned = shape.k.is_multiple_of(FP_K_BLOCK);Some(policy::fp_tile(gpu_core_count, apple_gpu_family, shape.m, shape.n, shape.k, input_aligned))};- Self::select_tile(shape, weights_data_type, input_data_type, output_data_type, tile?)+ Self::select_tile(shape, gemm_b_layout, weights_data_type, input_data_type, output_data_type, tile?).map(+ |specialization| Self {+ mxu_tiled_kernel,+ ..specialization+ },+ )}pub fn select_tile(shape: &MatmulShape,+ gemm_b_layout: GemmBLayout,weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,@@ -137,7 +165,9 @@ impl GemvSpecialization {group_lanes: tile.group_lanes,gathered: shape.gathered,signed_codes: shape.signed_codes,+ gemm_b_layout,full_tile: full_tile(shape, tile),+ mxu_tiled_kernel: false,};Some(specialization)}@@ -175,6 +205,7 @@ impl GemvSpecialization {self.output_transform,self.gathered,self.signed_codes,+ self.gemm_b_layout,self.full_tile,)}@@ -226,6 +257,7 @@ pub struct GemvKernel {input_data_type: DataType,output_data_type: DataType,pipelines: HashMap<GemvSpecialization, GemvMetalKernel>,+ mxu_tiled_pipelines: HashMap<GemvSpecialization, GemvMxuTiledMetalKernel>,}impl GemvKernel {@@ -239,6 +271,7 @@ impl GemvKernel {input_data_type,output_data_type,pipelines: HashMap::new(),+ mxu_tiled_pipelines: HashMap::new(),}}@@ -320,6 +353,38 @@ impl GemvKernel {});let output_group_count = n.div_ceil(specialization.output_row_tile());let context = command_buffer.context();+ if specialization.mxu_tiled_kernel {+ let MatmulB::Quantized(quantized) = b else {+ return Err(MatmulMetalError::UnsupportedLayout {+ path: "Gemv",+ });+ };+ let pipeline = match self.mxu_tiled_pipelines.entry(specialization) {+ Entry::Occupied(entry) => entry.into_mut(),+ Entry::Vacant(entry) => entry.insert(GemvMxuTiledMetalKernel::new(+ context,+ specialization.group_size,+ specialization.output_transform,+ specialization.signed_codes,+ )?),+ };+ pipeline.encode(+ quantized.codes,+ quantized.scales,+ a,+ d.reborrow(),+ gemv_bias,+ gemv_rht_factors,+ k,+ n,+ ab_scale,+ scale_strides.output_stride,+ scale_strides.group_stride,+ soft_cap,+ command_buffer,+ );+ return Ok(());+ }let pipeline = self.get_or_create(context, specialization)?;match b {MatmulB::FullPrecision {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rsindex deeb74426..ad2e00e1e 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs@@ -3,7 +3,7 @@ use metal::MTLGPUFamily;use crate::{backends::{common::{- BufferMut, BufferRef, CommandBufferEncoding,+ Backend, BufferMut, BufferRef, CommandBufferEncoding,kernel::{ActivationQuantization,activation_transform::ACTIVATION_SCALE_GROUP_SIZE,@@ -16,7 +16,8 @@ use crate::{Metal,command_buffer::MetalCommandBufferEncoding,context::MetalContext,- gpu_types::gemm::GemmTiling,+ error::MetalError,+ gpu_types::gemm::{GemmBLayout, GemmTiling},kernel::matmul::{GemmKernel, MatmulMetalError,gemm::{self, GemmPlan, GemmProblem},@@ -80,6 +81,7 @@ impl MatmulMetalKernel {pub fn choose_dispatch(shape: &MatmulShape,+ gemm_b_layout: GemmBLayout,device_name: &str,gpu_core_count: u32,apple_gpu_family: MTLGPUFamily,@@ -98,14 +100,22 @@ impl MatmulMetalKernel {if let Some(route) = qmv::route(device_name, apple_gpu_family, supports_mxu, shape, all_bf16) {return match route {QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) => MatmulDispatch::Gemv(- GemvSpecialization::select_tile(shape, weights_data_type, input_data_type, output_data_type, tile)- .expect("typed QMV route must contain a legal GEMV tile"),+ GemvSpecialization::select_tile(+ shape,+ gemm_b_layout,+ weights_data_type,+ input_data_type,+ output_data_type,+ tile,+ )+ .expect("typed QMV route must contain a legal GEMV tile"),),QmvRoute::MainGemm(plan) => MatmulDispatch::Gemm(plan),};}let gemv = GemvSpecialization::select_shape(shape,+ gemm_b_layout,weights_data_type,input_data_type,output_data_type,@@ -127,10 +137,12 @@ impl MatmulMetalKernel {fn select_dispatch(&self,shape: &MatmulShape,+ gemm_b_layout: GemmBLayout,context: &MetalContext,) -> MatmulDispatch {Self::choose_dispatch(shape,+ gemm_b_layout,&context.device_name,context.gpu_core_count,context.apple_gpu_family,@@ -145,6 +157,7 @@ impl MatmulMetalKernel {impl MatmulKernel for MatmulMetalKernel {type Backend = Metal;type Error = MatmulMetalError;+ type WeightsLayout = GemmBLayout;fn new(context: &MetalContext,@@ -193,6 +206,8 @@ impl MatmulKernel for MatmulMetalKernel {|| shape.b_leading_dimension.is_some()|| !shape.k.is_multiple_of(ACTIVATION_SCALE_GROUP_SIZE)|| !shape.k.is_multiple_of(weight_group_size)+ || !shape.n.is_multiple_of(32)+ || shape.b_bits.and_then(|bits| GemmKernel::mxu_tiled_words_per_thread(shape.k, bits)).is_none(){return None;}@@ -203,25 +218,48 @@ impl MatmulKernel for MatmulMetalKernel {fn select_activation_format(&self,bf16_shape: &MatmulShape,+ weights_layout: GemmBLayout,context: &MetalContext,) -> ActivationFormat {if bf16_shape.params_layout != Some(QuantParamsLayout::GroupOutput)|| !supports_integer_right_operand(bf16_shape)- || matches!(self.select_dispatch(bf16_shape, context), MatmulDispatch::Gemv(_))+ || matches!(self.select_dispatch(bf16_shape, weights_layout, context), MatmulDispatch::Gemv(_)){return ActivationFormat::Bf16;}+ if weights_layout == GemmBLayout::MxuTiled {+ return ActivationFormat::Int8;+ }let a8_shape = MatmulShape {a_full_precision: false,..*bf16_shape};- match self.select_dispatch(&a8_shape, context) {+ match self.select_dispatch(&a8_shape, weights_layout, context) {MatmulDispatch::Gemm(plan) if plan.engine == gemm::GemmEngine::Mxu => ActivationFormat::Int8,_ => ActivationFormat::Bf16,}}+ fn optimize_weights_layout(+ &self,+ a8_shape: &MatmulShape,+ weights: &mut <Metal as Backend>::GlobalBuffer,+ context: &MetalContext,+ ) -> Result<GemmBLayout, MetalError> {+ if self.select_activation_quantization(a8_shape, context).is_none() {+ return Ok(GemmBLayout::RowMajor);+ }+ GemmKernel::prepare_gemm_b_layout(+ weights,+ a8_shape.n,+ a8_shape.k,+ a8_shape.b_bits.expect("quantized weights"),+ context,+ )?;+ Ok(GemmBLayout::MxuTiled)+ }+fn encode(&mut self,arguments: MatmulArguments<@@ -235,7 +273,7 @@ impl MatmulKernel for MatmulMetalKernel {command_buffer: &mut MetalCommandBufferEncoding,) -> Result<(), Self::Error> {let shape = MatmulShape::from_arguments(&arguments);- let plan = match self.select_dispatch(&shape, command_buffer.context()) {+ let plan = match self.select_dispatch(&shape, arguments.b.weights_layout(), command_buffer.context()) {MatmulDispatch::Gemv(gemv) => {return self.gemv.encode(arguments, gemv, &self.output_work, command_buffer);},diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rsindex ce609b594..172ecda61 100644--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs@@ -118,6 +118,15 @@ impl<B: Backend> LinearMatmul<B> {self.matrix.try_prepare_a8_storage().then_some(quantization)}+ pub fn optimize_weights_layout(+ &mut self,+ context: &B::Context,+ ) -> Result<(), B::Error> {+ let shape = self.matmul_shape(1, false);+ let kernel = self.kernel.lock();+ self.matrix.optimize_weights_layout(|weights| kernel.optimize_weights_layout(&shape, weights, context))+ }+pub(super) fn encode_with_a(&self,a: MatmulA<impl BufferRef<Backend = B>>,@@ -204,6 +213,6 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {context: &B::Context,) -> ActivationFormat {let bf16_shape = self.matmul_shape(batch_dim, true);- self.kernel.lock().select_activation_format(&bf16_shape, context)+ self.kernel.lock().select_activation_format(&bf16_shape, self.matmul_b().weights_layout(), context)}}diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rsindex 4f8e5a9ef..4772b0f08 100644--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs@@ -73,6 +73,9 @@ impl<B: Backend> RHTLinearWrapper<B> {context,)?;let activation_quantization = inner_linear.prepare_a8(context);+ if activation_quantization.is_some() {+ inner_linear.optimize_weights_layout(context).map_err(RHTLinearWrapperError::BackendError)?;+ }Self::build_self_contained(input_data_type,LinearInputPreparation {@@ -115,6 +118,9 @@ impl<B: Backend> RHTLinearWrapper<B> {rht_signs,activation_quantization: inner_linear.prepare_a8(context),};+ if input_preparation.activation_quantization.is_some() {+ inner_linear.optimize_weights_layout(context).map_err(RHTLinearWrapperError::BackendError)?;+ }if input_preparation.activation_quantization.is_some() && !allow_prequantized_activation {let wrapper = Self::build_self_contained(input_data_type, input_preparation, inner_linear, context)?;Ok(Some((Box::new(wrapper), None)))diff --git a/crates/uzu-engine/src/encodable_block/weight_matrix.rs b/crates/uzu-engine/src/encodable_block/weight_matrix.rsindex a80036620..0ad954d29 100644--- a/crates/uzu-engine/src/encodable_block/weight_matrix.rs+++ b/crates/uzu-engine/src/encodable_block/weight_matrix.rs@@ -5,7 +5,7 @@ use crate::{backends::common::{Backend, BufferCpuAccessible, BufferMut,gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},- kernel::matmul::{MatmulB, QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection},+ kernel::matmul::{MatmulB, QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection, WeightsLayout},},config::weight_matrix::{AnyWeightMatrixSpec, Layout},data_type::DataType,@@ -86,6 +86,7 @@ struct Quantized<B: Backend> {params: QuantParams,info: QuantizationInfo,signed_codes: bool,+ weights_layout: WeightsLayout<B>,}pub struct WeightMatrix<B: Backend> {@@ -163,6 +164,7 @@ impl<B: Backend> WeightMatrix<B> {params,info,signed_codes: false,+ weights_layout: Default::default(),}),})}@@ -204,9 +206,23 @@ impl<B: Backend> WeightMatrix<B> {mode,group_size,signed_codes,+ weights_layout: quantized.weights_layout,})}+ pub fn optimize_weights_layout<E>(+ &mut self,+ optimize: impl FnOnce(&mut B::GlobalBuffer) -> Result<WeightsLayout<B>, E>,+ ) -> Result<(), E> {+ let Some(quantized) =+ self.quantized.as_mut().filter(|quantized| quantized.weights_layout == WeightsLayout::<B>::default())+ else {+ return Ok(());+ };+ quantized.weights_layout = optimize(&mut self.values)?;+ Ok(())+ }+pub fn try_prepare_a8_storage(&mut self) -> bool {let Some(quantized) = self.quantized.as_mut() else {return false;diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rsindex abf3ca05f..e3fb0eded 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs@@ -21,7 +21,7 @@ use crate::{},},},- metal::{Metal, context::MetalContext, kernel::matmul::gemm::GemmEngine},+ metal::{Metal, context::MetalContext, gpu_types::gemm::GemmBLayout, kernel::matmul::gemm::GemmEngine},},data_type::DataType,tests::{@@ -146,6 +146,7 @@ impl BenchmarkData {mode: self.mode,group_size: self.group_size,signed_codes: false,+ weights_layout: GemmBLayout::RowMajor,}),b_leading_dimension: None,b_transpose: true,@@ -201,6 +202,7 @@ fn encode_step(mode: data.mode,group_size: data.group_size,signed_codes: !matches!(data.mode, QuantizationMode::U4),+ weights_layout: GemmBLayout::RowMajor,}),b_leading_dimension: None,b_transpose: true,diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rsindex ac3233c8b..2dbc9e8bb 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs@@ -3,8 +3,12 @@ use uzu_engine_macros::uzu_test;use super::*;use crate::backends::{- common::{gpu_types::gemm::GemmDTransform, kernel::matmul::QuantParamsLayout},- metal::kernel::matmul::MatmulMetalKernel,+ common::{+ Context,+ gpu_types::gemm::GemmDTransform,+ kernel::matmul::{MatmulKernel, QuantParamsLayout},+ },+ metal::{context::MetalContext, kernel::matmul::MatmulMetalKernel},};fn shape(@@ -118,10 +122,9 @@ fn selection_fallbacks_and_split_k_are_preserved() {assert_eq!(problem(quant(shape(64, 4096, 4095)), DataType::BF16).select_plan().engine, Simdgroup);for (m, n, k, expected_tiling, expected_split_k) in [- (16, 4096, 4096, Tile16x32x256_Simdgroups1x1x4, 4),+ (16, 4096, 4096, Tile16x32x256_Simdgroups1x1x4, 2),(16, 1024, 4096, Tile16x32x256_Simdgroups1x1x4, 8),- (16, 34816, 4096, Tile16x32x256_Simdgroups1x1, 1),- (8, 34816, 4096, Tile16x32x256_Simdgroups1x1x4, 1),+ (16, 34816, 4096, Tile16x32x256_Simdgroups1x1x4, 1),(8, 4096, 384, Tile16x32x256_Simdgroups1x1, 1),] {let mut a8 = quant(shape(m, n, k));@@ -197,3 +200,39 @@ fn gemv_gemm_route_boundaries_are_preserved() {let plan = problem(gathered, DataType::BF16).select_plan();assert!(!MatmulMetalKernel::prefer_gemm_over_gemv(gathered, plan, DataType::BF16, DataType::BF16, DataType::BF16,));}++#[uzu_test]+fn int8_activations_require_mxu_tiled_weights() {+ let context = MetalContext::new().expect("Metal context");+ if !context.supports_mxu {+ return;+ }+ let kernel =+ MatmulMetalKernel::new(&context, DataType::BF16, DataType::BF16, DataType::BF16).expect("matmul kernel");+ for (n, k, bits, expected) in [+ (96, 256, 4, true),+ (72, 256, 4, false),+ (64, 40960, 4, true),+ (64, 41088, 4, false),+ (64, 20480, 8, true),+ (64, 20608, 8, false),+ ] {+ let shape = MatmulShape {+ b_prologue: if bits == 4 {+ GemmBPrologueKind::ScaleZeroPointDequant+ } else {+ GemmBPrologueKind::ScaleSymmetricDequant+ },+ b_bits: Some(bits),+ signed_codes: bits == 8,+ a_full_precision: false,+ params_layout: Some(QuantParamsLayout::GroupOutput),+ ..quant(shape(16, n, k))+ };+ assert_eq!(+ kernel.select_activation_quantization(&shape, &context).is_some(),+ expected,+ "n={n} k={k} bits={bits}"+ );+ }+}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rsindex c4109b892..4ebc8fa76 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs@@ -3,9 +3,12 @@ use uzu_engine_macros::uzu_test;use super::*;use crate::{- backends::common::{- gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},- kernel::matmul::{MatmulShape, QuantParamsLayout},+ backends::{+ common::{+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},+ kernel::matmul::{MatmulShape, QuantParamsLayout},+ },+ metal::gpu_types::gemm::GemmBLayout,},data_type::DataType,};@@ -151,6 +154,7 @@ fn block_unaligned_quantized_k_stays_on_gemv() {assert!(super::super::kernel::GemvSpecialization::select_shape(&shape,+ GemmBLayout::RowMajor,DataType::BF16,DataType::BF16,DataType::BF16,@@ -166,6 +170,7 @@ fn specialization_preserves_quantized_route() {let select = |n, transform| {super::super::kernel::GemvSpecialization::select_shape(&quant_shape(4, n, transform),+ GemmBLayout::RowMajor,DataType::BF16,DataType::BF16,DataType::BF16,@@ -186,6 +191,7 @@ fn gathered_group_major_is_a_gemv_route() {assert!(super::super::kernel::GemvSpecialization::select_tile(&shape,+ GemmBLayout::RowMajor,DataType::BF16,DataType::BF16,DataType::BF16,diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rsindex 14d21ae85..c9e647499 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs@@ -9,7 +9,10 @@ use crate::{gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},kernel::matmul::{MatmulShape, QuantParamsLayout},},- metal::kernel::matmul::{MatmulDispatch, MatmulMetalKernel, gemv::GemvSpecialization},+ metal::{+ gpu_types::gemm::GemmBLayout,+ kernel::matmul::{MatmulDispatch, MatmulMetalKernel, gemv::GemvSpecialization},+ },},data_type::DataType,};@@ -94,6 +97,7 @@ fn table_is_complete_and_fingerprint_is_stable() {assert_eq!(route(device_name, apple_gpu_family, supports_mxu, &problem, true), Some(selected));let runtime = MatmulMetalKernel::choose_dispatch(&problem,+ GemmBLayout::RowMajor,device_name,gpu_core_count,apple_gpu_family,@@ -105,6 +109,7 @@ fn table_is_complete_and_fingerprint_is_stable() {if let QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) = selected {let specialization = GemvSpecialization::select_tile(&problem,+ GemmBLayout::RowMajor,DataType::BF16,DataType::BF16,DataType::BF16,@@ -133,7 +138,17 @@ fn weaver_linears_select_the_tuned_tile() {p.d_transform = GemmDTransform::RHT;let tile = GemvTile::quantized_output_tile(2, 16, 4, 8, 1);assert_eq!(route("Apple M5 Max", MTLGPUFamily::Apple10, true, &p, true), Some(QmvRoute::Tuned(tile)));- assert!(GemvSpecialization::select_tile(&p, DataType::BF16, DataType::BF16, DataType::BF16, tile).is_some());+ assert!(+ GemvSpecialization::select_tile(+ &p,+ GemmBLayout::RowMajor,+ DataType::BF16,+ DataType::BF16,+ DataType::BF16,+ tile+ )+ .is_some()+ );}}@@ -161,11 +176,29 @@ fn exact_lookup_rejects_non_matrix_inputs() {let QmvRoute::Tuned(tile) = selected else {panic!("test anchor must use a tuned tile");};- let deferred = GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).unwrap();+ let deferred = GemvSpecialization::select_tile(+ &rht,+ GemmBLayout::RowMajor,+ DataType::BF16,+ DataType::BF16,+ DataType::BF16,+ tile,+ )+ .unwrap();assert_eq!(deferred.output_row_tile(), 16);assert!(!deferred.fuses_rht());rht.n -= 1;- assert!(GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).is_none());+ assert!(+ GemvSpecialization::select_tile(+ &rht,+ GemmBLayout::RowMajor,+ DataType::BF16,+ DataType::BF16,+ DataType::BF16,+ tile+ )+ .is_none()+ );let p = problem(7, 5120, 17408, 8, 64, GemmBPrologueKind::ScaleSymmetricDequant);assert!(matches!(route("Apple M5 Max", MTLGPUFamily::Apple10, true, &p, true), Some(QmvRoute::MainGemm(_))));@@ -179,6 +212,7 @@ fn normal_routing_handles_inputs_outside_the_frozen_matrix() {assert_eq!(route("Apple M1", MTLGPUFamily::Apple7, false, &problem, true), None);let specialization = GemvSpecialization::select_shape(&problem,+ GemmBLayout::RowMajor,DataType::BF16,DataType::BF16,DataType::BF16,@@ -188,7 +222,7 @@ fn normal_routing_handles_inputs_outside_the_frozen_matrix() {.expect("normal M1 policy should select GEMV for this anchor");assert!(matches!(MatmulMetalKernel::choose_dispatch(- &problem,+ &problem, GemmBLayout::RowMajor,"Apple M1",8,MTLGPUFamily::Apple7,@@ -208,12 +242,26 @@ fn normal_routing_handles_inputs_outside_the_frozen_matrix() {let mut with_rht = fp;with_rht.d_transform = GemmDTransform::RHT;for (cores, family) in [(8, MTLGPUFamily::Apple7), (40, MTLGPUFamily::Apple10)] {- let plain =- GemvSpecialization::select_shape(&fp, DataType::BF16, DataType::BF16, DataType::BF16, cores, family)- .expect("generic M=1 FP GEMV tile");- let rht =- GemvSpecialization::select_shape(&with_rht, DataType::BF16, DataType::BF16, DataType::BF16, cores, family)- .expect("generic M=1 FP RHT GEMV tile");+ let plain = GemvSpecialization::select_shape(+ &fp,+ GemmBLayout::RowMajor,+ DataType::BF16,+ DataType::BF16,+ DataType::BF16,+ cores,+ family,+ )+ .expect("generic M=1 FP GEMV tile");+ let rht = GemvSpecialization::select_shape(+ &with_rht,+ GemmBLayout::RowMajor,+ DataType::BF16,+ DataType::BF16,+ DataType::BF16,+ cores,+ family,+ )+ .expect("generic M=1 FP RHT GEMV tile");assert!(plain.output_row_tile() < 32);assert_eq!(rht, plain);}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rsindex 860ed111a..8b647b1a0 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs@@ -15,11 +15,19 @@ use crate::{kernel::{Kernels,activation_transform::ACTIVATION_SCALE_GROUP_SIZE,- matmul::{MatmulDOps, MatmulKernel, MatmulOutput, QuantParams, QuantParamsLayout, QuantParamsStrides},+ matmul::{+ MatmulDOps, MatmulKernel, MatmulOutput, MatmulShape, QuantParams, QuantParamsLayout,+ QuantParamsStrides,+ },},},cpu::Cpu,- metal::{Metal, context::MetalContext, kernel::matmul::gemm::GemmEngine},+ metal::{+ Metal,+ context::MetalContext,+ gpu_types::gemm::GemmBLayout,+ kernel::matmul::{MatmulMetalKernel, gemm::GemmEngine},+ },},data_type::DataTypeRepr,tests::{@@ -28,9 +36,25 @@ use crate::{},};+fn tile_weights<T: DataTypeRepr + Float>(+ matmul: &MatmulMetalKernel,+ buffers: &mut QuantBuffers<Metal, T>,+ input: &QuantInput<T>,+ context: &MetalContext,+) {+ let shape = MatmulShape {+ a_full_precision: false,+ ..MatmulShape::from_arguments(&quant_arguments(buffers, input))+ };+ buffers.weights_layout =+ matmul.optimize_weights_layout(&shape, &mut buffers.w, context).expect("optimize weights layout");+ assert_eq!(buffers.weights_layout, GemmBLayout::MxuTiled);+}+fn run_quant_metal<T: DataTypeRepr + Float>(context: &MetalContext,input: &QuantInput<T>,+ gemm_b_layout: GemmBLayout,dispatch: Option<GemmEngine>,) -> Vec<T> {let mut buffers = QuantBuffers::<Metal, T>::allocate(context, input);@@ -41,6 +65,9 @@ fn run_quant_metal<T: DataTypeRepr + Float>(T::data_type(),).expect("MatmulMetalKernel");+ if gemm_b_layout == GemmBLayout::MxuTiled {+ tile_weights(&matmul, &mut buffers, input, context);+ }let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");let args = quant_arguments(&mut buffers, input);if let Some(engine) = dispatch {@@ -118,7 +145,7 @@ fn run_parity<T: DataTypeRepr + Float + Debug + Display>() {let context = MetalContext::new().expect("Metal context");let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 0);- let unified = run_quant_metal::<T>(&context, &input, dispatch);+ let unified = run_quant_metal::<T>(&context, &input, GemmBLayout::RowMajor, dispatch);let reference = run_quant_cpu::<T>(&input);assert_parity::<T>(@@ -493,7 +520,7 @@ fn mxu_quant_parity_bf16(return;}let input = QuantInput::<bf16>::new(m, k, n, gs, bits, method, 0);- let actual = run_quant_metal::<bf16>(&context, &input, Some(GemmEngine::Mxu));+ let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::RowMajor, Some(GemmEngine::Mxu));let reference = run_quant_cpu::<bf16>(&input);assert_parity::<bf16>(&format!("MXU m={m} k={k} n={n} gs={gs} bits={bits} method={method:?}"),@@ -506,30 +533,43 @@ fn mxu_quant_parity_bf16(#[rstest]#[test_attr(uzu_test)]-#[case::sym_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]-#[case::bias_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleBias)]-#[case::zp_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleZeroPoint)]-#[case::m1_bias_w4_gs64(1u32, 64u32, 4u32, QuantizationMethod::ScaleBias)]+#[case::sym_w8_gs64(16u32, 256u32, 128u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]+#[case::bias_w4_gs32(16u32, 256u32, 128u32, 32u32, 4u32, QuantizationMethod::ScaleBias)]+#[case::zp_w4_gs32(16u32, 256u32, 128u32, 32u32, 4u32, QuantizationMethod::ScaleZeroPoint)]+#[case::m1_bias_w4_gs64(1u32, 256u32, 128u32, 64u32, 4u32, QuantizationMethod::ScaleBias)]+#[case::w8_m1(1u32, 256u32, 96u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]+#[case::w8_m16_split_k(16u32, 4096u32, 1024u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]+#[case::w8_m64(64u32, 512u32, 256u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]+#[case::w4_m1(1u32, 256u32, 96u32, 64u32, 4u32, QuantizationMethod::ScaleZeroPoint)]+#[case::w4_m16_split_k(16u32, 4096u32, 1024u32, 64u32, 4u32, QuantizationMethod::ScaleZeroPoint)]+#[case::w4_m24(24u32, 512u32, 96u32, 64u32, 4u32, QuantizationMethod::ScaleZeroPoint)]+#[case::w4_m128_bias(128u32, 512u32, 256u32, 64u32, 4u32, QuantizationMethod::ScaleBias)]+#[case::w4_m16_k17408(16u32, 17408u32, 64u32, 64u32, 4u32, QuantizationMethod::ScaleZeroPoint)]+#[case::w8_m16_k17408(16u32, 17408u32, 64u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]fn a8w_mxu_parity_bf16(#[case] m: u32,+ #[case] k: u32,+ #[case] n: u32,#[case] weight_gs: u32,#[case] bits: u32,#[case] method: QuantizationMethod,+ #[values(GemmBLayout::RowMajor, GemmBLayout::MxuTiled)] gemm_b_layout: GemmBLayout,) {let context = MetalContext::new().expect("Metal context");if !context.supports_mxu {return;}- let (k, n) = (256u32, 128u32);let (input, reference_input) = QuantInput::<bf16>::new(m, k, n, weight_gs, bits, method, 0).with_prepared_a_and_reference(ACTIVATION_SCALE_GROUP_SIZE,(method != QuantizationMethod::ScaleSymmetric).then_some(weight_gs),);- let actual = run_quant_metal(&context, &input.with_group_output(), Some(GemmEngine::Mxu));+ let actual = run_quant_metal(&context, &input.with_group_output(), gemm_b_layout, Some(GemmEngine::Mxu));let reference = run_quant_cpu::<bf16>(&reference_input);assert_parity::<bf16>(- &format!("A8W{bits} MXU m={m} weight_gs={weight_gs} method={method:?}"),+ &format!(+ "A8W{bits} MXU m={m} k={k} n={n} weight_gs={weight_gs} method={method:?} gemm_b_layout={gemm_b_layout:?}"+ ),&reference,&actual,0.08,@@ -558,7 +598,8 @@ fn a8w_independent_activation_group_parity_bf16(#[case] m: u32) {(method != QuantizationMethod::ScaleSymmetric).then_some(activation_group_size.min(weight_group_size)),);- let actual = run_quant_metal(&context, &input.with_group_output(), Some(GemmEngine::Mxu));+ let actual =+ run_quant_metal(&context, &input.with_group_output(), GemmBLayout::RowMajor, Some(GemmEngine::Mxu));let reference = run_quant_cpu::<bf16>(&reference_input);assert_parity::<bf16>(&format!("A8W{bits} act{activation_group_size} weight{weight_group_size} m={m} method={method:?}"),@@ -584,7 +625,7 @@ fn a8w4_zero_point_tail_parity(#[case] m: u32) {let input = QuantInput::<bf16>::new(m, 256, 72, 32, 4, QuantizationMethod::ScaleZeroPoint, 0).with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE, Some(32));let reference = run_quant_cpu::<bf16>(&input);- let actual = run_quant_metal(&context, &input.with_group_output(), Some(GemmEngine::Mxu));+ let actual = run_quant_metal(&context, &input.with_group_output(), GemmBLayout::RowMajor, Some(GemmEngine::Mxu));assert_parity::<bf16>(&format!("A8W4 ZP N-tail m={m}"), &reference, &actual, 0.08, 0.8);}@@ -605,7 +646,7 @@ fn signed_weights_full_precision_activations_parity_bf16(let reference = run_quant_cpu::<bf16>(&reference_input);for (label, dispatch) in [("gemv", None), ("simdgroup", Some(GemmEngine::Simdgroup))] {- let actual = run_quant_metal::<bf16>(&context, &input, dispatch);+ let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::RowMajor, dispatch);assert_parity::<bf16>(&format!("signed weights FP-A {label} bits={bits} method={method:?}"),&reference,@@ -766,3 +807,140 @@ fn parity_widened_f32_output(#[case] m: u32) {let actual = run_widened_f32::<Metal>(&context, &input);assert_parity::<f32>(&format!("widened f32 m={m}"), &reference, &actual, 0.05, 0.4);}++#[rstest]+#[test_attr(uzu_test)]+#[case::w4_zp_gs64(1024u32, 1024u32, 64u32, 4u32, ScaleZeroPoint)]+#[case::w4_zp_gs32_wide(1024u32, 3584u32, 32u32, 4u32, ScaleZeroPoint)]+#[case::w4_zp_gs64_deep(3584u32, 1024u32, 64u32, 4u32, ScaleZeroPoint)]+#[case::w4_bias_gs64(2048u32, 2048u32, 64u32, 4u32, ScaleBias)]+#[case::w8_sym_gs64(1024u32, 2048u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]+#[case::w8_sym_gs64_blocks(2048u32, 4096u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]+#[case::w8_sym_gs32_blocks(2048u32, 4096u32, 32u32, 8u32, QuantizationMethod::ScaleSymmetric)]+fn gemv_reads_mxu_tiled_layout_bf16(+ #[case] k: u32,+ #[case] n: u32,+ #[case] group_size: u32,+ #[case] bits: u32,+ #[case] method: QuantizationMethod,+ #[values(false, true)] output_hadamard: bool,+) {+ let context = MetalContext::new().expect("Metal context");+ if !context.supports_mxu {+ return;+ }+ let input = QuantInput::<bf16>::new(1, k, n, group_size, bits, method, 0).with_group_output();+ let input = if bits == 8 {+ input.with_signed_weight_codes()+ } else {+ input+ };+ let factors: Vec<i32> = (0..n)+ .map(|row| {+ if row.is_multiple_of(3) {+ -1+ } else {+ 1+ }+ })+ .collect();++ let cpu_context = <Cpu as Backend>::Context::new().expect("CPU context");+ let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &input);+ let cpu_factors = crate::tests::helpers::create_buffer_with_data::<Cpu, i32>(&cpu_context, &factors);+ let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(+ &cpu_context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ )+ .expect("CPU matmul kernel");+ let mut cpu_command_buffer = cpu_context.create_command_buffer(None, None).expect("CPU command buffer");+ let mut cpu_arguments = quant_arguments(&mut cpu_buffers, &input);+ cpu_arguments.output.ops = MatmulDOps {+ rht_factors: output_hadamard.then_some(&cpu_factors),+ ..MatmulDOps::none()+ };+ cpu_matmul.encode(cpu_arguments, &mut cpu_command_buffer).expect("CPU matmul");+ cpu_command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let reference = buffer_to_vec::<Cpu, bf16>(&cpu_buffers.y);++ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);+ let metal_factors = crate::tests::helpers::create_buffer_with_data::<Metal, i32>(&context, &factors);+ let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(+ &context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ )+ .expect("Metal matmul kernel");+ tile_weights(&matmul, &mut buffers, &input, &context);+ let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");+ let mut arguments = quant_arguments(&mut buffers, &input);+ arguments.output.ops = MatmulDOps {+ rht_factors: output_hadamard.then_some(&metal_factors),+ ..MatmulDOps::none()+ };+ matmul.encode(arguments, &mut command_buffer).expect("Metal matmul");+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let actual = buffer_to_vec::<Metal, bf16>(&buffers.y);++ assert_parity::<bf16>(+ &format!("MxuTiled GEMV k={k} n={n} gs={group_size} bits={bits} method={method:?} hadamard={output_hadamard}"),+ &reference,+ &actual,+ 0.05,+ 0.5,+ );+}++#[rstest]+#[test_attr(uzu_test)]+#[case::w4_zp_gs32(4u32, 32u32, ScaleZeroPoint)]+#[case::w8_sym_gs64(8u32, 64u32, QuantizationMethod::ScaleSymmetric)]+fn gathered_gemv_reads_mxu_tiled_layout_bf16(+ #[case] bits: u32,+ #[case] group_size: u32,+ #[case] method: QuantizationMethod,+) {+ let context = MetalContext::new().expect("Metal context");+ if !context.supports_mxu {+ return;+ }+ let (k, vocab, gathered_rows) = (256u32, 256u32, 10u32);+ let input = QuantInput::<bf16>::new(1, k, vocab, group_size, bits, method, 0).with_group_output();+ let input = if bits == 8 {+ input.with_signed_weight_codes()+ } else {+ input+ };+ let ids: Vec<u32> = (0..gathered_rows).map(|row| (row * 37 + 11) % vocab).collect();+ let dense = run_quant_cpu::<bf16>(&input);+ let expected: Vec<bf16> = ids.iter().map(|&id| dense[id as usize]).collect();++ let ids_buffer = crate::tests::helpers::create_buffer_with_data::<Metal, u32>(&context, &ids);+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);+ let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(+ &context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ )+ .expect("Metal matmul kernel");+ tile_weights(&matmul, &mut buffers, &input, &context);+ let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");+ let mut arguments = quant_arguments(&mut buffers, &input);+ arguments.gather_indices = Some(&ids_buffer);+ arguments.n = gathered_rows;+ matmul.encode(arguments, &mut command_buffer).expect("gathered GEMV");+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let actual = buffer_to_vec::<Metal, bf16>(&buffers.y)[..gathered_rows as usize].to_vec();++ assert_parity::<bf16>(+ &format!("gathered MxuTiled GEMV bits={bits} gs={group_size} method={method:?}"),+ &expected,+ &actual,+ 0.05,+ 0.5,+ );+}diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rsindex 083b34269..eb9e8c9e4 100644--- a/crates/uzu-engine/unit/common/matmul/quant.rs+++ b/crates/uzu-engine/unit/common/matmul/quant.rs@@ -13,7 +13,7 @@ use crate::{ActivationQuantization, ActivationTransform, Kernels,matmul::{Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput,- QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection,+ QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection, WeightsLayout,},},},@@ -293,6 +293,7 @@ pub struct QuantBuffers<B: Backend, T: DataTypeRepr + Float> {pub prepared_a_scales: Option<B::GlobalBuffer>,pub prepared_a_group_sums: Option<B::GlobalBuffer>,pub y: B::GlobalBuffer,+ pub weights_layout: WeightsLayout<B>,_t: std::marker::PhantomData<T>,}@@ -332,6 +333,7 @@ impl<B: Backend, T: DataTypeRepr + Float> QuantBuffers<B, T> {.filter(|prepared| !prepared.group_sums.is_empty()).map(|prepared| create_buffer_with_data::<B, i32>(context, &prepared.group_sums)),y: create_buffer::<B, T>(context, (input.m as usize) * (input.n as usize)),+ weights_layout: Default::default(),_t: std::marker::PhantomData,};if params_layout == QuantParamsLayout::GroupOutput {@@ -344,7 +346,15 @@ impl<B: Backend, T: DataTypeRepr + Float> QuantBuffers<B, T> {&'a self,input: &QuantInput<T>,) -> MatmulB<&'a B::GlobalBuffer> {- quant_b_variant(&self.w, &self.scales, self.zp.as_ref(), self.bias.as_ref(), input.params_layout, input)+ quant_b_variant(+ &self.w,+ &self.scales,+ self.zp.as_ref(),+ self.bias.as_ref(),+ input.params_layout,+ self.weights_layout,+ input,+ )}fn transpose_quant_params(@@ -384,6 +394,7 @@ fn quant_b_variant<TB: BufferRef, T: DataTypeRepr + Float>(zero_points: Option<TB>,biases: Option<TB>,params_layout: QuantParamsLayout,+ weights_layout: WeightsLayout<TB::Backend>,input: &QuantInput<T>,) -> MatmulB<TB> {let params = QuantParams::new(params_layout, input.n, input.k.div_ceil(input.group_size));@@ -400,6 +411,7 @@ fn quant_b_variant<TB: BufferRef, T: DataTypeRepr + Float>(mode: input.mode,group_size: input.group_size,signed_codes: input.signed_codes,+ weights_layout,})}@@ -417,9 +429,10 @@ pub fn quant_arguments<'a, B: Backend, T: DataTypeRepr + Float>(prepared_a_scales,prepared_a_group_sums,y,+ weights_layout,..} = buffers;- let b = quant_b_variant(&*w, &*scales, zp.as_ref(), bias.as_ref(), input.params_layout, input);+ let b = quant_b_variant(&*w, &*scales, zp.as_ref(), bias.as_ref(), input.params_layout, *weights_layout, input);let a = match &input.prepared_a {Some(prepared) => MatmulA::Int8Symmetric {values: prepared_a.as_ref().expect("prepared activation buffer"),