A high-performance inference engine for AI models
interleaved rows w4 layout
CCYeh48d8469b327a
diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rsindex 39e94d78c..0469af9dc 100644--- a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs+++ b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs@@ -15,6 +15,7 @@ pub enum GemmBLayout {#[default]RowMajor,MxuTiled,+ InterleavedRows,}#[repr(C)]@@ -35,6 +36,7 @@ pub enum GemmTiling {Tile8x32x64_Simdgroups1x2x4,Tile8x32x128_Simdgroups1x2x4,Tile16x32x32_Simdgroups2x1,+ Tile8x32x64_Simdgroups1x1x4,}pub const MXU_SIMDGROUP_BLOCK_K: u32 = 32;@@ -54,7 +56,9 @@ impl GemmTiling {Self::Tile64x32x256_Simdgroups4x1 => 64,Self::Tile64x64x256_Simdgroups2x2 => 64,Self::Tile128x128x256_Simdgroups4x4 => 128,- Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 8,+ Self::Tile8x32x64_Simdgroups1x2x4+ | Self::Tile8x32x64_Simdgroups1x1x4+ | Self::Tile8x32x128_Simdgroups1x2x4 => 8,Self::Tile16x32x32_Simdgroups2x1 => 16,}}@@ -72,7 +76,9 @@ impl GemmTiling {Self::Tile64x32x256_Simdgroups4x1 => 32,Self::Tile64x64x256_Simdgroups2x2 => 64,Self::Tile128x128x256_Simdgroups4x4 => 128,- Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 32,+ Self::Tile8x32x64_Simdgroups1x2x4+ | Self::Tile8x32x64_Simdgroups1x1x4+ | Self::Tile8x32x128_Simdgroups1x2x4 => 32,Self::Tile16x32x32_Simdgroups2x1 => 32,}}@@ -90,7 +96,7 @@ impl GemmTiling {Self::Tile64x32x256_Simdgroups4x1 => 256,Self::Tile64x64x256_Simdgroups2x2 => 256,Self::Tile128x128x256_Simdgroups4x4 => 256,- Self::Tile8x32x64_Simdgroups1x2x4 => 64,+ Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x64_Simdgroups1x1x4 => 64,Self::Tile8x32x128_Simdgroups1x2x4 => 128,Self::Tile16x32x32_Simdgroups2x1 => 32,}@@ -109,7 +115,9 @@ impl GemmTiling {Self::Tile64x32x256_Simdgroups4x1 => 4,Self::Tile64x64x256_Simdgroups2x2 => 2,Self::Tile128x128x256_Simdgroups4x4 => 4,- Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 1,+ Self::Tile8x32x64_Simdgroups1x2x4+ | Self::Tile8x32x64_Simdgroups1x1x4+ | Self::Tile8x32x128_Simdgroups1x2x4 => 1,Self::Tile16x32x32_Simdgroups2x1 => 2,}}@@ -128,7 +136,7 @@ impl GemmTiling {Self::Tile64x64x256_Simdgroups2x2 => 2,Self::Tile128x128x256_Simdgroups4x4 => 4,Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 2,- Self::Tile16x32x32_Simdgroups2x1 => 1,+ Self::Tile8x32x64_Simdgroups1x1x4 | Self::Tile16x32x32_Simdgroups2x1 => 1,}}@@ -136,6 +144,7 @@ impl GemmTiling {match self {Self::Tile16x32x256_Simdgroups1x1x4| Self::Tile8x32x64_Simdgroups1x2x4+ | Self::Tile8x32x64_Simdgroups1x1x4| Self::Tile8x32x128_Simdgroups1x2x4 => 4,_ => 1,}@@ -155,7 +164,10 @@ impl GemmTiling {}pub const fn is_small_batch_variant(self) -> bool {- matches!(self, Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4)+ matches!(+ self,+ Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x64_Simdgroups1x1x4 | Self::Tile8x32x128_Simdgroups1x2x4+ )}}diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/matmul.rs b/crates/uzu-engine/src/backends/metal/gpu_types/matmul.rsindex 533ef00c8..b9b767816 100644--- a/crates/uzu-engine/src/backends/metal/gpu_types/matmul.rs+++ b/crates/uzu-engine/src/backends/metal/gpu_types/matmul.rs@@ -1,6 +1,9 @@pub const MXU_TILED_BLOCK_ROWS: u32 = 32;pub const MXU_TILED_W2_BLOCK_ROWS: u32 = 16;pub const MXU_TILED_PREPARE_THREADS: u32 = 1024;+pub const INTERLEAVED_ROWS_STRIP_ROWS: u32 = 32;+pub const INTERLEAVED_ROWS_LANE_ROWS: u32 = 8;+pub const INTERLEAVED_ROWS_RELAYOUT_THREADS: u32 = 1024;#[repr(C)]#[allow(non_snake_case)]diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.hindex 59b475209..5eaf1cb0d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h@@ -35,6 +35,7 @@ enum class GemmAPrologueKind : uint32_t {enum class GemmBLayout : uint32_t {RowMajor = 0,MxuTiled = 1,+ InterleavedRows = 2,};enum class GemmTiling : uint32_t {@@ -52,6 +53,7 @@ enum class GemmTiling : uint32_t {Tile8x32x64_Simdgroups1x2x4 = 11,Tile8x32x128_Simdgroups1x2x4 = 12,Tile16x32x32_Simdgroups2x1 = 13,+ Tile8x32x64_Simdgroups1x1x4 = 14,};static constant constexpr uint32_t MXU_SIMDGROUP_BLOCK_K = 32;diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.h b/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.hindex 33993c836..404fe1f55 100644--- a/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.h+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.h@@ -13,6 +13,12 @@ static constant constexpr uint32_t MXU_TILED_W2_BLOCK_ROWS = 16;static constant constexpr uint32_t MXU_TILED_PREPARE_THREADS = 1024;+static constant constexpr uint32_t INTERLEAVED_ROWS_STRIP_ROWS = 32;++static constant constexpr uint32_t INTERLEAVED_ROWS_LANE_ROWS = 8;++static constant constexpr uint32_t INTERLEAVED_ROWS_RELAYOUT_THREADS = 1024;+typedef struct {uint32_t M;uint32_t N;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.hindex bcc86d4a0..27b728218 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h@@ -28,6 +28,7 @@ METAL_FUNC uint decode_zero_point(const device uint8_t* zero_points_row, uint gr}UZU_CONST ushort W4_BITS = 4;+UZU_CONST uint W4_CODES_PER_WORD = sizeof(uint) * 8 / W4_BITS;UZU_CONST uint W4_SIGN_MASK = symmetric_zero_point<W4_BITS>() * 0x11111111u;UZU_CONST uint W8_SIGN_MASK = symmetric_zero_point<8>() * 0x01010101u;UZU_CONST uint W4_NIBBLE_MASK = 0x0F0F0F0Fu;@@ -90,7 +91,8 @@ inline void dequantize(float scale,float bias,threadgroup StageT* w_local,- const bool signed_codes+ const bool signed_codes,+ const uint word_stride = 1) {static_assert(bits == 2 || bits == 4 || bits == 8, "Only int2, int4 and int8 supported");static_assert(bits == 8 || N % 2 == 0, "int2 and int4 values are unpacked in pairs");@@ -103,7 +105,8 @@ inline void dequantize(if constexpr (READ_CODE_WORDS) {const device uint* words = reinterpret_cast<const device uint*>(w);for (int i = 0; i < N / 8; i++) {- const uint word = signed_codes ? words[i] ^ W4_SIGN_MASK : words[i];+ const uint raw_word = words[i * word_stride];+ const uint word = signed_codes ? raw_word ^ W4_SIGN_MASK : raw_word;for (int j = 0; j < 4; j++) {pairs[i * 4 + j] = vec<StageT, 2>(vec<ComputeT, 2>(dequantize_code((word >> (8 * j)) & 0x0fu), dequantize_code((word >> (8 * j + 4)) & 0x0fu))@@ -118,7 +121,7 @@ inline void dequantize(} else if constexpr (bits == 4) {const uint sign_flip = signed_codes ? 0x88u : 0u;for (int i = 0; i < (N / 2); i++) {- const uint code = uint(w[i]) ^ sign_flip;+ const uint code = uint(w[i / 4 * 4 * word_stride + i % 4]) ^ sign_flip;pairs[i] = vec<StageT, 2>(vec<ComputeT, 2>(dequantize_code(code & 0x0fu), dequantize_code(code >> 4)));}} else {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.hindex 5460dcac1..07353767c 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h@@ -19,6 +19,7 @@ constexpr uint gemm_tiling_block_m(GemmTiling t) {: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 128: t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 8: t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 8+ : t == GemmTiling::Tile8x32x64_Simdgroups1x1x4 ? 8: t == GemmTiling::Tile16x32x32_Simdgroups2x1 ? 16: 0;}@@ -37,6 +38,7 @@ constexpr uint gemm_tiling_block_n(GemmTiling t) {: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 128: t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 32: t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 32+ : t == GemmTiling::Tile8x32x64_Simdgroups1x1x4 ? 32: t == GemmTiling::Tile16x32x32_Simdgroups2x1 ? 32: 0;}@@ -55,6 +57,7 @@ constexpr uint gemm_tiling_block_k(GemmTiling t) {: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 256: t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 64: t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 128+ : t == GemmTiling::Tile8x32x64_Simdgroups1x1x4 ? 64: t == GemmTiling::Tile16x32x32_Simdgroups2x1 ? 32: 0;}@@ -73,6 +76,7 @@ constexpr uint gemm_tiling_simdgroups_per_row(GemmTiling t) {: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 4: t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 1: t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 1+ : t == GemmTiling::Tile8x32x64_Simdgroups1x1x4 ? 1: t == GemmTiling::Tile16x32x32_Simdgroups2x1 ? 2: 0;}@@ -91,6 +95,7 @@ constexpr uint gemm_tiling_simdgroups_per_column(GemmTiling t) {: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 4: t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 2: t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 2+ : t == GemmTiling::Tile8x32x64_Simdgroups1x1x4 ? 1: t == GemmTiling::Tile16x32x32_Simdgroups2x1 ? 1: 0;}@@ -99,6 +104,7 @@ constexpr uint gemm_tiling_simdgroups_per_k(GemmTiling t) {return t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 4: t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 4: t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 4+ : t == GemmTiling::Tile8x32x64_Simdgroups1x1x4 ? 4: 1;}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.hindex ec9d94221..187659cf4 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h@@ -5,6 +5,7 @@#include "../../common/defines.h"#include "../../common/quant_pack.h"+#include "../../layout/interleaved_rows_layout.h"#include "../../common/quant_unpack.h"using namespace metal;@@ -54,12 +55,18 @@ struct QuantizedBlockLoaderScaleBias {const device T* scales;const device T* biases;const bool signed_codes;+ const bool interleaved_rows;++ static METAL_FUNC int source_offset(bool interleaved_rows, int byte) {+ return interleaved_rows ? InterleavedRowsLayout::byte_offset(byte) : byte;+ }QuantizedBlockLoaderScaleBias(const device uint8_t* src_,const device T* scales_,const device T* biases_,const bool signed_codes_,+ const bool interleaved_rows_,const int src_leading_dim_,const int params_group_stride_,const int params_output_stride_,@@ -67,15 +74,22 @@ struct QuantizedBlockLoaderScaleBias {ushort simd_group_id [[simdgroup_index_in_threadgroup]],ushort simd_lane_id [[thread_index_in_simdgroup]])- : src_leading_dim(src_leading_dim_), tile_stride(THREADGROUP_TILE_COLS_PACKED * BYTES_PER_PACK),+ : src_leading_dim(src_leading_dim_),+ tile_stride(source_offset(interleaved_rows_, THREADGROUP_TILE_COLS_PACKED * BYTES_PER_PACK)),group_step_counter(0), group_stride(params_group_stride_), thread_index(simd_group_id * 32 + simd_lane_id),tile_row_index(READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED),tile_col_index((READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED),dst(reinterpret_cast<threadgroup StageT*>(dst_) + tile_row_index * DESTINATION_LEADING_DIMENSION +tile_col_index * PACK_FACTOR),- src(src_ + tile_row_index * src_leading_dim_ * BYTES_PER_PACK / PACK_FACTOR + tile_col_index * BYTES_PER_PACK),+ src(src_ ++ (interleaved_rows_+ ? sizeof(uint) *+ InterleavedRowsLayout::word_index(tile_row_index, 0, src_leading_dim_ / W4_CODES_PER_WORD)+ : tile_row_index * src_leading_dim_ * BYTES_PER_PACK / PACK_FACTOR) ++ source_offset(interleaved_rows_, tile_col_index * BYTES_PER_PACK)),scales(scales_ + tile_row_index * params_output_stride_),- biases(biases_ + tile_row_index * params_output_stride_), signed_codes(signed_codes_) {}+ biases(biases_ + tile_row_index * params_output_stride_), signed_codes(signed_codes_),+ interleaved_rows(interleaved_rows_) {}void load_unsafe() const {if constexpr (TILE_HAS_IDLE_THREADS) {@@ -89,7 +103,8 @@ struct QuantizedBlockLoaderScaleBias {float(*scales),float(*biases),dst,- signed_codes+ signed_codes,+ interleaved_rows ? InterleavedRowsLayout::STRIP_ROWS : 1);}@@ -115,7 +130,7 @@ struct QuantizedBlockLoaderScaleBias {const int pack_index = tile_col_index + i;if (pack_index < valid_packs) {dequantize<StageT, ComputeT, PACK_FACTOR, BITS>(- src + i * BYTES_PER_PACK,+ src + source_offset(interleaved_rows, i * BYTES_PER_PACK),scale,bias,dst + i * PACK_FACTOR,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.hindex 3ab8d9b59..1026259a3 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h@@ -5,6 +5,7 @@#include "../../common/defines.h"#include "../../common/quant_pack.h"+#include "../../layout/interleaved_rows_layout.h"#include "../../common/quant_unpack.h"using namespace metal;@@ -57,12 +58,18 @@ struct QuantizedBlockLoaderScaleZeroPoint {const device uint8_t* zero_points_row_start;uint zero_point_index;const bool signed_codes;+ const bool interleaved_rows;++ static METAL_FUNC int source_offset(bool interleaved_rows, int byte) {+ return interleaved_rows ? InterleavedRowsLayout::byte_offset(byte) : byte;+ }QuantizedBlockLoaderScaleZeroPoint(const device uint8_t* src_,const device T* scales_,const device uint8_t* zero_points_row_start_,const bool signed_codes_,+ const bool interleaved_rows_,const int src_leading_dim_,const int params_group_stride_,const int params_output_stride_,@@ -73,18 +80,24 @@ struct QuantizedBlockLoaderScaleZeroPoint {ushort simd_group_id [[simdgroup_index_in_threadgroup]],ushort simd_lane_id [[thread_index_in_simdgroup]])- : src_leading_dim(src_leading_dim_), tile_stride(THREADGROUP_TILE_COLS_PACKED * BYTES_PER_PACK),+ : src_leading_dim(src_leading_dim_),+ tile_stride(source_offset(interleaved_rows_, THREADGROUP_TILE_COLS_PACKED * BYTES_PER_PACK)),group_step_counter(0), group_stride(params_group_stride_), zero_point_group_stride(zero_point_group_stride_),thread_index(simd_group_id * 32 + simd_lane_id),tile_row_index(READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED),tile_col_index((READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED),dst(reinterpret_cast<threadgroup StageT*>(dst_) + tile_row_index * DESTINATION_LEADING_DIMENSION +tile_col_index * PACK_FACTOR),- src(src_ + tile_row_index * src_leading_dim_ * BYTES_PER_PACK / PACK_FACTOR + tile_col_index * BYTES_PER_PACK),+ src(src_ ++ (interleaved_rows_+ ? sizeof(uint) *+ InterleavedRowsLayout::word_index(tile_row_index, 0, src_leading_dim_ / W4_CODES_PER_WORD)+ : tile_row_index * src_leading_dim_ * BYTES_PER_PACK / PACK_FACTOR) ++ source_offset(interleaved_rows_, tile_col_index * BYTES_PER_PACK)),scales(scales_ + tile_row_index * params_output_stride_),zero_points_row_start(SCALE_SYMMETRIC ? nullptr : zero_points_row_start_),zero_point_index(zero_point_index_ + uint(tile_row_index) * zero_point_output_stride_),- signed_codes(signed_codes_) {}+ signed_codes(signed_codes_), interleaved_rows(interleaved_rows_) {}QuantizedBlockLoaderScaleZeroPoint(const device uint8_t* src_,@@ -102,6 +115,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {scales_,static_cast<const device uint8_t*>(nullptr),signed_codes_,+ false,src_leading_dim_,params_group_stride_,params_output_stride_,@@ -139,7 +153,14 @@ struct QuantizedBlockLoaderScaleZeroPoint {float scale;float bias;current_scale_bias(scale, bias);- dequantize<StageT, ComputeT, READS_PER_THREAD * PACK_FACTOR, BITS>(src, scale, bias, dst, signed_codes);+ dequantize<StageT, ComputeT, READS_PER_THREAD * PACK_FACTOR, BITS>(+ src,+ scale,+ bias,+ dst,+ signed_codes,+ interleaved_rows ? InterleavedRowsLayout::STRIP_ROWS : 1+ );}void load_safe(short2 src_tile_dim) const {@@ -165,7 +186,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {const int pack_index = tile_col_index + i;if (pack_index < valid_packs) {dequantize<StageT, ComputeT, PACK_FACTOR, BITS>(- src + i * BYTES_PER_PACK,+ src + source_offset(interleaved_rows, i * BYTES_PER_PACK),scale,bias,dst + i * PACK_FACTOR,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.hindex 937ee0302..c20ac13e1 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h@@ -7,7 +7,7 @@#include "../../../common/fragment.h"#include "../../../common/quant_pack.h"#include "../../../common/quant_unpack.h"-#include "../mxu_tiled_layout.h"+#include "../../../layout/mxu_tiled_layout.h"#include "../schedules/tile_context.h"using namespace metal;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/packed.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/packed.hindex 70c359ed5..6fb7304fc 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/packed.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/packed.h@@ -4,6 +4,7 @@#include "../../../../../generated/gemm.h"#include "../../../../common/fragment.h"+#include "../../../../layout/interleaved_rows_layout.h"#include "../../../../common/quant_unpack.h"#include "../../../../common/simdgroup_fragment_ops.h"#include "../../../../common/packed_half_layout.h"@@ -55,6 +56,7 @@ struct PackedSchedule {const uint lane_row = uint(tile_context.lane_position.y);const uint weight_sign_mask = right.sign_mask();+ const bool interleaved_rows = right.gemm_b_layout == GemmBLayout::InterleavedRows;const bool row_scaled = right.hoist_row_metadata;typename Core::AccumFragment accumulator;accumulator.clear();@@ -65,6 +67,7 @@ struct PackedSchedule {for (uint k_block = tile_context.k_block_begin + tile_context.simdgroup_k_slice; k_block < k_block_end;k_block += Core::SIMDGROUPS_PER_K) {const uint first_weight_group = k_block * GROUPS_PER_K_BLOCK;+ METAL_PRAGMA_UNROLLfor (ushort group_in_block = 0; group_in_block < GROUPS_PER_K_BLOCK; ++group_in_block) {const uint weight_group_index = first_weight_group + group_in_block;const device uint* packed_group = packed_activations + weight_group_index * WORDS_PER_GROUP;@@ -88,6 +91,19 @@ struct PackedSchedule {float weight_scales[Core::TILES_M];float weight_offsets[Core::TILES_M]; // zero point, or biasuint packed_weight_words[Core::TILES_M * WORDS_PER_LANE];+ uint4 word_runs[WORDS_PER_LANE];+ if (interleaved_rows) {+ METAL_PRAGMA_UNROLL+ for (ushort word_index = 0; word_index < WORDS_PER_LANE; ++word_index) {+ word_runs[word_index] = InterleavedRowsLayout::lane_run(+ right.codes,+ tile_context.simdgroup_column_base + lane_row,+ weight_group_index * (WEIGHT_GROUP_SIZE / W4_CODES_PER_WORD) + fragment_column_pair * WORDS_PER_LANE ++ word_index,+ params->K / W4_CODES_PER_WORD+ );+ }+ }METAL_PRAGMA_UNROLLfor (ushort tile_m = 0; tile_m < Core::TILES_M; ++tile_m) {weight_rows[tile_m] =@@ -98,7 +114,8 @@ struct PackedSchedule {);METAL_PRAGMA_UNROLLfor (ushort word_index = 0; word_index < WORDS_PER_LANE; ++word_index) {- packed_weight_words[tile_m * WORDS_PER_LANE + word_index] = weight_source[word_index] ^ weight_sign_mask;+ const uint word = interleaved_rows ? word_runs[word_index][tile_m] : weight_source[word_index];+ packed_weight_words[tile_m * WORDS_PER_LANE + word_index] = word ^ weight_sign_mask;}if (!row_scaled) {weight_scales[tile_m] = float(right.scale(weight_rows[tile_m], weight_group_index, params));diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.hindex 5d6d97d22..2a49f98cd 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h@@ -33,9 +33,10 @@ static METAL_FUNC auto make_staged_loader(const int params_offset =int(block_col) * int(params->scale_output_stride) + int(first_group) * int(params->scale_group_stride);const device Element* scales = right.scales + params_offset;+ const bool interleaved_rows = right.gemm_b_layout == GemmBLayout::InterleavedRows;+ const int k_bytes = int(k_offset) * get_bytes_per_pack<RightOperand::BITS>() / get_pack_factor<RightOperand::BITS>();const device uint8_t* values = right.codes + size_t(block_col) * row_stride +- size_t(k_offset) * size_t(get_bytes_per_pack<RightOperand::BITS>()) /- size_t(get_pack_factor<RightOperand::BITS>());+ (interleaved_rows ? InterleavedRowsLayout::byte_offset(k_bytes) : k_bytes);if constexpr (RightOperand::SCHEME == GemmBPrologueKind::ScaleBiasDequant) {const device Element* biases = right.bias() + params_offset;using Loader = QuantizedBlockLoaderScaleBias<@@ -53,6 +54,7 @@ static METAL_FUNC auto make_staged_loader(scales,biases,right.signed_codes,+ interleaved_rows,int(params->K),int(params->scale_group_stride),int(params->scale_output_stride),@@ -77,6 +79,7 @@ static METAL_FUNC auto make_staged_loader(scales,right.zp(),right.signed_codes,+ interleaved_rows,int(params->K),int(params->scale_group_stride),int(params->scale_output_stride),diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metalindex 2af9b7c13..b29e37139 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -13,7 +13,8 @@ using namespace uzu::gemm;#define A_IS_INT8 (A_PROLOGUE == GemmAPrologueKind::Int8Symmetric)#define GEMM_SMALL_BATCH \- (GEMM_TILING == GemmTiling::Tile8x32x64_Simdgroups1x2x4 || GEMM_TILING == GemmTiling::Tile8x32x128_Simdgroups1x2x4)+ (GEMM_TILING == GemmTiling::Tile8x32x64_Simdgroups1x2x4 || GEMM_TILING == GemmTiling::Tile8x32x64_Simdgroups1x1x4 || \+ GEMM_TILING == GemmTiling::Tile8x32x128_Simdgroups1x2x4)#define GEMM_SMALL_BATCH_W8 (GEMM_SMALL_BATCH && A_PROLOGUE != GemmAPrologueKind::PackedHalf)#define NEEDS_ASYMMETRIC_WEIGHT_CORRECTION (A_IS_INT8 && B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant)#define GEMM_MXU_QUANT (USE_MXU && B_PROLOGUE != GemmBPrologueKind::FullPrecision && !A_IS_INT8)@@ -64,7 +65,8 @@ VARIANTS(GemmTiling::Tile64x64x256_Simdgroups2x2,GemmTiling::Tile128x128x256_Simdgroups4x4,GemmTiling::Tile8x32x64_Simdgroups1x2x4,- GemmTiling::Tile8x32x128_Simdgroups1x2x4)+ GemmTiling::Tile8x32x128_Simdgroups1x2x4,+ GemmTiling::Tile8x32x64_Simdgroups1x1x4)VARIANTS(TRANSPOSE_B, false, true)VARIANTS(USE_MXU, false, true)VARIANTS(@@ -117,11 +119,14 @@ CONSTRAINT(GEMM_TILING != GemmTiling::Tile16x128x256_Simdgroups1x4 ||(TRANSPOSE_B && B_PROLOGUE == GemmBPrologueKind::FullPrecision))CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || A_PROLOGUE == GemmAPrologueKind::PackedHalf || USE_MXU)-CONSTRAINT(GEMM_TILING != GemmTiling::Tile8x32x64_Simdgroups1x2x4 || A_PROLOGUE == GemmAPrologueKind::PackedHalf)+CONSTRAINT(+ (GEMM_TILING != GemmTiling::Tile8x32x64_Simdgroups1x2x4 && GEMM_TILING != GemmTiling::Tile8x32x64_Simdgroups1x1x4) ||+ A_PROLOGUE == GemmAPrologueKind::PackedHalf)CONSTRAINT(A_PROLOGUE != GemmAPrologueKind::PackedHalf ||(GEMM_SMALL_BATCH && !USE_MXU && TRANSPOSE_B &&- ((GEMM_TILING == GemmTiling::Tile8x32x64_Simdgroups1x2x4 && BITS == 4) ||+ (((GEMM_TILING == GemmTiling::Tile8x32x64_Simdgroups1x2x4 || GEMM_TILING == GemmTiling::Tile8x32x64_Simdgroups1x1x4) &&+ BITS == 4) ||(GEMM_TILING == GemmTiling::Tile8x32x128_Simdgroups1x2x4 && BITS == 2))))CONSTRAINT(!GEMM_SMALL_BATCH_W8 ||diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 71531c122..f713ab311 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -1,12 +1,9 @@use std::collections::{HashMap, hash_map::Entry};-use futures::executor::block_on;-use crate::{backends::{common::{- Backend, BufferMut, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,- Context,+ Backend, BufferMut, BufferRef, CommandBufferEncoding,gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},kernel::matmul::{Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulOutput, MatmulShape, QuantParamsLayout,@@ -20,10 +17,10 @@ use crate::{error::MetalError,gpu_types::{gemm::{GemmAPrologueKind, GemmAlignment, GemmBLayout},- matmul::{GemmParams, MXU_TILED_BLOCK_ROWS, MXU_TILED_PREPARE_THREADS, MXU_TILED_W2_BLOCK_ROWS},+ matmul::GemmParams,},kernel::{- GemmMetalKernel, GemmPrepareBLayoutMetalKernel, GemmSplitKReduceMetalKernel,+ GemmMetalKernel, GemmSplitKReduceMetalKernel,matmul::{MatmulMetalError, MatmulOutputWork,gemm::{@@ -102,43 +99,6 @@ impl GemmKernel {}}- fn mxu_tiled_block_rows(bits: u32) -> u32 {- if bits == 2 {- MXU_TILED_W2_BLOCK_ROWS- } else {- MXU_TILED_BLOCK_ROWS- }- }-- pub fn mxu_tiled_words_per_thread(- k: u32,- bits: u32,- ) -> Option<u32> {- let words_per_thread =- (Self::mxu_tiled_block_rows(bits) * (k * bits / u32::BITS)).div_ceil(MXU_TILED_PREPARE_THREADS);- [32, 72, 160].into_iter().find(|&variant| variant >= words_per_thread)- }-- pub fn prepare_gemm_b_layout(- weights: &mut <Metal as Backend>::GlobalGpuBuffer,- rows: u32,- k: u32,- bits: u32,- context: &MetalContext,- ) -> Result<(), MetalError> {- let row_words = k * bits / u32::BITS;- let variant = Self::mxu_tiled_words_per_thread(k, bits).expect("int8 eligibility admits only tileable weights");- let mut command_buffer = context.create_command_buffer(Some("prepare GEMM B layout"), None)?;- GemmPrepareBLayoutMetalKernel::new(context, bits, variant)?.encode(- weights,- row_words,- rows / Self::mxu_tiled_block_rows(bits),- &mut command_buffer,- );- block_on(command_buffer.end_encoding().submit().wait_until_completed())?;- Ok(())- }-fn get_or_create_split_k_reduce(&mut self,context: &MetalContext,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rsindex 16b6a68fb..5342771c7 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs@@ -64,11 +64,15 @@ pub fn small_batch_format(shape: MatmulShape,layout: GemmBLayout,) -> Option<(ActivationFormat, GemmTiling)> {- let (activation, tiling, k_alignment) = small_m_weights(&shape)?;+ let (activation, row_major_tiling, k_alignment) = small_m_weights(&shape)?;+ let tiling = match layout {+ GemmBLayout::RowMajor => row_major_tiling,+ GemmBLayout::InterleavedRows if shape.b_bits == Some(4) => GemmTiling::Tile8x32x64_Simdgroups1x1x4,+ _ => return None,+ };(shape.b_transpose&& !shape.gathered&& !shape.b_is_trellis- && layout == GemmBLayout::RowMajor&& shape.b_leading_dimension.is_none()&& shape.k.is_multiple_of(k_alignment)&& (!shape.d_transform.contains(GemmDTransform::RHT) || shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE)))@@ -170,6 +174,7 @@ impl GemmProblem {if !splittable|| !self.split_k_output_supported()|| (shape.b_bits == Some(2) && tiling == GemmTiling::Tile8x32x128_Simdgroups1x2x4)+ || tiling == GemmTiling::Tile8x32x64_Simdgroups1x1x4{return 1;}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.hindex 314bb65c5..f38714833 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h@@ -35,7 +35,7 @@ struct Epilogue {}if constexpr (Tile::OUTPUT_ROWS >= METAL_SIMD_SIZE) {- static_assert(Tile::K_SPLIT == 1, "RHT reuses shared results after Reduce");+ static_assert(Tile::K_SPLIT == 1 || Tile::INPUT_ROWS == 1, "RHT reuses shared results after Reduce");threadgroup_barrier(mem_flags::mem_threadgroup);constexpr uint OUTPUT_BLOCKS = Tile::OUTPUT_ROWS / METAL_SIMD_SIZE;for (uint job = tile.simd_group; job < Tile::INPUT_ROWS * OUTPUT_BLOCKS; job += Tile::NUM_SIMDGROUPS) {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.hindex 9bdeefe12..5bd08a31a 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h@@ -42,11 +42,12 @@ struct QuantBSource {const device uint8_t* weights = reinterpret_cast<const device uint8_t*>(ops.b);const uint batch_remaining = params.batch_size - tile.input_row;- uint group = group_slot;+ const uint slice_groups = groups / Tile::K_SPLIT;+ const uint group_end = (tile.k_slice + 1) * slice_groups;Slice current;- QuantPosition position = {group, 0};- while (position.valid(groups)) {+ QuantPosition position = {group_slot + tile.k_slice * slice_groups, 0};+ while (position.valid(group_end)) {Metadata metadata;metadata.load(position.group, weight_row_indices, ops, params);for (position.slice = 0; position.slice < Slice::SLICES_PER_LANE; position.slice++) {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.hindex 56ace9003..25528d695 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h@@ -1,6 +1,7 @@#pragma once-#include "../../gemm/common/mxu_tiled_layout.h"+#include "../../layout/interleaved_rows_layout.h"+#include "../../layout/mxu_tiled_layout.h"#include "arguments.h"#include "quant_chunk.h"#include "quant_metadata.h"@@ -118,6 +119,23 @@ private:});}+ METAL_FUNC void load_interleaved_rows_weights(+ const device uint8_t* weights_base,+ const thread uint (&weight_row_indices)[Tile::ROWS_PER_LANE],+ uint row_words,+ uint first_word+ ) thread {+ const device uint* words = reinterpret_cast<const device uint*>(weights_base);+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {+ constexpr uint R = decltype(output_index)::value;+ METAL_PRAGMA_UNROLL+ for (uint word = 0; word < SLICE_BYTES / sizeof(uint); ++word) {+ reinterpret_cast<thread uint*>(weights[R])[word] =+ words[InterleavedRowsLayout::word_index(weight_row_indices[R], first_word + word, row_words)];+ }+ });+ }+public:static METAL_FUNC uint row_stride(const thread GemvParams& params) {return params.in_vec_size * BITS / QuantChunk<BITS>::BITS_PER_BYTE;@@ -141,6 +159,17 @@ public:}return;}+ if constexpr (BITS == 4) {+ if (params.gemm_b_layout == GemmBLayout::InterleavedRows) {+ load_interleaved_rows_weights(+ weights_base,+ weight_row_indices,+ params.in_vec_size / W4_CODES_PER_WORD,+ k / W4_CODES_PER_WORD+ );+ return;+ }+ }Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {constexpr uint R = decltype(output_index)::value;const uint row = weight_row_indices[R];diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metalindex 4293081e5..65050b21f 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -20,6 +20,9 @@ using namespace uzu::gemm;(K_SPLIT == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE == 1 && \((BITS == 2 && GROUP_SIZE == 32 * GROUP_LANES) || (BITS == 4 && GROUP_SIZE == 16 * GROUP_LANES) || \(BITS == 8 && GROUP_SIZE == 8 * GROUP_LANES)))+#define GEMV_INTERLEAVED_ROWS_TILE \+ (K_SPLIT == 8 && REDUCTION_LANES == 4 && NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE == 1 && \+ BITS == 4 && GROUP_SIZE == 16 * GROUP_LANES)#define GEMV_BATCHED_FMA_32_TILE \(K_SPLIT == 1 && REDUCTION_LANES == 8 && NUM_SIMDGROUPS == 4 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE >= 2 && \INPUT_ROW_TILE <= 4)@@ -57,7 +60,7 @@ VARIANTS(K_SPLIT, 1, 2, 4, 8)VARIANTS(INPUT_ALIGNED, false, true)VARIANTS(INPUT_ROW_TILE, 1, 2, 3, 4)VARIANTS(OUTPUT_ROW_TILE, 1, 2, 4, 8, 16, 32)-VARIANTS(REDUCTION_LANES, 8, 32)+VARIANTS(REDUCTION_LANES, 4, 8, 32)VARIANTS(GROUP_LANES, 1, 2, 4, 8)VARIANTS(NUM_SIMDGROUPS, 2, 4, 8)@@ -72,7 +75,8 @@ CONSTRAINT((BITS == 2) == (GROUP_SIZE == 128))CONSTRAINT(INPUT_ROW_TILE == 1 || GEMV_BATCHED_INPUT)CONSTRAINT(GEMV_FULL_PRECISION_TILE ||- (BITS != 0 && (GEMV_QUANTIZED_ROW_TILE || GEMV_BATCHED_FMA_32_TILE || GEMV_BATCHED_FMA_16_TILE)))+ (BITS != 0 && (GEMV_QUANTIZED_ROW_TILE || GEMV_INTERLEAVED_ROWS_TILE || GEMV_BATCHED_FMA_32_TILE ||+ GEMV_BATCHED_FMA_16_TILE)))KERNEL(Gemv)(const device uint32_t* b,const device BT* scalesdiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 985d5fe46..ede302e1b 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -69,7 +69,9 @@ impl GemvSpecialization {&& weights_data_type == DataType::BF16&& shape.k >= MXU_TILED_KERNEL_MIN_K&& shape.n >= 32 * gpu_core_count;- let tile = if is_quant && shape.gathered {+ let tile = if is_quant && !shape.gathered && gemm_b_layout == GemmBLayout::InterleavedRows && shape.m == 1 {+ policy::interleaved_rows_tile(shape.b_group_size.unwrap_or(0), shape.k)+ } else if is_quant && shape.gathered {policy::gathered_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n)} else if is_quant {policy::quantized_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n, bf16_io)@@ -191,7 +193,7 @@ fn output_transform_for_tile(}let output_rows = tile.output_row_tile();- let can_fuse = tile.k_split == 1+ let can_fuse = (tile.k_split == 1 || (shape.is_quant() && tile.input_row_tile == 1))&& output_rows >= HADAMARD_TRANSFORM_BLOCK_SIZE&& output_rows.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE);if can_fuse {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rsindex ce64b7eb8..e2240c977 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs@@ -1,6 +1,6 @@mod quantized;-pub(super) use quantized::{gathered_tile, select as quantized_tile};+pub(super) use quantized::{gathered_tile, interleaved_rows_tile, select as quantized_tile};// Full-precision GEMV accumulates four K values per SIMD lane, so one full// vectorized K block is 4 * 32 lanes.diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rsindex 7d790efc1..7989b5f54 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs@@ -45,3 +45,17 @@ pub fn select(}gathered_tile(bits, group, m, n)}++pub fn interleaved_rows_tile(+ group: u32,+ k: u32,+) -> Option<GemvTile> {+ const VALUES_PER_LANE: u32 = 16;+ let strip_tile = GemvTile::quantized(8, 32, 1, 4, group / VALUES_PER_LANE);+ let tile = GemvTile {+ k_split: 8,+ ..strip_tile+ };+ let k_per_step = tile.k_split * tile.reduction_lanes * VALUES_PER_LANE;+ (matches!(group, 32 | 64) && k.is_multiple_of(k_per_step)).then_some(tile)+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/gemm_prepare_b_layout.metalsimilarity index 98%rename from crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metalrename to crates/uzu-engine/src/backends/metal/kernel/matmul/layout/gemm_prepare_b_layout.metalindex 0402a38e2..5d40b8445 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/gemm_prepare_b_layout.metal@@ -1,6 +1,6 @@#include "../../common/dsl.h"#include "../../common/defines.h"-#include "common/mxu_tiled_layout.h"+#include "mxu_tiled_layout.h"using namespace metal;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/interleaved_rows.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/interleaved_rows.rsnew file mode 100644index 000000000..62ef2fad0--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/interleaved_rows.rs@@ -0,0 +1,58 @@+use futures::executor::block_on;++use crate::backends::{+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,+ kernel::matmul::{MatmulShape, QuantParamsLayout},+ },+ metal::{+ Metal,+ context::MetalContext,+ error::MetalError,+ gpu_types::{+ gemm::GemmBLayout,+ matmul::{INTERLEAVED_ROWS_RELAYOUT_THREADS, INTERLEAVED_ROWS_STRIP_ROWS},+ },+ kernel::{+ PrepareInterleavedRowsMetalKernel,+ matmul::{MatmulMetalKernel, gemm::small_batch_format},+ },+ },+};++const CODE_BITS: u32 = 4;++fn words_per_thread(row_words: u32) -> Option<u32> {+ let words_per_thread = (INTERLEAVED_ROWS_STRIP_ROWS * row_words).div_ceil(INTERLEAVED_ROWS_RELAYOUT_THREADS);+ [16, 48, 96].into_iter().find(|&variant| variant >= words_per_thread)+}++/// Relays W4 weights read by the packed small-batch GEMM in place and returns true, or returns false if they are not.+pub fn prepare(+ kernel: &MatmulMetalKernel,+ shape: &MatmulShape,+ weights: &mut <Metal as Backend>::GlobalGpuBuffer,+ context: &MetalContext,+) -> Result<bool, MetalError> {+ let eligible = !context.supports_mxu+ && kernel.is_all_bf16()+ && shape.params_layout == Some(QuantParamsLayout::GroupOutput)+ && shape.n.is_multiple_of(INTERLEAVED_ROWS_STRIP_ROWS)+ && small_batch_format(*shape, GemmBLayout::InterleavedRows).is_some();+ let row_words = shape.k * CODE_BITS / u32::BITS;+ if !eligible {+ return Ok(false);+ }+ let Some(variant) = words_per_thread(row_words) else {+ return Ok(false);+ };+ let mut command_buffer = context.create_command_buffer(Some("prepare interleaved-row weights"), None)?;+ PrepareInterleavedRowsMetalKernel::new(context, variant)?.encode(+ weights,+ row_words,+ shape.n / INTERLEAVED_ROWS_STRIP_ROWS,+ &mut command_buffer,+ );+ block_on(command_buffer.end_encoding().submit().wait_until_completed())?;+ Ok(true)+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/interleaved_rows_layout.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/interleaved_rows_layout.hnew file mode 100644index 000000000..c329e7ac4--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/interleaved_rows_layout.h@@ -0,0 +1,38 @@+#pragma once++#include "../../common/defines.h"+#include "../../generated/matmul.h"++namespace uzu {+namespace gemm {++// 32-row strips, word j of rows ordered 0, 8, 16, 24 | 1, 9, 17, 25 | ... so a GEMM lane's 4 rows are one uint4.+struct InterleavedRowsLayout {+ UZU_CONST uint STRIP_ROWS = uzu::matmul::INTERLEAVED_ROWS_STRIP_ROWS;+ UZU_CONST uint LANE_ROWS = uzu::matmul::INTERLEAVED_ROWS_LANE_ROWS;+ UZU_CONST uint ROWS_PER_LANE = STRIP_ROWS / LANE_ROWS;++ static METAL_FUNC size_t word_index(uint row, uint word, uint row_words) {+ return size_t(row / STRIP_ROWS) * STRIP_ROWS * row_words + word * STRIP_ROWS + row % LANE_ROWS * ROWS_PER_LANE ++ row / LANE_ROWS % ROWS_PER_LANE;+ }++ static METAL_FUNC uint row_major_word_in_strip(uint index, uint row_words) {+ const uint slot = index % STRIP_ROWS;+ return (slot % ROWS_PER_LANE * LANE_ROWS + slot / ROWS_PER_LANE) * row_words + index / STRIP_ROWS;+ }++ static METAL_FUNC int byte_offset(int byte) {+ const int word_bytes = sizeof(uint);+ return byte / word_bytes * int(STRIP_ROWS) * word_bytes + byte % word_bytes;+ }++ static METAL_FUNC uint4 lane_run(const device uint8_t* codes, uint row, uint word, uint row_words) {+ return *reinterpret_cast<const device uint4*>(+ reinterpret_cast<const device uint*>(codes) + word_index(row, word, row_words)+ );+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mod.rsnew file mode 100644index 000000000..3865b1c04--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mod.rs@@ -0,0 +1,29 @@+mod interleaved_rows;+mod mxu_tiled;++pub use mxu_tiled::words_per_thread as mxu_tiled_words_per_thread;++use crate::backends::{+ common::{Backend, kernel::matmul::MatmulShape},+ metal::{+ Metal, context::MetalContext, error::MetalError, gpu_types::gemm::GemmBLayout,+ kernel::matmul::MatmulMetalKernel,+ },+};++/// Picks the layout the weights are read in, relays them in place and returns it with their code signedness.+pub fn prepare(+ kernel: &MatmulMetalKernel,+ shape: &MatmulShape,+ weights: &mut <Metal as Backend>::GlobalGpuBuffer,+ context: &MetalContext,+) -> Result<(GemmBLayout, bool), MetalError> {+ if mxu_tiled::eligible(kernel, shape, context) {+ return Ok((GemmBLayout::MxuTiled, mxu_tiled::prepare(shape, weights, context)?));+ }+ if interleaved_rows::prepare(kernel, shape, weights, context)? {+ Ok((GemmBLayout::InterleavedRows, shape.signed_codes))+ } else {+ Ok((GemmBLayout::RowMajor, shape.signed_codes))+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mxu_tiled.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mxu_tiled.rsnew file mode 100644index 000000000..7c5b1eb89--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mxu_tiled.rs@@ -0,0 +1,78 @@+use futures::executor::block_on;++use crate::backends::{+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, CpuBuffer,+ kernel::matmul::{MatmulKernel, MatmulShape},+ },+ metal::{+ Metal,+ context::MetalContext,+ error::MetalError,+ gpu_types::matmul::{MXU_TILED_BLOCK_ROWS, MXU_TILED_PREPARE_THREADS, MXU_TILED_W2_BLOCK_ROWS},+ kernel::{GemmPrepareBLayoutMetalKernel, matmul::MatmulMetalKernel},+ },+};++fn block_rows(bits: u32) -> u32 {+ if bits == 2 {+ MXU_TILED_W2_BLOCK_ROWS+ } else {+ MXU_TILED_BLOCK_ROWS+ }+}++pub fn words_per_thread(+ k: u32,+ bits: u32,+) -> Option<u32> {+ let words_per_thread = (block_rows(bits) * (k * bits / u32::BITS)).div_ceil(MXU_TILED_PREPARE_THREADS);+ [32, 72, 160].into_iter().find(|&variant| variant >= words_per_thread)+}++fn prepared_shape(shape: &MatmulShape) -> MatmulShape {+ MatmulShape {+ signed_codes: shape.signed_codes || shape.b_bits == Some(8),+ ..*shape+ }+}++pub fn eligible(+ kernel: &MatmulMetalKernel,+ shape: &MatmulShape,+ context: &MetalContext,+) -> bool {+ kernel.select_activation_quantization(&prepared_shape(shape), context).is_some()+}++/// Relays the weights in place and returns their code signedness.+pub fn prepare(+ shape: &MatmulShape,+ weights: &mut <Metal as Backend>::GlobalGpuBuffer,+ context: &MetalContext,+) -> Result<bool, MetalError> {+ let prepared_shape = prepared_shape(shape);+ if !shape.signed_codes && prepared_shape.signed_codes {+ let bytes = unsafe {+ std::slice::from_raw_parts_mut(+ weights.cpu_ptr().as_ptr().cast::<u8>(),+ crate::backends::common::Buffer::size(weights),+ )+ };+ let (prefix, words, suffix) = bytemuck::pod_align_to_mut::<u8, u64>(bytes);+ words.iter_mut().for_each(|word| *word ^= 0x8080_8080_8080_8080);+ prefix.iter_mut().chain(suffix.iter_mut()).for_each(|code| *code ^= 0x80);+ }+ let bits = shape.b_bits.expect("quantized weights");+ let row_words = shape.k * bits / u32::BITS;+ let variant = words_per_thread(shape.k, bits).expect("int8 eligibility admits only tileable weights");+ let mut command_buffer = context.create_command_buffer(Some("prepare GEMM B layout"), None)?;+ GemmPrepareBLayoutMetalKernel::new(context, bits, variant)?.encode(+ weights,+ row_words,+ shape.n / block_rows(bits),+ &mut command_buffer,+ );+ block_on(command_buffer.end_encoding().submit().wait_until_completed())?;+ Ok(prepared_shape.signed_codes)+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_tiled_layout.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mxu_tiled_layout.hsimilarity index 85%rename from crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_tiled_layout.hrename to crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mxu_tiled_layout.hindex 82c84900d..6783bdf73 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_tiled_layout.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mxu_tiled_layout.h@@ -1,8 +1,8 @@#pragma once-#include "../../../common/defines.h"-#include "../../../generated/matmul.h"-#include "../../common/mxu_fragment/ops.h"+#include "../../common/defines.h"+#include "../../generated/matmul.h"+#include "../common/mxu_fragment/ops.h"namespace uzu {namespace gemm {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/prepare_interleaved_rows.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/prepare_interleaved_rows.metalnew file mode 100644index 000000000..a07fab759--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/prepare_interleaved_rows.metal@@ -0,0 +1,39 @@+#include "../../common/dsl.h"+#include "../../common/defines.h"+#include "../../generated/matmul.h"+#include "interleaved_rows_layout.h"++using namespace metal;++using uzu::gemm::InterleavedRowsLayout;+using uzu::matmul::INTERLEAVED_ROWS_RELAYOUT_THREADS;++// Relays a strip's row-major W4 codes into InterleavedRowsLayout in place, through registers.+template <uint WORDS_PER_THREAD>+VARIANTS(WORDS_PER_THREAD, 16, 48, 96)+KERNEL(PrepareInterleavedRows)(+ device uint* weights,+ const constant uint& row_words,+ const constant uint& strip_count,+ const uint strip GROUPS(strip_count),+ const uint thread_index THREADS(INTERLEAVED_ROWS_RELAYOUT_THREADS)+) {+ const uint strip_words = InterleavedRowsLayout::STRIP_ROWS * row_words;+ device uint* codes = weights + size_t(strip) * strip_words;+ uint words[WORDS_PER_THREAD];+ METAL_PRAGMA_UNROLL+ for (uint slot = 0; slot < WORDS_PER_THREAD; ++slot) {+ const uint index = slot * INTERLEAVED_ROWS_RELAYOUT_THREADS + thread_index;+ if (index < strip_words) {+ words[slot] = codes[InterleavedRowsLayout::row_major_word_in_strip(index, row_words)];+ }+ }+ threadgroup_barrier(mem_flags::mem_device);+ METAL_PRAGMA_UNROLL+ for (uint slot = 0; slot < WORDS_PER_THREAD; ++slot) {+ const uint index = slot * INTERLEAVED_ROWS_RELAYOUT_THREADS + thread_index;+ if (index < strip_words) {+ codes[index] = words[slot];+ }+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rsindex 7a4604178..cba7c0612 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs@@ -1,7 +1,7 @@use crate::{backends::{common::{- Backend, BufferMut, BufferRef, CommandBufferEncoding, CpuBuffer,+ Backend, BufferMut, BufferRef, CommandBufferEncoding,kernel::{ActivationQuantization,activation_transform::ACTIVATION_SCALE_GROUP_SIZE,@@ -20,6 +20,7 @@ use crate::{GemmKernel, MatmulMetalError,gemm::{self, GemmPlan, GemmProblem, small_batch_format, small_m_weights},gemv::{GemvKernel, GemvSpecialization, GemvTile},+ layout,matmul_dispatch::MatmulDispatch,matmul_output_work::MatmulOutputWork,},@@ -42,6 +43,10 @@ pub fn supports_integer_right_operand(shape: &MatmulShape) -> bool {}impl MatmulMetalKernel {+ pub fn is_all_bf16(&self) -> bool {+ all_bf16(self.weights_data_type, self.input_data_type, self.output_data_type)+ }+fn prefer_gemm_over_gemv(shape: MatmulShape,plan: GemmPlan,@@ -204,7 +209,7 @@ impl MatmulKernel for MatmulMetalKernel {|| !shape.k.is_multiple_of(ACTIVATION_SCALE_GROUP_SIZE)|| !shape.k.is_multiple_of(weight_group_size)|| !shape.n.is_multiple_of(32)- || shape.b_bits.and_then(|bits| GemmKernel::mxu_tiled_words_per_thread(shape.k, bits)).is_none()+ || shape.b_bits.and_then(|bits| layout::mxu_tiled_words_per_thread(shape.k, bits)).is_none(){return None;}@@ -267,32 +272,7 @@ impl MatmulKernel for MatmulMetalKernel {weights: &mut <Metal as Backend>::GlobalGpuBuffer,context: &MetalContext,) -> Result<(GemmBLayout, bool), MetalError> {- let prepared_shape = MatmulShape {- signed_codes: a8_shape.signed_codes || a8_shape.b_bits == Some(8),- ..*a8_shape- };- if self.select_activation_quantization(&prepared_shape, context).is_none() {- return Ok((GemmBLayout::RowMajor, a8_shape.signed_codes));- }- if !a8_shape.signed_codes && prepared_shape.signed_codes {- let bytes = unsafe {- std::slice::from_raw_parts_mut(- weights.cpu_ptr().as_ptr().cast::<u8>(),- crate::backends::common::Buffer::size(weights),- )- };- let (prefix, words, suffix) = bytemuck::pod_align_to_mut::<u8, u64>(bytes);- words.iter_mut().for_each(|word| *word ^= 0x8080_8080_8080_8080);- prefix.iter_mut().chain(suffix.iter_mut()).for_each(|code| *code ^= 0x80);- }- GemmKernel::prepare_gemm_b_layout(- weights,- a8_shape.n,- a8_shape.k,- a8_shape.b_bits.expect("quantized weights"),- context,- )?;- Ok((GemmBLayout::MxuTiled, prepared_shape.signed_codes))+ layout::prepare(self, a8_shape, weights, context)}fn encode(diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rsindex 58e6b5ce8..88210c211 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs@@ -1,6 +1,7 @@mod error;pub mod gemm;mod gemv;+mod layout;mod matmul_dispatch;mod matmul_metal_kernel;mod matmul_output_work;diff --git a/crates/uzu-engine/src/backends/metal/kernel/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/mod.rsindex 50d836c38..06f0bcb26 100644--- a/crates/uzu-engine/src/backends/metal/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/mod.rs@@ -5,7 +5,7 @@ use crate::backends::{buffer::MetalBufferExt,gpu_types::{gemm::{gemm_tiling_simdgroups_per_column, gemm_tiling_simdgroups_per_k, gemm_tiling_simdgroups_per_row},- matmul::MXU_TILED_PREPARE_THREADS,+ matmul::{INTERLEAVED_ROWS_RELAYOUT_THREADS, MXU_TILED_PREPARE_THREADS},},},};diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/interleaved_rows_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/interleaved_rows_test.rsnew file mode 100644index 000000000..aa07bc456--- /dev/null+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/interleaved_rows_test.rs@@ -0,0 +1,105 @@+//! `GemmBLayout::InterleavedRows`: every kernel that reads interleaved-row W4 strips matches the row-major layout.+use QuantizationMethod::{ScaleBias, ScaleZeroPoint};+use futures::executor::block_on;+use half::bf16;+use uzu_engine_macros::uzu_test;++use crate::{+ backends::{+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,+ gpu_types::quantization_method::QuantizationMethod,+ kernel::{+ ActivationTransform,+ matmul::{ActivationFormat, MatmulA, MatmulKernel, MatmulShape, packed_half_bytes},+ },+ },+ metal::{Metal, context::MetalContext, gpu_types::gemm::GemmBLayout, kernel::matmul::MatmulMetalKernel},+ },+ data_type::DataType,+ tests::{+ assert::assert_eq_float,+ helpers::{buffer_to_vec, create_buffer, create_buffer_with_data},+ matmul::{QuantBuffers, QuantInput, quant_arguments},+ },+};++fn run(+ context: &MetalContext,+ input: &QuantInput<bf16>,+ interleaved_rows: bool,+ gather_indices: Option<&<Metal as Backend>::GlobalGpuBuffer>,+ n: u32,+) -> Vec<bf16> {+ let mut matmul = MatmulMetalKernel::new(context, DataType::BF16, DataType::BF16, DataType::BF16).expect("matmul");+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(context, input);+ if interleaved_rows {+ let shape = MatmulShape::from_arguments(&quant_arguments(&mut buffers, input));+ buffers.weights_layout = matmul.optimize_weights_layout(&shape, &mut buffers.w, context).expect("relayout").0;+ assert_eq!(buffers.weights_layout, GemmBLayout::InterleavedRows);+ }+ let format = ActivationFormat::PackedHalf {+ block_k: input.group_size,+ bits: 4,+ };+ let packed = gather_indices.is_none() && (4..=8).contains(&input.m);+ let mut packed_activations =+ create_buffer::<Metal, u32>(context, packed_half_bytes(input.k, input.group_size, 4) / size_of::<u32>());+ let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");+ if packed {+ let factors = create_buffer_with_data::<Metal, i32>(context, &vec![1; input.k as usize]);+ ActivationTransform::<Metal>::quantize(context, DataType::BF16, format).expect("pack").encode_pack_half(+ &buffers.x,+ &mut packed_activations,+ &factors,+ input.m,+ input.k,+ &mut command_buffer,+ );+ }+ let mut arguments = quant_arguments(&mut buffers, input);+ if packed {+ arguments.a = MatmulA::PackedHalf {+ values: &packed_activations,+ block_k: input.group_size,+ bits: 4,+ };+ }+ arguments.gather_indices = gather_indices;+ arguments.n = n;+ matmul.encode(arguments, &mut command_buffer).expect("matmul encode");+ block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();+ buffer_to_vec::<Metal, bf16>(context, &buffers.y)[..(input.m * n) as usize].to_vec()+}++fn assert_close(+ label: &str,+ expected: &[bf16],+ actual: &[bf16],+) {+ let rms = (expected.iter().map(|v| v.to_f32().powi(2)).sum::<f32>() / expected.len() as f32).sqrt();+ assert_eq_float(expected, actual, 2e-2 * rms, label);+}++#[uzu_test]+fn interleaved_rows_matches_row_major() {+ let context = block_on(MetalContext::new()).expect("Metal context");+ if context.supports_mxu {+ return;+ }+ for (k, group_size, method) in [(1024, 32, ScaleZeroPoint), (16384, 64, ScaleBias)] {+ for m in [1, 3, 8, 16, 64] {+ let input = QuantInput::<bf16>::new(m, k, 256, group_size, 4, method, 7).with_group_output();+ let label = format!("m={m} k={k} G{group_size} {method:?}");+ assert_close(&label, &run(&context, &input, false, None, 256), &run(&context, &input, true, None, 256));+ }+ let input = QuantInput::<bf16>::new(3, k, 256, group_size, 4, method, 11).with_group_output();+ let ids = create_buffer_with_data::<Metal, u32>(&context, &(0..120).map(|i| i * 7 % 256).collect::<Vec<u32>>());+ let label = format!("gathered k={k} G{group_size} {method:?}");+ assert_close(+ &label,+ &run(&context, &input, false, Some(&ids), 40),+ &run(&context, &input, true, Some(&ids), 40),+ );+ }+}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/mod.rsindex d33e91f1b..5835d8e17 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/mod.rs@@ -1,6 +1,7 @@mod a8w_bench;mod dispatch_paths_test;mod gemm_bench;+mod interleaved_rows_test;mod quant_dispatch_test;mod quant_gemm_bench;mod qwen3_bench;