Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

interleaved rows w4 layout

CCYeh48d8469b327a
diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
index 39e94d78c..0469af9dc 100644
--- a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
+++ b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
@@ -15,6 +15,7 @@ pub enum GemmBLayout {
#[default]
RowMajor,
MxuTiled,
+ InterleavedRows,
}
#[repr(C)]
@@ -35,6 +36,7 @@ pub enum GemmTiling {
Tile8x32x64_Simdgroups1x2x4,
Tile8x32x128_Simdgroups1x2x4,
Tile16x32x32_Simdgroups2x1,
+ Tile8x32x64_Simdgroups1x1x4,
}
pub const MXU_SIMDGROUP_BLOCK_K: u32 = 32;
@@ -54,7 +56,9 @@ impl GemmTiling {
Self::Tile64x32x256_Simdgroups4x1 => 64,
Self::Tile64x64x256_Simdgroups2x2 => 64,
Self::Tile128x128x256_Simdgroups4x4 => 128,
- Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 8,
+ Self::Tile8x32x64_Simdgroups1x2x4
+ | Self::Tile8x32x64_Simdgroups1x1x4
+ | Self::Tile8x32x128_Simdgroups1x2x4 => 8,
Self::Tile16x32x32_Simdgroups2x1 => 16,
}
}
@@ -72,7 +76,9 @@ impl GemmTiling {
Self::Tile64x32x256_Simdgroups4x1 => 32,
Self::Tile64x64x256_Simdgroups2x2 => 64,
Self::Tile128x128x256_Simdgroups4x4 => 128,
- Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 32,
+ Self::Tile8x32x64_Simdgroups1x2x4
+ | Self::Tile8x32x64_Simdgroups1x1x4
+ | Self::Tile8x32x128_Simdgroups1x2x4 => 32,
Self::Tile16x32x32_Simdgroups2x1 => 32,
}
}
@@ -90,7 +96,7 @@ impl GemmTiling {
Self::Tile64x32x256_Simdgroups4x1 => 256,
Self::Tile64x64x256_Simdgroups2x2 => 256,
Self::Tile128x128x256_Simdgroups4x4 => 256,
- Self::Tile8x32x64_Simdgroups1x2x4 => 64,
+ Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x64_Simdgroups1x1x4 => 64,
Self::Tile8x32x128_Simdgroups1x2x4 => 128,
Self::Tile16x32x32_Simdgroups2x1 => 32,
}
@@ -109,7 +115,9 @@ impl GemmTiling {
Self::Tile64x32x256_Simdgroups4x1 => 4,
Self::Tile64x64x256_Simdgroups2x2 => 2,
Self::Tile128x128x256_Simdgroups4x4 => 4,
- Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 1,
+ Self::Tile8x32x64_Simdgroups1x2x4
+ | Self::Tile8x32x64_Simdgroups1x1x4
+ | Self::Tile8x32x128_Simdgroups1x2x4 => 1,
Self::Tile16x32x32_Simdgroups2x1 => 2,
}
}
@@ -128,7 +136,7 @@ impl GemmTiling {
Self::Tile64x64x256_Simdgroups2x2 => 2,
Self::Tile128x128x256_Simdgroups4x4 => 4,
Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 2,
- Self::Tile16x32x32_Simdgroups2x1 => 1,
+ Self::Tile8x32x64_Simdgroups1x1x4 | Self::Tile16x32x32_Simdgroups2x1 => 1,
}
}
@@ -136,6 +144,7 @@ impl GemmTiling {
match self {
Self::Tile16x32x256_Simdgroups1x1x4
| Self::Tile8x32x64_Simdgroups1x2x4
+ | Self::Tile8x32x64_Simdgroups1x1x4
| Self::Tile8x32x128_Simdgroups1x2x4 => 4,
_ => 1,
}
@@ -155,7 +164,10 @@ impl GemmTiling {
}
pub const fn is_small_batch_variant(self) -> bool {
- matches!(self, Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4)
+ matches!(
+ self,
+ Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x64_Simdgroups1x1x4 | Self::Tile8x32x128_Simdgroups1x2x4
+ )
}
}
diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/matmul.rs b/crates/uzu-engine/src/backends/metal/gpu_types/matmul.rs
index 533ef00c8..b9b767816 100644
--- a/crates/uzu-engine/src/backends/metal/gpu_types/matmul.rs
+++ b/crates/uzu-engine/src/backends/metal/gpu_types/matmul.rs
@@ -1,6 +1,9 @@
pub const MXU_TILED_BLOCK_ROWS: u32 = 32;
pub const MXU_TILED_W2_BLOCK_ROWS: u32 = 16;
pub const MXU_TILED_PREPARE_THREADS: u32 = 1024;
+pub const INTERLEAVED_ROWS_STRIP_ROWS: u32 = 32;
+pub const INTERLEAVED_ROWS_LANE_ROWS: u32 = 8;
+pub const INTERLEAVED_ROWS_RELAYOUT_THREADS: u32 = 1024;
#[repr(C)]
#[allow(non_snake_case)]
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
index 59b475209..5eaf1cb0d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
@@ -35,6 +35,7 @@ enum class GemmAPrologueKind : uint32_t {
enum class GemmBLayout : uint32_t {
RowMajor = 0,
MxuTiled = 1,
+ InterleavedRows = 2,
};
enum class GemmTiling : uint32_t {
@@ -52,6 +53,7 @@ enum class GemmTiling : uint32_t {
Tile8x32x64_Simdgroups1x2x4 = 11,
Tile8x32x128_Simdgroups1x2x4 = 12,
Tile16x32x32_Simdgroups2x1 = 13,
+ Tile8x32x64_Simdgroups1x1x4 = 14,
};
static constant constexpr uint32_t MXU_SIMDGROUP_BLOCK_K = 32;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.h b/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.h
index 33993c836..404fe1f55 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.h
@@ -13,6 +13,12 @@ static constant constexpr uint32_t MXU_TILED_W2_BLOCK_ROWS = 16;
static constant constexpr uint32_t MXU_TILED_PREPARE_THREADS = 1024;
+static constant constexpr uint32_t INTERLEAVED_ROWS_STRIP_ROWS = 32;
+
+static constant constexpr uint32_t INTERLEAVED_ROWS_LANE_ROWS = 8;
+
+static constant constexpr uint32_t INTERLEAVED_ROWS_RELAYOUT_THREADS = 1024;
+
typedef struct {
uint32_t M;
uint32_t N;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
index bcc86d4a0..27b728218 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
@@ -28,6 +28,7 @@ METAL_FUNC uint decode_zero_point(const device uint8_t* zero_points_row, uint gr
}
UZU_CONST ushort W4_BITS = 4;
+UZU_CONST uint W4_CODES_PER_WORD = sizeof(uint) * 8 / W4_BITS;
UZU_CONST uint W4_SIGN_MASK = symmetric_zero_point<W4_BITS>() * 0x11111111u;
UZU_CONST uint W8_SIGN_MASK = symmetric_zero_point<8>() * 0x01010101u;
UZU_CONST uint W4_NIBBLE_MASK = 0x0F0F0F0Fu;
@@ -90,7 +91,8 @@ inline void dequantize(
float scale,
float bias,
threadgroup StageT* w_local,
- const bool signed_codes
+ const bool signed_codes,
+ const uint word_stride = 1
) {
static_assert(bits == 2 || bits == 4 || bits == 8, "Only int2, int4 and int8 supported");
static_assert(bits == 8 || N % 2 == 0, "int2 and int4 values are unpacked in pairs");
@@ -103,7 +105,8 @@ inline void dequantize(
if constexpr (READ_CODE_WORDS) {
const device uint* words = reinterpret_cast<const device uint*>(w);
for (int i = 0; i < N / 8; i++) {
- const uint word = signed_codes ? words[i] ^ W4_SIGN_MASK : words[i];
+ const uint raw_word = words[i * word_stride];
+ const uint word = signed_codes ? raw_word ^ W4_SIGN_MASK : raw_word;
for (int j = 0; j < 4; j++) {
pairs[i * 4 + j] = vec<StageT, 2>(
vec<ComputeT, 2>(dequantize_code((word >> (8 * j)) & 0x0fu), dequantize_code((word >> (8 * j + 4)) & 0x0fu))
@@ -118,7 +121,7 @@ inline void dequantize(
} else if constexpr (bits == 4) {
const uint sign_flip = signed_codes ? 0x88u : 0u;
for (int i = 0; i < (N / 2); i++) {
- const uint code = uint(w[i]) ^ sign_flip;
+ const uint code = uint(w[i / 4 * 4 * word_stride + i % 4]) ^ sign_flip;
pairs[i] = vec<StageT, 2>(vec<ComputeT, 2>(dequantize_code(code & 0x0fu), dequantize_code(code >> 4)));
}
} else {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
index 5460dcac1..07353767c 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
@@ -19,6 +19,7 @@ constexpr uint gemm_tiling_block_m(GemmTiling t) {
: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 128
: t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 8
: t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 8
+ : t == GemmTiling::Tile8x32x64_Simdgroups1x1x4 ? 8
: t == GemmTiling::Tile16x32x32_Simdgroups2x1 ? 16
: 0;
}
@@ -37,6 +38,7 @@ constexpr uint gemm_tiling_block_n(GemmTiling t) {
: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 128
: t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 32
: t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 32
+ : t == GemmTiling::Tile8x32x64_Simdgroups1x1x4 ? 32
: t == GemmTiling::Tile16x32x32_Simdgroups2x1 ? 32
: 0;
}
@@ -55,6 +57,7 @@ constexpr uint gemm_tiling_block_k(GemmTiling t) {
: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 256
: t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 64
: t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 128
+ : t == GemmTiling::Tile8x32x64_Simdgroups1x1x4 ? 64
: t == GemmTiling::Tile16x32x32_Simdgroups2x1 ? 32
: 0;
}
@@ -73,6 +76,7 @@ constexpr uint gemm_tiling_simdgroups_per_row(GemmTiling t) {
: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 4
: t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 1
: t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 1
+ : t == GemmTiling::Tile8x32x64_Simdgroups1x1x4 ? 1
: t == GemmTiling::Tile16x32x32_Simdgroups2x1 ? 2
: 0;
}
@@ -91,6 +95,7 @@ constexpr uint gemm_tiling_simdgroups_per_column(GemmTiling t) {
: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 4
: t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 2
: t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 2
+ : t == GemmTiling::Tile8x32x64_Simdgroups1x1x4 ? 1
: t == GemmTiling::Tile16x32x32_Simdgroups2x1 ? 1
: 0;
}
@@ -99,6 +104,7 @@ constexpr uint gemm_tiling_simdgroups_per_k(GemmTiling t) {
return t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 4
: t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 4
: t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 4
+ : t == GemmTiling::Tile8x32x64_Simdgroups1x1x4 ? 4
: 1;
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
index ec9d94221..187659cf4 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
@@ -5,6 +5,7 @@
#include "../../common/defines.h"
#include "../../common/quant_pack.h"
+#include "../../layout/interleaved_rows_layout.h"
#include "../../common/quant_unpack.h"
using namespace metal;
@@ -54,12 +55,18 @@ struct QuantizedBlockLoaderScaleBias {
const device T* scales;
const device T* biases;
const bool signed_codes;
+ const bool interleaved_rows;
+
+ static METAL_FUNC int source_offset(bool interleaved_rows, int byte) {
+ return interleaved_rows ? InterleavedRowsLayout::byte_offset(byte) : byte;
+ }
QuantizedBlockLoaderScaleBias(
const device uint8_t* src_,
const device T* scales_,
const device T* biases_,
const bool signed_codes_,
+ const bool interleaved_rows_,
const int src_leading_dim_,
const int params_group_stride_,
const int params_output_stride_,
@@ -67,15 +74,22 @@ struct QuantizedBlockLoaderScaleBias {
ushort simd_group_id [[simdgroup_index_in_threadgroup]],
ushort simd_lane_id [[thread_index_in_simdgroup]]
)
- : src_leading_dim(src_leading_dim_), tile_stride(THREADGROUP_TILE_COLS_PACKED * BYTES_PER_PACK),
+ : src_leading_dim(src_leading_dim_),
+ tile_stride(source_offset(interleaved_rows_, THREADGROUP_TILE_COLS_PACKED * BYTES_PER_PACK)),
group_step_counter(0), group_stride(params_group_stride_), thread_index(simd_group_id * 32 + simd_lane_id),
tile_row_index(READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED),
tile_col_index((READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED),
dst(reinterpret_cast<threadgroup StageT*>(dst_) + tile_row_index * DESTINATION_LEADING_DIMENSION +
tile_col_index * PACK_FACTOR),
- src(src_ + tile_row_index * src_leading_dim_ * BYTES_PER_PACK / PACK_FACTOR + tile_col_index * BYTES_PER_PACK),
+ src(src_ +
+ (interleaved_rows_
+ ? sizeof(uint) *
+ InterleavedRowsLayout::word_index(tile_row_index, 0, src_leading_dim_ / W4_CODES_PER_WORD)
+ : tile_row_index * src_leading_dim_ * BYTES_PER_PACK / PACK_FACTOR) +
+ source_offset(interleaved_rows_, tile_col_index * BYTES_PER_PACK)),
scales(scales_ + tile_row_index * params_output_stride_),
- biases(biases_ + tile_row_index * params_output_stride_), signed_codes(signed_codes_) {}
+ biases(biases_ + tile_row_index * params_output_stride_), signed_codes(signed_codes_),
+ interleaved_rows(interleaved_rows_) {}
void load_unsafe() const {
if constexpr (TILE_HAS_IDLE_THREADS) {
@@ -89,7 +103,8 @@ struct QuantizedBlockLoaderScaleBias {
float(*scales),
float(*biases),
dst,
- signed_codes
+ signed_codes,
+ interleaved_rows ? InterleavedRowsLayout::STRIP_ROWS : 1
);
}
@@ -115,7 +130,7 @@ struct QuantizedBlockLoaderScaleBias {
const int pack_index = tile_col_index + i;
if (pack_index < valid_packs) {
dequantize<StageT, ComputeT, PACK_FACTOR, BITS>(
- src + i * BYTES_PER_PACK,
+ src + source_offset(interleaved_rows, i * BYTES_PER_PACK),
scale,
bias,
dst + i * PACK_FACTOR,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
index 3ab8d9b59..1026259a3 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
@@ -5,6 +5,7 @@
#include "../../common/defines.h"
#include "../../common/quant_pack.h"
+#include "../../layout/interleaved_rows_layout.h"
#include "../../common/quant_unpack.h"
using namespace metal;
@@ -57,12 +58,18 @@ struct QuantizedBlockLoaderScaleZeroPoint {
const device uint8_t* zero_points_row_start;
uint zero_point_index;
const bool signed_codes;
+ const bool interleaved_rows;
+
+ static METAL_FUNC int source_offset(bool interleaved_rows, int byte) {
+ return interleaved_rows ? InterleavedRowsLayout::byte_offset(byte) : byte;
+ }
QuantizedBlockLoaderScaleZeroPoint(
const device uint8_t* src_,
const device T* scales_,
const device uint8_t* zero_points_row_start_,
const bool signed_codes_,
+ const bool interleaved_rows_,
const int src_leading_dim_,
const int params_group_stride_,
const int params_output_stride_,
@@ -73,18 +80,24 @@ struct QuantizedBlockLoaderScaleZeroPoint {
ushort simd_group_id [[simdgroup_index_in_threadgroup]],
ushort simd_lane_id [[thread_index_in_simdgroup]]
)
- : src_leading_dim(src_leading_dim_), tile_stride(THREADGROUP_TILE_COLS_PACKED * BYTES_PER_PACK),
+ : src_leading_dim(src_leading_dim_),
+ tile_stride(source_offset(interleaved_rows_, THREADGROUP_TILE_COLS_PACKED * BYTES_PER_PACK)),
group_step_counter(0), group_stride(params_group_stride_), zero_point_group_stride(zero_point_group_stride_),
thread_index(simd_group_id * 32 + simd_lane_id),
tile_row_index(READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED),
tile_col_index((READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED),
dst(reinterpret_cast<threadgroup StageT*>(dst_) + tile_row_index * DESTINATION_LEADING_DIMENSION +
tile_col_index * PACK_FACTOR),
- src(src_ + tile_row_index * src_leading_dim_ * BYTES_PER_PACK / PACK_FACTOR + tile_col_index * BYTES_PER_PACK),
+ src(src_ +
+ (interleaved_rows_
+ ? sizeof(uint) *
+ InterleavedRowsLayout::word_index(tile_row_index, 0, src_leading_dim_ / W4_CODES_PER_WORD)
+ : tile_row_index * src_leading_dim_ * BYTES_PER_PACK / PACK_FACTOR) +
+ source_offset(interleaved_rows_, tile_col_index * BYTES_PER_PACK)),
scales(scales_ + tile_row_index * params_output_stride_),
zero_points_row_start(SCALE_SYMMETRIC ? nullptr : zero_points_row_start_),
zero_point_index(zero_point_index_ + uint(tile_row_index) * zero_point_output_stride_),
- signed_codes(signed_codes_) {}
+ signed_codes(signed_codes_), interleaved_rows(interleaved_rows_) {}
QuantizedBlockLoaderScaleZeroPoint(
const device uint8_t* src_,
@@ -102,6 +115,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {
scales_,
static_cast<const device uint8_t*>(nullptr),
signed_codes_,
+ false,
src_leading_dim_,
params_group_stride_,
params_output_stride_,
@@ -139,7 +153,14 @@ struct QuantizedBlockLoaderScaleZeroPoint {
float scale;
float bias;
current_scale_bias(scale, bias);
- dequantize<StageT, ComputeT, READS_PER_THREAD * PACK_FACTOR, BITS>(src, scale, bias, dst, signed_codes);
+ dequantize<StageT, ComputeT, READS_PER_THREAD * PACK_FACTOR, BITS>(
+ src,
+ scale,
+ bias,
+ dst,
+ signed_codes,
+ interleaved_rows ? InterleavedRowsLayout::STRIP_ROWS : 1
+ );
}
void load_safe(short2 src_tile_dim) const {
@@ -165,7 +186,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {
const int pack_index = tile_col_index + i;
if (pack_index < valid_packs) {
dequantize<StageT, ComputeT, PACK_FACTOR, BITS>(
- src + i * BYTES_PER_PACK,
+ src + source_offset(interleaved_rows, i * BYTES_PER_PACK),
scale,
bias,
dst + i * PACK_FACTOR,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
index 937ee0302..c20ac13e1 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
@@ -7,7 +7,7 @@
#include "../../../common/fragment.h"
#include "../../../common/quant_pack.h"
#include "../../../common/quant_unpack.h"
-#include "../mxu_tiled_layout.h"
+#include "../../../layout/mxu_tiled_layout.h"
#include "../schedules/tile_context.h"
using namespace metal;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/packed.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/packed.h
index 70c359ed5..6fb7304fc 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/packed.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/packed.h
@@ -4,6 +4,7 @@
#include "../../../../../generated/gemm.h"
#include "../../../../common/fragment.h"
+#include "../../../../layout/interleaved_rows_layout.h"
#include "../../../../common/quant_unpack.h"
#include "../../../../common/simdgroup_fragment_ops.h"
#include "../../../../common/packed_half_layout.h"
@@ -55,6 +56,7 @@ struct PackedSchedule {
const uint lane_row = uint(tile_context.lane_position.y);
const uint weight_sign_mask = right.sign_mask();
+ const bool interleaved_rows = right.gemm_b_layout == GemmBLayout::InterleavedRows;
const bool row_scaled = right.hoist_row_metadata;
typename Core::AccumFragment accumulator;
accumulator.clear();
@@ -65,6 +67,7 @@ struct PackedSchedule {
for (uint k_block = tile_context.k_block_begin + tile_context.simdgroup_k_slice; k_block < k_block_end;
k_block += Core::SIMDGROUPS_PER_K) {
const uint first_weight_group = k_block * GROUPS_PER_K_BLOCK;
+ METAL_PRAGMA_UNROLL
for (ushort group_in_block = 0; group_in_block < GROUPS_PER_K_BLOCK; ++group_in_block) {
const uint weight_group_index = first_weight_group + group_in_block;
const device uint* packed_group = packed_activations + weight_group_index * WORDS_PER_GROUP;
@@ -88,6 +91,19 @@ struct PackedSchedule {
float weight_scales[Core::TILES_M];
float weight_offsets[Core::TILES_M]; // zero point, or bias
uint packed_weight_words[Core::TILES_M * WORDS_PER_LANE];
+ uint4 word_runs[WORDS_PER_LANE];
+ if (interleaved_rows) {
+ METAL_PRAGMA_UNROLL
+ for (ushort word_index = 0; word_index < WORDS_PER_LANE; ++word_index) {
+ word_runs[word_index] = InterleavedRowsLayout::lane_run(
+ right.codes,
+ tile_context.simdgroup_column_base + lane_row,
+ weight_group_index * (WEIGHT_GROUP_SIZE / W4_CODES_PER_WORD) + fragment_column_pair * WORDS_PER_LANE +
+ word_index,
+ params->K / W4_CODES_PER_WORD
+ );
+ }
+ }
METAL_PRAGMA_UNROLL
for (ushort tile_m = 0; tile_m < Core::TILES_M; ++tile_m) {
weight_rows[tile_m] =
@@ -98,7 +114,8 @@ struct PackedSchedule {
);
METAL_PRAGMA_UNROLL
for (ushort word_index = 0; word_index < WORDS_PER_LANE; ++word_index) {
- packed_weight_words[tile_m * WORDS_PER_LANE + word_index] = weight_source[word_index] ^ weight_sign_mask;
+ const uint word = interleaved_rows ? word_runs[word_index][tile_m] : weight_source[word_index];
+ packed_weight_words[tile_m * WORDS_PER_LANE + word_index] = word ^ weight_sign_mask;
}
if (!row_scaled) {
weight_scales[tile_m] = float(right.scale(weight_rows[tile_m], weight_group_index, params));
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h
index 5d6d97d22..2a49f98cd 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h
@@ -33,9 +33,10 @@ static METAL_FUNC auto make_staged_loader(
const int params_offset =
int(block_col) * int(params->scale_output_stride) + int(first_group) * int(params->scale_group_stride);
const device Element* scales = right.scales + params_offset;
+ const bool interleaved_rows = right.gemm_b_layout == GemmBLayout::InterleavedRows;
+ const int k_bytes = int(k_offset) * get_bytes_per_pack<RightOperand::BITS>() / get_pack_factor<RightOperand::BITS>();
const device uint8_t* values = right.codes + size_t(block_col) * row_stride +
- size_t(k_offset) * size_t(get_bytes_per_pack<RightOperand::BITS>()) /
- size_t(get_pack_factor<RightOperand::BITS>());
+ (interleaved_rows ? InterleavedRowsLayout::byte_offset(k_bytes) : k_bytes);
if constexpr (RightOperand::SCHEME == GemmBPrologueKind::ScaleBiasDequant) {
const device Element* biases = right.bias() + params_offset;
using Loader = QuantizedBlockLoaderScaleBias<
@@ -53,6 +54,7 @@ static METAL_FUNC auto make_staged_loader(
scales,
biases,
right.signed_codes,
+ interleaved_rows,
int(params->K),
int(params->scale_group_stride),
int(params->scale_output_stride),
@@ -77,6 +79,7 @@ static METAL_FUNC auto make_staged_loader(
scales,
right.zp(),
right.signed_codes,
+ interleaved_rows,
int(params->K),
int(params->scale_group_stride),
int(params->scale_output_stride),
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
index 2af9b7c13..b29e37139 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -13,7 +13,8 @@ using namespace uzu::gemm;
#define A_IS_INT8 (A_PROLOGUE == GemmAPrologueKind::Int8Symmetric)
#define GEMM_SMALL_BATCH \
- (GEMM_TILING == GemmTiling::Tile8x32x64_Simdgroups1x2x4 || GEMM_TILING == GemmTiling::Tile8x32x128_Simdgroups1x2x4)
+ (GEMM_TILING == GemmTiling::Tile8x32x64_Simdgroups1x2x4 || GEMM_TILING == GemmTiling::Tile8x32x64_Simdgroups1x1x4 || \
+ GEMM_TILING == GemmTiling::Tile8x32x128_Simdgroups1x2x4)
#define GEMM_SMALL_BATCH_W8 (GEMM_SMALL_BATCH && A_PROLOGUE != GemmAPrologueKind::PackedHalf)
#define NEEDS_ASYMMETRIC_WEIGHT_CORRECTION (A_IS_INT8 && B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant)
#define GEMM_MXU_QUANT (USE_MXU && B_PROLOGUE != GemmBPrologueKind::FullPrecision && !A_IS_INT8)
@@ -64,7 +65,8 @@ VARIANTS(
GemmTiling::Tile64x64x256_Simdgroups2x2,
GemmTiling::Tile128x128x256_Simdgroups4x4,
GemmTiling::Tile8x32x64_Simdgroups1x2x4,
- GemmTiling::Tile8x32x128_Simdgroups1x2x4)
+ GemmTiling::Tile8x32x128_Simdgroups1x2x4,
+ GemmTiling::Tile8x32x64_Simdgroups1x1x4)
VARIANTS(TRANSPOSE_B, false, true)
VARIANTS(USE_MXU, false, true)
VARIANTS(
@@ -117,11 +119,14 @@ CONSTRAINT(
GEMM_TILING != GemmTiling::Tile16x128x256_Simdgroups1x4 ||
(TRANSPOSE_B && B_PROLOGUE == GemmBPrologueKind::FullPrecision))
CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || A_PROLOGUE == GemmAPrologueKind::PackedHalf || USE_MXU)
-CONSTRAINT(GEMM_TILING != GemmTiling::Tile8x32x64_Simdgroups1x2x4 || A_PROLOGUE == GemmAPrologueKind::PackedHalf)
+CONSTRAINT(
+ (GEMM_TILING != GemmTiling::Tile8x32x64_Simdgroups1x2x4 && GEMM_TILING != GemmTiling::Tile8x32x64_Simdgroups1x1x4) ||
+ A_PROLOGUE == GemmAPrologueKind::PackedHalf)
CONSTRAINT(
A_PROLOGUE != GemmAPrologueKind::PackedHalf ||
(GEMM_SMALL_BATCH && !USE_MXU && TRANSPOSE_B &&
- ((GEMM_TILING == GemmTiling::Tile8x32x64_Simdgroups1x2x4 && BITS == 4) ||
+ (((GEMM_TILING == GemmTiling::Tile8x32x64_Simdgroups1x2x4 || GEMM_TILING == GemmTiling::Tile8x32x64_Simdgroups1x1x4) &&
+ BITS == 4) ||
(GEMM_TILING == GemmTiling::Tile8x32x128_Simdgroups1x2x4 && BITS == 2))))
CONSTRAINT(
!GEMM_SMALL_BATCH_W8 ||
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 71531c122..f713ab311 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -1,12 +1,9 @@
use std::collections::{HashMap, hash_map::Entry};
-use futures::executor::block_on;
-
use crate::{
backends::{
common::{
- Backend, BufferMut, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,
- Context,
+ Backend, BufferMut, BufferRef, CommandBufferEncoding,
gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
kernel::matmul::{
Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulOutput, MatmulShape, QuantParamsLayout,
@@ -20,10 +17,10 @@ use crate::{
error::MetalError,
gpu_types::{
gemm::{GemmAPrologueKind, GemmAlignment, GemmBLayout},
- matmul::{GemmParams, MXU_TILED_BLOCK_ROWS, MXU_TILED_PREPARE_THREADS, MXU_TILED_W2_BLOCK_ROWS},
+ matmul::GemmParams,
},
kernel::{
- GemmMetalKernel, GemmPrepareBLayoutMetalKernel, GemmSplitKReduceMetalKernel,
+ GemmMetalKernel, GemmSplitKReduceMetalKernel,
matmul::{
MatmulMetalError, MatmulOutputWork,
gemm::{
@@ -102,43 +99,6 @@ impl GemmKernel {
}
}
- fn mxu_tiled_block_rows(bits: u32) -> u32 {
- if bits == 2 {
- MXU_TILED_W2_BLOCK_ROWS
- } else {
- MXU_TILED_BLOCK_ROWS
- }
- }
-
- pub fn mxu_tiled_words_per_thread(
- k: u32,
- bits: u32,
- ) -> Option<u32> {
- let words_per_thread =
- (Self::mxu_tiled_block_rows(bits) * (k * bits / u32::BITS)).div_ceil(MXU_TILED_PREPARE_THREADS);
- [32, 72, 160].into_iter().find(|&variant| variant >= words_per_thread)
- }
-
- pub fn prepare_gemm_b_layout(
- weights: &mut <Metal as Backend>::GlobalGpuBuffer,
- rows: u32,
- k: u32,
- bits: u32,
- context: &MetalContext,
- ) -> Result<(), MetalError> {
- let row_words = k * bits / u32::BITS;
- let variant = Self::mxu_tiled_words_per_thread(k, bits).expect("int8 eligibility admits only tileable weights");
- let mut command_buffer = context.create_command_buffer(Some("prepare GEMM B layout"), None)?;
- GemmPrepareBLayoutMetalKernel::new(context, bits, variant)?.encode(
- weights,
- row_words,
- rows / Self::mxu_tiled_block_rows(bits),
- &mut command_buffer,
- );
- block_on(command_buffer.end_encoding().submit().wait_until_completed())?;
- Ok(())
- }
-
fn get_or_create_split_k_reduce(
&mut self,
context: &MetalContext,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
index 16b6a68fb..5342771c7 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
@@ -64,11 +64,15 @@ pub fn small_batch_format(
shape: MatmulShape,
layout: GemmBLayout,
) -> Option<(ActivationFormat, GemmTiling)> {
- let (activation, tiling, k_alignment) = small_m_weights(&shape)?;
+ let (activation, row_major_tiling, k_alignment) = small_m_weights(&shape)?;
+ let tiling = match layout {
+ GemmBLayout::RowMajor => row_major_tiling,
+ GemmBLayout::InterleavedRows if shape.b_bits == Some(4) => GemmTiling::Tile8x32x64_Simdgroups1x1x4,
+ _ => return None,
+ };
(shape.b_transpose
&& !shape.gathered
&& !shape.b_is_trellis
- && layout == GemmBLayout::RowMajor
&& shape.b_leading_dimension.is_none()
&& shape.k.is_multiple_of(k_alignment)
&& (!shape.d_transform.contains(GemmDTransform::RHT) || shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE)))
@@ -170,6 +174,7 @@ impl GemmProblem {
if !splittable
|| !self.split_k_output_supported()
|| (shape.b_bits == Some(2) && tiling == GemmTiling::Tile8x32x128_Simdgroups1x2x4)
+ || tiling == GemmTiling::Tile8x32x64_Simdgroups1x1x4
{
return 1;
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h
index 314bb65c5..f38714833 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/epilogue.h
@@ -35,7 +35,7 @@ struct Epilogue {
}
if constexpr (Tile::OUTPUT_ROWS >= METAL_SIMD_SIZE) {
- static_assert(Tile::K_SPLIT == 1, "RHT reuses shared results after Reduce");
+ static_assert(Tile::K_SPLIT == 1 || Tile::INPUT_ROWS == 1, "RHT reuses shared results after Reduce");
threadgroup_barrier(mem_flags::mem_threadgroup);
constexpr uint OUTPUT_BLOCKS = Tile::OUTPUT_ROWS / METAL_SIMD_SIZE;
for (uint job = tile.simd_group; job < Tile::INPUT_ROWS * OUTPUT_BLOCKS; job += Tile::NUM_SIMDGROUPS) {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h
index 9bdeefe12..5bd08a31a 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_b_source.h
@@ -42,11 +42,12 @@ struct QuantBSource {
const device uint8_t* weights = reinterpret_cast<const device uint8_t*>(ops.b);
const uint batch_remaining = params.batch_size - tile.input_row;
- uint group = group_slot;
+ const uint slice_groups = groups / Tile::K_SPLIT;
+ const uint group_end = (tile.k_slice + 1) * slice_groups;
Slice current;
- QuantPosition position = {group, 0};
- while (position.valid(groups)) {
+ QuantPosition position = {group_slot + tile.k_slice * slice_groups, 0};
+ while (position.valid(group_end)) {
Metadata metadata;
metadata.load(position.group, weight_row_indices, ops, params);
for (position.slice = 0; position.slice < Slice::SLICES_PER_LANE; position.slice++) {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h
index 56ace9003..25528d695 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h
@@ -1,6 +1,7 @@
#pragma once
-#include "../../gemm/common/mxu_tiled_layout.h"
+#include "../../layout/interleaved_rows_layout.h"
+#include "../../layout/mxu_tiled_layout.h"
#include "arguments.h"
#include "quant_chunk.h"
#include "quant_metadata.h"
@@ -118,6 +119,23 @@ private:
});
}
+ METAL_FUNC void load_interleaved_rows_weights(
+ const device uint8_t* weights_base,
+ const thread uint (&weight_row_indices)[Tile::ROWS_PER_LANE],
+ uint row_words,
+ uint first_word
+ ) thread {
+ const device uint* words = reinterpret_cast<const device uint*>(weights_base);
+ Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
+ constexpr uint R = decltype(output_index)::value;
+ METAL_PRAGMA_UNROLL
+ for (uint word = 0; word < SLICE_BYTES / sizeof(uint); ++word) {
+ reinterpret_cast<thread uint*>(weights[R])[word] =
+ words[InterleavedRowsLayout::word_index(weight_row_indices[R], first_word + word, row_words)];
+ }
+ });
+ }
+
public:
static METAL_FUNC uint row_stride(const thread GemvParams& params) {
return params.in_vec_size * BITS / QuantChunk<BITS>::BITS_PER_BYTE;
@@ -141,6 +159,17 @@ public:
}
return;
}
+ if constexpr (BITS == 4) {
+ if (params.gemm_b_layout == GemmBLayout::InterleavedRows) {
+ load_interleaved_rows_weights(
+ weights_base,
+ weight_row_indices,
+ params.in_vec_size / W4_CODES_PER_WORD,
+ k / W4_CODES_PER_WORD
+ );
+ return;
+ }
+ }
Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
constexpr uint R = decltype(output_index)::value;
const uint row = weight_row_indices[R];
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
index 4293081e5..65050b21f 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -20,6 +20,9 @@ using namespace uzu::gemm;
(K_SPLIT == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE == 1 && \
((BITS == 2 && GROUP_SIZE == 32 * GROUP_LANES) || (BITS == 4 && GROUP_SIZE == 16 * GROUP_LANES) || \
(BITS == 8 && GROUP_SIZE == 8 * GROUP_LANES)))
+#define GEMV_INTERLEAVED_ROWS_TILE \
+ (K_SPLIT == 8 && REDUCTION_LANES == 4 && NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE == 1 && \
+ BITS == 4 && GROUP_SIZE == 16 * GROUP_LANES)
#define GEMV_BATCHED_FMA_32_TILE \
(K_SPLIT == 1 && REDUCTION_LANES == 8 && NUM_SIMDGROUPS == 4 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE >= 2 && \
INPUT_ROW_TILE <= 4)
@@ -57,7 +60,7 @@ VARIANTS(K_SPLIT, 1, 2, 4, 8)
VARIANTS(INPUT_ALIGNED, false, true)
VARIANTS(INPUT_ROW_TILE, 1, 2, 3, 4)
VARIANTS(OUTPUT_ROW_TILE, 1, 2, 4, 8, 16, 32)
-VARIANTS(REDUCTION_LANES, 8, 32)
+VARIANTS(REDUCTION_LANES, 4, 8, 32)
VARIANTS(GROUP_LANES, 1, 2, 4, 8)
VARIANTS(NUM_SIMDGROUPS, 2, 4, 8)
@@ -72,7 +75,8 @@ CONSTRAINT((BITS == 2) == (GROUP_SIZE == 128))
CONSTRAINT(INPUT_ROW_TILE == 1 || GEMV_BATCHED_INPUT)
CONSTRAINT(
GEMV_FULL_PRECISION_TILE ||
- (BITS != 0 && (GEMV_QUANTIZED_ROW_TILE || GEMV_BATCHED_FMA_32_TILE || GEMV_BATCHED_FMA_16_TILE)))
+ (BITS != 0 && (GEMV_QUANTIZED_ROW_TILE || GEMV_INTERLEAVED_ROWS_TILE || GEMV_BATCHED_FMA_32_TILE ||
+ GEMV_BATCHED_FMA_16_TILE)))
KERNEL(Gemv)(
const device uint32_t* b,
const device BT* scales
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 985d5fe46..ede302e1b 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -69,7 +69,9 @@ impl GemvSpecialization {
&& weights_data_type == DataType::BF16
&& shape.k >= MXU_TILED_KERNEL_MIN_K
&& shape.n >= 32 * gpu_core_count;
- let tile = if is_quant && shape.gathered {
+ let tile = if is_quant && !shape.gathered && gemm_b_layout == GemmBLayout::InterleavedRows && shape.m == 1 {
+ policy::interleaved_rows_tile(shape.b_group_size.unwrap_or(0), shape.k)
+ } else if is_quant && shape.gathered {
policy::gathered_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n)
} else if is_quant {
policy::quantized_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n, bf16_io)
@@ -191,7 +193,7 @@ fn output_transform_for_tile(
}
let output_rows = tile.output_row_tile();
- let can_fuse = tile.k_split == 1
+ let can_fuse = (tile.k_split == 1 || (shape.is_quant() && tile.input_row_tile == 1))
&& output_rows >= HADAMARD_TRANSFORM_BLOCK_SIZE
&& output_rows.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE);
if can_fuse {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs
index ce64b7eb8..e2240c977 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs
@@ -1,6 +1,6 @@
mod quantized;
-pub(super) use quantized::{gathered_tile, select as quantized_tile};
+pub(super) use quantized::{gathered_tile, interleaved_rows_tile, select as quantized_tile};
// Full-precision GEMV accumulates four K values per SIMD lane, so one full
// vectorized K block is 4 * 32 lanes.
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
index 7d790efc1..7989b5f54 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
@@ -45,3 +45,17 @@ pub fn select(
}
gathered_tile(bits, group, m, n)
}
+
+pub fn interleaved_rows_tile(
+ group: u32,
+ k: u32,
+) -> Option<GemvTile> {
+ const VALUES_PER_LANE: u32 = 16;
+ let strip_tile = GemvTile::quantized(8, 32, 1, 4, group / VALUES_PER_LANE);
+ let tile = GemvTile {
+ k_split: 8,
+ ..strip_tile
+ };
+ let k_per_step = tile.k_split * tile.reduction_lanes * VALUES_PER_LANE;
+ (matches!(group, 32 | 64) && k.is_multiple_of(k_per_step)).then_some(tile)
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/gemm_prepare_b_layout.metal
similarity index 98%
rename from crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal
rename to crates/uzu-engine/src/backends/metal/kernel/matmul/layout/gemm_prepare_b_layout.metal
index 0402a38e2..5d40b8445 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/gemm_prepare_b_layout.metal
@@ -1,6 +1,6 @@
#include "../../common/dsl.h"
#include "../../common/defines.h"
-#include "common/mxu_tiled_layout.h"
+#include "mxu_tiled_layout.h"
using namespace metal;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/interleaved_rows.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/interleaved_rows.rs
new file mode 100644
index 000000000..62ef2fad0
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/interleaved_rows.rs
@@ -0,0 +1,58 @@
+use futures::executor::block_on;
+
+use crate::backends::{
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
+ kernel::matmul::{MatmulShape, QuantParamsLayout},
+ },
+ metal::{
+ Metal,
+ context::MetalContext,
+ error::MetalError,
+ gpu_types::{
+ gemm::GemmBLayout,
+ matmul::{INTERLEAVED_ROWS_RELAYOUT_THREADS, INTERLEAVED_ROWS_STRIP_ROWS},
+ },
+ kernel::{
+ PrepareInterleavedRowsMetalKernel,
+ matmul::{MatmulMetalKernel, gemm::small_batch_format},
+ },
+ },
+};
+
+const CODE_BITS: u32 = 4;
+
+fn words_per_thread(row_words: u32) -> Option<u32> {
+ let words_per_thread = (INTERLEAVED_ROWS_STRIP_ROWS * row_words).div_ceil(INTERLEAVED_ROWS_RELAYOUT_THREADS);
+ [16, 48, 96].into_iter().find(|&variant| variant >= words_per_thread)
+}
+
+/// Relays W4 weights read by the packed small-batch GEMM in place and returns true, or returns false if they are not.
+pub fn prepare(
+ kernel: &MatmulMetalKernel,
+ shape: &MatmulShape,
+ weights: &mut <Metal as Backend>::GlobalGpuBuffer,
+ context: &MetalContext,
+) -> Result<bool, MetalError> {
+ let eligible = !context.supports_mxu
+ && kernel.is_all_bf16()
+ && shape.params_layout == Some(QuantParamsLayout::GroupOutput)
+ && shape.n.is_multiple_of(INTERLEAVED_ROWS_STRIP_ROWS)
+ && small_batch_format(*shape, GemmBLayout::InterleavedRows).is_some();
+ let row_words = shape.k * CODE_BITS / u32::BITS;
+ if !eligible {
+ return Ok(false);
+ }
+ let Some(variant) = words_per_thread(row_words) else {
+ return Ok(false);
+ };
+ let mut command_buffer = context.create_command_buffer(Some("prepare interleaved-row weights"), None)?;
+ PrepareInterleavedRowsMetalKernel::new(context, variant)?.encode(
+ weights,
+ row_words,
+ shape.n / INTERLEAVED_ROWS_STRIP_ROWS,
+ &mut command_buffer,
+ );
+ block_on(command_buffer.end_encoding().submit().wait_until_completed())?;
+ Ok(true)
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/interleaved_rows_layout.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/interleaved_rows_layout.h
new file mode 100644
index 000000000..c329e7ac4
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/interleaved_rows_layout.h
@@ -0,0 +1,38 @@
+#pragma once
+
+#include "../../common/defines.h"
+#include "../../generated/matmul.h"
+
+namespace uzu {
+namespace gemm {
+
+// 32-row strips, word j of rows ordered 0, 8, 16, 24 | 1, 9, 17, 25 | ... so a GEMM lane's 4 rows are one uint4.
+struct InterleavedRowsLayout {
+ UZU_CONST uint STRIP_ROWS = uzu::matmul::INTERLEAVED_ROWS_STRIP_ROWS;
+ UZU_CONST uint LANE_ROWS = uzu::matmul::INTERLEAVED_ROWS_LANE_ROWS;
+ UZU_CONST uint ROWS_PER_LANE = STRIP_ROWS / LANE_ROWS;
+
+ static METAL_FUNC size_t word_index(uint row, uint word, uint row_words) {
+ return size_t(row / STRIP_ROWS) * STRIP_ROWS * row_words + word * STRIP_ROWS + row % LANE_ROWS * ROWS_PER_LANE +
+ row / LANE_ROWS % ROWS_PER_LANE;
+ }
+
+ static METAL_FUNC uint row_major_word_in_strip(uint index, uint row_words) {
+ const uint slot = index % STRIP_ROWS;
+ return (slot % ROWS_PER_LANE * LANE_ROWS + slot / ROWS_PER_LANE) * row_words + index / STRIP_ROWS;
+ }
+
+ static METAL_FUNC int byte_offset(int byte) {
+ const int word_bytes = sizeof(uint);
+ return byte / word_bytes * int(STRIP_ROWS) * word_bytes + byte % word_bytes;
+ }
+
+ static METAL_FUNC uint4 lane_run(const device uint8_t* codes, uint row, uint word, uint row_words) {
+ return *reinterpret_cast<const device uint4*>(
+ reinterpret_cast<const device uint*>(codes) + word_index(row, word, row_words)
+ );
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mod.rs
new file mode 100644
index 000000000..3865b1c04
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mod.rs
@@ -0,0 +1,29 @@
+mod interleaved_rows;
+mod mxu_tiled;
+
+pub use mxu_tiled::words_per_thread as mxu_tiled_words_per_thread;
+
+use crate::backends::{
+ common::{Backend, kernel::matmul::MatmulShape},
+ metal::{
+ Metal, context::MetalContext, error::MetalError, gpu_types::gemm::GemmBLayout,
+ kernel::matmul::MatmulMetalKernel,
+ },
+};
+
+/// Picks the layout the weights are read in, relays them in place and returns it with their code signedness.
+pub fn prepare(
+ kernel: &MatmulMetalKernel,
+ shape: &MatmulShape,
+ weights: &mut <Metal as Backend>::GlobalGpuBuffer,
+ context: &MetalContext,
+) -> Result<(GemmBLayout, bool), MetalError> {
+ if mxu_tiled::eligible(kernel, shape, context) {
+ return Ok((GemmBLayout::MxuTiled, mxu_tiled::prepare(shape, weights, context)?));
+ }
+ if interleaved_rows::prepare(kernel, shape, weights, context)? {
+ Ok((GemmBLayout::InterleavedRows, shape.signed_codes))
+ } else {
+ Ok((GemmBLayout::RowMajor, shape.signed_codes))
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mxu_tiled.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mxu_tiled.rs
new file mode 100644
index 000000000..7c5b1eb89
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mxu_tiled.rs
@@ -0,0 +1,78 @@
+use futures::executor::block_on;
+
+use crate::backends::{
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, CpuBuffer,
+ kernel::matmul::{MatmulKernel, MatmulShape},
+ },
+ metal::{
+ Metal,
+ context::MetalContext,
+ error::MetalError,
+ gpu_types::matmul::{MXU_TILED_BLOCK_ROWS, MXU_TILED_PREPARE_THREADS, MXU_TILED_W2_BLOCK_ROWS},
+ kernel::{GemmPrepareBLayoutMetalKernel, matmul::MatmulMetalKernel},
+ },
+};
+
+fn block_rows(bits: u32) -> u32 {
+ if bits == 2 {
+ MXU_TILED_W2_BLOCK_ROWS
+ } else {
+ MXU_TILED_BLOCK_ROWS
+ }
+}
+
+pub fn words_per_thread(
+ k: u32,
+ bits: u32,
+) -> Option<u32> {
+ let words_per_thread = (block_rows(bits) * (k * bits / u32::BITS)).div_ceil(MXU_TILED_PREPARE_THREADS);
+ [32, 72, 160].into_iter().find(|&variant| variant >= words_per_thread)
+}
+
+fn prepared_shape(shape: &MatmulShape) -> MatmulShape {
+ MatmulShape {
+ signed_codes: shape.signed_codes || shape.b_bits == Some(8),
+ ..*shape
+ }
+}
+
+pub fn eligible(
+ kernel: &MatmulMetalKernel,
+ shape: &MatmulShape,
+ context: &MetalContext,
+) -> bool {
+ kernel.select_activation_quantization(&prepared_shape(shape), context).is_some()
+}
+
+/// Relays the weights in place and returns their code signedness.
+pub fn prepare(
+ shape: &MatmulShape,
+ weights: &mut <Metal as Backend>::GlobalGpuBuffer,
+ context: &MetalContext,
+) -> Result<bool, MetalError> {
+ let prepared_shape = prepared_shape(shape);
+ if !shape.signed_codes && prepared_shape.signed_codes {
+ let bytes = unsafe {
+ std::slice::from_raw_parts_mut(
+ weights.cpu_ptr().as_ptr().cast::<u8>(),
+ crate::backends::common::Buffer::size(weights),
+ )
+ };
+ let (prefix, words, suffix) = bytemuck::pod_align_to_mut::<u8, u64>(bytes);
+ words.iter_mut().for_each(|word| *word ^= 0x8080_8080_8080_8080);
+ prefix.iter_mut().chain(suffix.iter_mut()).for_each(|code| *code ^= 0x80);
+ }
+ let bits = shape.b_bits.expect("quantized weights");
+ let row_words = shape.k * bits / u32::BITS;
+ let variant = words_per_thread(shape.k, bits).expect("int8 eligibility admits only tileable weights");
+ let mut command_buffer = context.create_command_buffer(Some("prepare GEMM B layout"), None)?;
+ GemmPrepareBLayoutMetalKernel::new(context, bits, variant)?.encode(
+ weights,
+ row_words,
+ shape.n / block_rows(bits),
+ &mut command_buffer,
+ );
+ block_on(command_buffer.end_encoding().submit().wait_until_completed())?;
+ Ok(prepared_shape.signed_codes)
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_tiled_layout.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mxu_tiled_layout.h
similarity index 85%
rename from crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_tiled_layout.h
rename to crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mxu_tiled_layout.h
index 82c84900d..6783bdf73 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_tiled_layout.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/mxu_tiled_layout.h
@@ -1,8 +1,8 @@
#pragma once
-#include "../../../common/defines.h"
-#include "../../../generated/matmul.h"
-#include "../../common/mxu_fragment/ops.h"
+#include "../../common/defines.h"
+#include "../../generated/matmul.h"
+#include "../common/mxu_fragment/ops.h"
namespace uzu {
namespace gemm {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/prepare_interleaved_rows.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/prepare_interleaved_rows.metal
new file mode 100644
index 000000000..a07fab759
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/layout/prepare_interleaved_rows.metal
@@ -0,0 +1,39 @@
+#include "../../common/dsl.h"
+#include "../../common/defines.h"
+#include "../../generated/matmul.h"
+#include "interleaved_rows_layout.h"
+
+using namespace metal;
+
+using uzu::gemm::InterleavedRowsLayout;
+using uzu::matmul::INTERLEAVED_ROWS_RELAYOUT_THREADS;
+
+// Relays a strip's row-major W4 codes into InterleavedRowsLayout in place, through registers.
+template <uint WORDS_PER_THREAD>
+VARIANTS(WORDS_PER_THREAD, 16, 48, 96)
+KERNEL(PrepareInterleavedRows)(
+ device uint* weights,
+ const constant uint& row_words,
+ const constant uint& strip_count,
+ const uint strip GROUPS(strip_count),
+ const uint thread_index THREADS(INTERLEAVED_ROWS_RELAYOUT_THREADS)
+) {
+ const uint strip_words = InterleavedRowsLayout::STRIP_ROWS * row_words;
+ device uint* codes = weights + size_t(strip) * strip_words;
+ uint words[WORDS_PER_THREAD];
+ METAL_PRAGMA_UNROLL
+ for (uint slot = 0; slot < WORDS_PER_THREAD; ++slot) {
+ const uint index = slot * INTERLEAVED_ROWS_RELAYOUT_THREADS + thread_index;
+ if (index < strip_words) {
+ words[slot] = codes[InterleavedRowsLayout::row_major_word_in_strip(index, row_words)];
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_device);
+ METAL_PRAGMA_UNROLL
+ for (uint slot = 0; slot < WORDS_PER_THREAD; ++slot) {
+ const uint index = slot * INTERLEAVED_ROWS_RELAYOUT_THREADS + thread_index;
+ if (index < strip_words) {
+ codes[index] = words[slot];
+ }
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
index 7a4604178..cba7c0612 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
@@ -1,7 +1,7 @@
use crate::{
backends::{
common::{
- Backend, BufferMut, BufferRef, CommandBufferEncoding, CpuBuffer,
+ Backend, BufferMut, BufferRef, CommandBufferEncoding,
kernel::{
ActivationQuantization,
activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
@@ -20,6 +20,7 @@ use crate::{
GemmKernel, MatmulMetalError,
gemm::{self, GemmPlan, GemmProblem, small_batch_format, small_m_weights},
gemv::{GemvKernel, GemvSpecialization, GemvTile},
+ layout,
matmul_dispatch::MatmulDispatch,
matmul_output_work::MatmulOutputWork,
},
@@ -42,6 +43,10 @@ pub fn supports_integer_right_operand(shape: &MatmulShape) -> bool {
}
impl MatmulMetalKernel {
+ pub fn is_all_bf16(&self) -> bool {
+ all_bf16(self.weights_data_type, self.input_data_type, self.output_data_type)
+ }
+
fn prefer_gemm_over_gemv(
shape: MatmulShape,
plan: GemmPlan,
@@ -204,7 +209,7 @@ impl MatmulKernel for MatmulMetalKernel {
|| !shape.k.is_multiple_of(ACTIVATION_SCALE_GROUP_SIZE)
|| !shape.k.is_multiple_of(weight_group_size)
|| !shape.n.is_multiple_of(32)
- || shape.b_bits.and_then(|bits| GemmKernel::mxu_tiled_words_per_thread(shape.k, bits)).is_none()
+ || shape.b_bits.and_then(|bits| layout::mxu_tiled_words_per_thread(shape.k, bits)).is_none()
{
return None;
}
@@ -267,32 +272,7 @@ impl MatmulKernel for MatmulMetalKernel {
weights: &mut <Metal as Backend>::GlobalGpuBuffer,
context: &MetalContext,
) -> Result<(GemmBLayout, bool), MetalError> {
- let prepared_shape = MatmulShape {
- signed_codes: a8_shape.signed_codes || a8_shape.b_bits == Some(8),
- ..*a8_shape
- };
- if self.select_activation_quantization(&prepared_shape, context).is_none() {
- return Ok((GemmBLayout::RowMajor, a8_shape.signed_codes));
- }
- if !a8_shape.signed_codes && prepared_shape.signed_codes {
- let bytes = unsafe {
- std::slice::from_raw_parts_mut(
- weights.cpu_ptr().as_ptr().cast::<u8>(),
- crate::backends::common::Buffer::size(weights),
- )
- };
- let (prefix, words, suffix) = bytemuck::pod_align_to_mut::<u8, u64>(bytes);
- words.iter_mut().for_each(|word| *word ^= 0x8080_8080_8080_8080);
- prefix.iter_mut().chain(suffix.iter_mut()).for_each(|code| *code ^= 0x80);
- }
- GemmKernel::prepare_gemm_b_layout(
- weights,
- a8_shape.n,
- a8_shape.k,
- a8_shape.b_bits.expect("quantized weights"),
- context,
- )?;
- Ok((GemmBLayout::MxuTiled, prepared_shape.signed_codes))
+ layout::prepare(self, a8_shape, weights, context)
}
fn encode(
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
index 58e6b5ce8..88210c211 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
@@ -1,6 +1,7 @@
mod error;
pub mod gemm;
mod gemv;
+mod layout;
mod matmul_dispatch;
mod matmul_metal_kernel;
mod matmul_output_work;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/mod.rs
index 50d836c38..06f0bcb26 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/mod.rs
@@ -5,7 +5,7 @@ use crate::backends::{
buffer::MetalBufferExt,
gpu_types::{
gemm::{gemm_tiling_simdgroups_per_column, gemm_tiling_simdgroups_per_k, gemm_tiling_simdgroups_per_row},
- matmul::MXU_TILED_PREPARE_THREADS,
+ matmul::{INTERLEAVED_ROWS_RELAYOUT_THREADS, MXU_TILED_PREPARE_THREADS},
},
},
};
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/interleaved_rows_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/interleaved_rows_test.rs
new file mode 100644
index 000000000..aa07bc456
--- /dev/null
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/interleaved_rows_test.rs
@@ -0,0 +1,105 @@
+//! `GemmBLayout::InterleavedRows`: every kernel that reads interleaved-row W4 strips matches the row-major layout.
+use QuantizationMethod::{ScaleBias, ScaleZeroPoint};
+use futures::executor::block_on;
+use half::bf16;
+use uzu_engine_macros::uzu_test;
+
+use crate::{
+ backends::{
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
+ gpu_types::quantization_method::QuantizationMethod,
+ kernel::{
+ ActivationTransform,
+ matmul::{ActivationFormat, MatmulA, MatmulKernel, MatmulShape, packed_half_bytes},
+ },
+ },
+ metal::{Metal, context::MetalContext, gpu_types::gemm::GemmBLayout, kernel::matmul::MatmulMetalKernel},
+ },
+ data_type::DataType,
+ tests::{
+ assert::assert_eq_float,
+ helpers::{buffer_to_vec, create_buffer, create_buffer_with_data},
+ matmul::{QuantBuffers, QuantInput, quant_arguments},
+ },
+};
+
+fn run(
+ context: &MetalContext,
+ input: &QuantInput<bf16>,
+ interleaved_rows: bool,
+ gather_indices: Option<&<Metal as Backend>::GlobalGpuBuffer>,
+ n: u32,
+) -> Vec<bf16> {
+ let mut matmul = MatmulMetalKernel::new(context, DataType::BF16, DataType::BF16, DataType::BF16).expect("matmul");
+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(context, input);
+ if interleaved_rows {
+ let shape = MatmulShape::from_arguments(&quant_arguments(&mut buffers, input));
+ buffers.weights_layout = matmul.optimize_weights_layout(&shape, &mut buffers.w, context).expect("relayout").0;
+ assert_eq!(buffers.weights_layout, GemmBLayout::InterleavedRows);
+ }
+ let format = ActivationFormat::PackedHalf {
+ block_k: input.group_size,
+ bits: 4,
+ };
+ let packed = gather_indices.is_none() && (4..=8).contains(&input.m);
+ let mut packed_activations =
+ create_buffer::<Metal, u32>(context, packed_half_bytes(input.k, input.group_size, 4) / size_of::<u32>());
+ let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");
+ if packed {
+ let factors = create_buffer_with_data::<Metal, i32>(context, &vec![1; input.k as usize]);
+ ActivationTransform::<Metal>::quantize(context, DataType::BF16, format).expect("pack").encode_pack_half(
+ &buffers.x,
+ &mut packed_activations,
+ &factors,
+ input.m,
+ input.k,
+ &mut command_buffer,
+ );
+ }
+ let mut arguments = quant_arguments(&mut buffers, input);
+ if packed {
+ arguments.a = MatmulA::PackedHalf {
+ values: &packed_activations,
+ block_k: input.group_size,
+ bits: 4,
+ };
+ }
+ arguments.gather_indices = gather_indices;
+ arguments.n = n;
+ matmul.encode(arguments, &mut command_buffer).expect("matmul encode");
+ block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
+ buffer_to_vec::<Metal, bf16>(context, &buffers.y)[..(input.m * n) as usize].to_vec()
+}
+
+fn assert_close(
+ label: &str,
+ expected: &[bf16],
+ actual: &[bf16],
+) {
+ let rms = (expected.iter().map(|v| v.to_f32().powi(2)).sum::<f32>() / expected.len() as f32).sqrt();
+ assert_eq_float(expected, actual, 2e-2 * rms, label);
+}
+
+#[uzu_test]
+fn interleaved_rows_matches_row_major() {
+ let context = block_on(MetalContext::new()).expect("Metal context");
+ if context.supports_mxu {
+ return;
+ }
+ for (k, group_size, method) in [(1024, 32, ScaleZeroPoint), (16384, 64, ScaleBias)] {
+ for m in [1, 3, 8, 16, 64] {
+ let input = QuantInput::<bf16>::new(m, k, 256, group_size, 4, method, 7).with_group_output();
+ let label = format!("m={m} k={k} G{group_size} {method:?}");
+ assert_close(&label, &run(&context, &input, false, None, 256), &run(&context, &input, true, None, 256));
+ }
+ let input = QuantInput::<bf16>::new(3, k, 256, group_size, 4, method, 11).with_group_output();
+ let ids = create_buffer_with_data::<Metal, u32>(&context, &(0..120).map(|i| i * 7 % 256).collect::<Vec<u32>>());
+ let label = format!("gathered k={k} G{group_size} {method:?}");
+ assert_close(
+ &label,
+ &run(&context, &input, false, Some(&ids), 40),
+ &run(&context, &input, true, Some(&ids), 40),
+ );
+ }
+}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/mod.rs
index d33e91f1b..5835d8e17 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/mod.rs
@@ -1,6 +1,7 @@
mod a8w_bench;
mod dispatch_paths_test;
mod gemm_bench;
+mod interleaved_rows_test;
mod quant_dispatch_test;
mod quant_gemm_bench;
mod qwen3_bench;