Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

delete unused matmul shapes and variants (#905)

eugene6c4fa4f04e83
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs
index 92e1b7877..ebdf056d2 100644
--- a/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs
+++ b/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs
@@ -46,7 +46,7 @@ pub enum GemmTiling {
Tile128x128x256_Simdgroups4x4,
}
-const MXU_SIMDGROUP_BLOCK_K: u32 = 32;
+pub const MXU_SIMDGROUP_BLOCK_K: u32 = 32;
impl GemmTiling {
pub const fn block_m(self) -> u32 {
@@ -140,24 +140,6 @@ impl GemmTiling {
| Self::Tile128x128x256_Simdgroups4x4
)
}
-
- pub const fn simdgroup_block_k(self) -> u32 {
- if self.is_mxu_variant() {
- MXU_SIMDGROUP_BLOCK_K
- } else {
- self.block_k()
- }
- }
-
- pub const fn fits_quant_group_size(
- self,
- group_size: u32,
- ) -> bool {
- match self {
- Self::Tile128x128x256_Simdgroups4x4 => group_size <= 64,
- _ => true,
- }
- }
}
pub const fn gemm_tiling_simdgroups_per_row(t: GemmTiling) -> u32 {
diff --git a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
index 07dc5b311..cf81fce78 100644
--- a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
@@ -30,9 +30,7 @@ impl ActivationQuantization {
emit_group_sums: bool,
code_layout: Int8CodeLayout,
) -> Option<Self> {
- (matches!(scale_group_size, 32 | 64 | 128)
- && matches!(weight_group_size, 32 | 64 | 128)
- && scale_group_size >= weight_group_size)
+ (scale_group_size == ACTIVATION_SCALE_GROUP_SIZE && (!emit_group_sums || matches!(weight_group_size, 32 | 64)))
.then_some(Self {
scale_group_size,
sum_group_size: emit_group_sums.then_some(weight_group_size),
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h
index 71eaa9816..326dc2c7d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h
@@ -432,19 +432,5 @@ METAL_FUNC void fragment_mma(thread OutputFragment& output, thread LeftFragment&
);
}
-template <class OutputFragment, class LeftFragment, class RightFragment>
-METAL_FUNC void fragment_mm(thread OutputFragment& output, thread LeftFragment& left, thread RightFragment& right) {
- static_assert(
- metal::is_same_v<typename OutputFragment::FragmentOpsType, typename LeftFragment::FragmentOpsType> &&
- metal::is_same_v<typename OutputFragment::FragmentOpsType, typename RightFragment::FragmentOpsType>,
- "fragment_mm requires output, left, and right fragments to use the same FragmentOps"
- );
- OutputFragment::FragmentOpsType::template fragment_mm<LeftFragment::MMA_TRANSPOSE, RightFragment::MMA_TRANSPOSE>(
- output,
- left,
- right
- );
-}
-
} // namespace matmul
} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h
index 77e4f7a3d..b9fd94728 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h
@@ -131,12 +131,8 @@ METAL_FUNC static void fragment_matmul(
for (ushort row = 0; row < rows; row += output_row_step) {
METAL_PRAGMA_UNROLL
for (ushort col = 0; col < output_col_count; col += output_col_step) {
- if constexpr (OUTPUT_MODE == MatmulMode::multiply) {
- matmul_paired_outputs(row, col, 0, uzu::integral_constant<MatmulMode, MatmulMode::multiply>{});
- }
METAL_PRAGMA_UNROLL
- for (ushort depth_index = OUTPUT_MODE == MatmulMode::multiply_accumulate ? 0 : 1; depth_index < depth;
- ++depth_index) {
+ for (ushort depth_index = 0; depth_index < depth; ++depth_index) {
matmul_paired_outputs(
row,
col,
@@ -156,13 +152,3 @@ METAL_FUNC static void fragment_mma(
) {
fragment_matmul<MatmulMode::multiply_accumulate, transpose_left, transpose_right>(output, left, right);
}
-
-template <bool transpose_left, bool transpose_right, class OutputFragment, class LeftFragment, class RightFragment>
-METAL_FUNC static void fragment_mm(
- thread OutputFragment& output,
- thread LeftFragment& left,
- thread RightFragment& right
-) {
- // MXU relaxed multiply is slightly faster than multiply_accumulate for pure matmul.
- fragment_matmul<MatmulMode::multiply, transpose_left, transpose_right>(output, left, right);
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_pack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_pack.h
index b1a960ca9..c324073aa 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_pack.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_pack.h
@@ -7,13 +7,12 @@ namespace gemm {
template <int bits, int word_size_bits = 8>
METAL_FUNC constexpr short get_pack_factor() {
- return (bits == 3 || bits == 5) ? 8 : (bits == 6 ? 4 : word_size_bits / bits);
+ return word_size_bits / bits;
}
template <int bits, int word_size_bits = 8>
METAL_FUNC constexpr short get_bytes_per_pack() {
- constexpr int is_power_of_two_bits = (bits & (bits - 1)) == 0;
- return is_power_of_two_bits ? (word_size_bits / 8) : (bits == 5 ? 5 : 3);
+ return word_size_bits / 8;
}
} // namespace gemm
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
index 759bf3e09..71532ffd6 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
@@ -10,47 +10,6 @@ using namespace metal;
namespace uzu {
namespace gemm {
-template <typename U>
-METAL_FUNC U uint_to_fp(uint32_t x) {
- return static_cast<U>(as_type<float>(x | 0x4B000000u) - 8388608.0f);
-}
-
-template <>
-METAL_FUNC bfloat uint_to_fp<bfloat>(uint32_t x) {
- return as_type<bfloat>(uint16_t(x | 0x4300u)) - bfloat(128.0f);
-}
-
-template <int BITS>
-METAL_FUNC float4 _uint4_to_fp4_float(uint4 n) {
- static_assert(BITS > 0 && BITS <= 23, "BITS must fit in float23 mantissa");
- constexpr uint mask = (1u << BITS) - 1u;
- n &= uint4(mask);
- return as_type<float4>(n | uint4(0x4B000000u)) - float4(8388608.0f);
-}
-
-template <typename U, int BITS>
-METAL_FUNC vec<U, 4> uint4_to_fp4(uint4 n);
-
-template <>
-METAL_FUNC float4 uint4_to_fp4<float, 4>(uint4 n) {
- return _uint4_to_fp4_float<4>(n);
-}
-
-template <>
-METAL_FUNC float4 uint4_to_fp4<float, 8>(uint4 n) {
- return _uint4_to_fp4_float<8>(n);
-}
-
-template <>
-METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 4>(uint4 n) {
- return bfloat4(_uint4_to_fp4_float<4>(n));
-}
-
-template <>
-METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 8>(uint4 n) {
- return bfloat4(_uint4_to_fp4_float<8>(n));
-}
-
template <ushort BITS>
METAL_FUNC constexpr uint symmetric_zero_point() {
return 1u << (BITS - 1);
@@ -58,21 +17,14 @@ METAL_FUNC constexpr uint symmetric_zero_point() {
template <ushort BITS>
METAL_FUNC uint decode_zero_point(uint8_t packed, uint group_index) {
- static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 zero points supported");
- if constexpr (BITS == 4) {
- return read_packed<4>(packed, group_index);
- } else {
- return uint(packed);
- }
+ static_assert(BITS == 4, "zero points are 4-bit");
+ return read_packed<4>(packed, group_index);
}
template <ushort BITS>
METAL_FUNC uint decode_zero_point(const device uint8_t* zero_points_row, uint group_index) {
- if constexpr (BITS == 4) {
- return read_packed<4>(zero_points_row, group_index);
- } else {
- return decode_zero_point<BITS>(zero_points_row[group_index], group_index);
- }
+ static_assert(BITS == 4, "zero points are 4-bit");
+ return read_packed<4>(zero_points_row, group_index);
}
UZU_CONST ushort W4_BITS = 4;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h
index 27a6fed7d..f20405cc7 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h
@@ -71,17 +71,6 @@ struct SimdgroupFragmentOps {
}
}
}
-
- template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>
- METAL_FUNC static void fragment_mm(
- thread OutputFragment& output,
- thread LeftFragment& left,
- thread RightFragment& right
- ) {
- // simdgroup_multiply is slower here.
- output.clear();
- fragment_mma<transpose_a, transpose_b>(output, left, right);
- }
};
template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y, int STRIDE_X, int STRIDE_Y, typename FragT>
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
index 23a023f29..376cd8200 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
@@ -65,38 +65,6 @@ struct SimdgroupMMA<AccumulatorT, SIMDGROUP_MMA_ROWS, SIMDGROUP_MMA_COLS, LeftT,
}
}
- template <
- typename SourcePointerType,
- typename RowStride,
- typename ColStride,
- typename RowLimit,
- typename ColLimit,
- typename RowOffset,
- typename ColOffset>
- METAL_FUNC static constexpr void load_safe(
- thread ThreadDataType& destination,
- SourcePointerType source,
- RowStride row_stride,
- ColStride col_stride,
- RowLimit row_limit,
- ColLimit col_limit,
- RowOffset row_offset = 0,
- ColOffset col_offset = 0
- ) {
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < THREAD_ELEMENT_ROWS; i++) {
- METAL_PRAGMA_UNROLL
- for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {
- if ((row_offset + i) < row_limit && (col_offset + j) < col_limit) {
- destination[i * THREAD_ELEMENT_COLS + j] =
- static_cast<AccumulatorT>(source[(row_offset + i) * row_stride + (col_offset + j) * col_stride]);
- } else {
- destination[i * THREAD_ELEMENT_COLS + j] = AccumulatorT(0);
- }
- }
- }
- }
-
template <typename DestinationPointerType, typename RowStride, typename ColStride>
METAL_FUNC static constexpr void store(
const thread ThreadDataType& source,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
index 271d99673..487a90a54 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
@@ -171,16 +171,12 @@ struct IntegerSchedule {
metadata_context.right_zero_point_group_stride
);
}
- if constexpr (RightOperand::BITS == 4) {
- const ushort packed =
- *reinterpret_cast<const device ushort*>(zero_point_row + (right_zero_point_column_start >> 1));
- uint spread = (uint(packed) | (uint(packed) << 8)) & 0x00FF00FFu;
- spread = (spread | (spread << 4)) & 0x0F0F0F0Fu;
- zero_points[tile_n] = as_type<ZeroPointVector>(spread);
- } else {
- zero_points[tile_n] =
- *reinterpret_cast<const device ZeroPointVector*>(zero_point_row + right_zero_point_column_start);
- }
+ static_assert(RightOperand::BITS == 4, "zero points are 4-bit");
+ const ushort packed =
+ *reinterpret_cast<const device ushort*>(zero_point_row + (right_zero_point_column_start >> 1));
+ uint spread = (uint(packed) | (uint(packed) << 8)) & 0x00FF00FFu;
+ spread = (spread | (spread << 4)) & 0x0F0F0F0Fu;
+ zero_points[tile_n] = as_type<ZeroPointVector>(spread);
}
if constexpr (!ALIGNED_N) {
const bool4 live =
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h
index 2ef2001b0..4adb2da49 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h
@@ -5,7 +5,6 @@
#include "../../../../common/thread_context.h"
#include "../../../../generated/gemm.h"
#include "../../../common/fragment.h"
-#include "../../../common/mxu_gemm_loop.h"
#include "../gemm_alignment.h"
#include "../operands.h"
#include "../quant_scale_bias.h"
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rs
index 59455cf65..bb635d08c 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rs
@@ -4,15 +4,10 @@ use crate::backends::common::gpu_types::gemm::GemmTiling;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
pub enum GemmSpecializationError {
- #[error("unsupported A group size {a_group_size:?}; expected None or 32/64/128")]
+ #[error("unsupported A group size {a_group_size:?}; expected None or 128")]
InvalidAGroupSize {
a_group_size: Option<u32>,
},
- #[error("simdgroup K={simdgroup_k} exceeds group size {group_size}")]
- SimdgroupKExceedsGroupSize {
- simdgroup_k: u32,
- group_size: u32,
- },
#[error("quantized B requires transposed layout")]
QuantizedRequiresTransposedB,
#[error("tiling {tiling} does not match use_mxu={use_mxu}")]
@@ -20,11 +15,4 @@ pub enum GemmSpecializationError {
tiling: GemmTiling,
use_mxu: bool,
},
- #[error(
- "MXU quantized GEMM with tile {tiling} requires group_size <= 64 (got {group_size}) due to threadgroup memory budget"
- )]
- MxuQuantTileTooLarge {
- tiling: GemmTiling,
- group_size: u32,
- },
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
index 91e9d5cd3..5905cdb15 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -59,12 +59,12 @@ VARIANTS(
GemmBPrologueKind::ScaleZeroPointDequant,
GemmBPrologueKind::ScaleSymmetricDequant)
VARIANTS(BITS, 0, 4, 8)
-VARIANTS(GROUP_SIZE, 0, 16, 32, 64, 128)
+VARIANTS(GROUP_SIZE, 0, 32, 64)
VARIANTS(
A_PROLOGUE,
GemmAPrologueKind::FullPrecision,
GemmAPrologueKind::Int8Symmetric)
-VARIANTS(A_GROUP_SIZE, 0, 32, 64, 128)
+VARIANTS(A_GROUP_SIZE, 0, 128)
CONSTRAINT(
USE_MXU ==
(GEMM_TILING == GemmTiling::Tile16x32x256_Simdgroups1x1 ||
@@ -76,37 +76,31 @@ CONSTRAINT(
CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))
CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))
CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || BT != "float")
-CONSTRAINT(
- GROUP_SIZE != 16 ||
- GEMM_TILING == GemmTiling::Tile64x64x16_Simdgroups2x2)
+CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleZeroPointDequant || BITS == 4)
+CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant || BITS == 8)
CONSTRAINT(
B_PROLOGUE == GemmBPrologueKind::FullPrecision ||
(TRANSPOSE_B &&
- (GEMM_TILING != GemmTiling::Tile64x64x16_Simdgroups2x2 ||
- GROUP_SIZE == 16)))
+ GEMM_TILING != GemmTiling::Tile64x64x16_Simdgroups2x2 &&
+ GEMM_TILING != GemmTiling::Tile64x32x32_Simdgroups2x2))
CONSTRAINT(
- B_PROLOGUE == GemmBPrologueKind::FullPrecision ||
- GEMM_TILING != GemmTiling::Tile128x128x256_Simdgroups4x4 ||
- GROUP_SIZE <= 64)
+ B_PROLOGUE != GemmBPrologueKind::FullPrecision || USE_MXU ||
+ GEMM_TILING == GemmTiling::Tile64x64x16_Simdgroups2x2 ||
+ GEMM_TILING == GemmTiling::Tile64x32x32_Simdgroups2x2)
CONSTRAINT(
- !(GEMM_TILING == GemmTiling::Tile16x32x256_Simdgroups1x1 ||
- GEMM_TILING == GemmTiling::Tile16x128x256_Simdgroups1x4) ||
+ GEMM_TILING != GemmTiling::Tile16x32x256_Simdgroups1x1 ||
(TRANSPOSE_B &&
(B_PROLOGUE == GemmBPrologueKind::FullPrecision ||
A_PROLOGUE == GemmAPrologueKind::Int8Symmetric)))
-CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || USE_MXU)
-CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || BITS == 4 || BITS == 8)
CONSTRAINT(
- A_PROLOGUE == GemmAPrologueKind::FullPrecision ||
- (GROUP_SIZE % METAL_SIMD_SIZE == 0 && GROUP_SIZE != 0))
+ GEMM_TILING != GemmTiling::Tile16x128x256_Simdgroups1x4 ||
+ (TRANSPOSE_B && B_PROLOGUE == GemmBPrologueKind::FullPrecision))
+CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || USE_MXU)
CONSTRAINT(
A_PROLOGUE == GemmAPrologueKind::FullPrecision ||
(TRANSPOSE_B && B_PROLOGUE != GemmBPrologueKind::FullPrecision))
CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || (AT == "bfloat" && DT == "bfloat"))
CONSTRAINT((A_PROLOGUE == GemmAPrologueKind::FullPrecision) == (A_GROUP_SIZE == 0))
-// The integer schedule drains one weight group at a time, so an activation
-// group narrower than the weight group has no kernel.
-CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || A_GROUP_SIZE >= GROUP_SIZE)
KERNEL(Gemm)(
const device AT* a OPTIONAL(A_PROLOGUE == GemmAPrologueKind::FullPrecision),
const device BT* b,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal
index ef20f04a3..7faab6ed8 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal
@@ -6,7 +6,7 @@ using namespace metal;
using namespace uzu::gemm;
template <typename T>
-VARIANTS(T, float, half, bfloat)
+VARIANTS(T, float, bfloat)
KERNEL(GemmSplitKReduce)(
const device T* partial_sums,
device T* output,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
index 9e5ba1d04..b7beec182 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
@@ -8,7 +8,7 @@ use metal::MTLGPUFamily;
use crate::backends::common::gpu_types::gemm::GemmTiling;
-pub(super) const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;
+const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;
const MXU_SKINNY_M_MAX: u32 = 16;
const SKINNY_SQUARE_K_MAX: u32 = 2560;
@@ -108,12 +108,9 @@ fn prefers_wide_partial_m_tile(apple_gpu_family: MTLGPUFamily) -> bool {
pub(super) fn simdgroup_quant_tile(
m: u32,
n: u32,
- group_size: u32,
apple_gpu_family: MTLGPUFamily,
) -> GemmTiling {
- if group_size < 32 {
- GemmTiling::Tile64x64x16_Simdgroups2x2
- } else if m < SIMDGROUP_QUANT_SMALL_M_MAX {
+ if m < SIMDGROUP_QUANT_SMALL_M_MAX {
if !prefers_wide_partial_m_tile(apple_gpu_family)
|| m <= SIMDGROUP_QUANT_NARROW_BLOCK_M
|| m.is_multiple_of(SIMDGROUP_QUANT_NARROW_BLOCK_M)
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
index b5c49acd9..4f8921c8e 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
@@ -117,16 +117,7 @@ impl GemmProblem {
let Some(step) = outer_block_k(shape, engine, tiling) else {
return 1;
};
- let group_size = shape.b_group_size.unwrap_or(0);
- let mut align = if engine == GemmEngine::Mxu || !shape.is_quant() {
- step
- } else {
- step.max(group_size)
- };
- if shape.b_prologue == GemmBPrologueKind::ScaleZeroPointDequant && shape.b_bits == Some(4) {
- align = align.max(2_u32.saturating_mul(group_size));
- }
- let align = align.max(ACTIVATION_SCALE_GROUP_SIZE).max(group_size);
+ let align = step.max(ACTIVATION_SCALE_GROUP_SIZE);
let target_tiles = policy::split_k_target_tiles(!shape.a_full_precision, tiling, shape.b_bits);
let mut split_k = (target_tiles / base_tiles).max(1).min((shape.k / align).max(1));
if !shape.a_full_precision && engine == GemmEngine::Mxu && tiling.block_k() != 0 {
@@ -201,7 +192,7 @@ fn mxu_is_eligible(shape: MatmulShape) -> bool {
}
shape.b_transpose
&& shape.b_leading_dimension.is_none()
- && shape.k.is_multiple_of(select_mxu_quant_tiling(shape).block_k())
+ && shape.k.is_multiple_of(policy::mxu_mn_tile(!shape.a_full_precision, shape.m, shape.n).block_k())
}
fn select_tiling(
@@ -210,24 +201,15 @@ fn select_tiling(
apple_gpu_family: MTLGPUFamily,
) -> GemmTiling {
match engine {
- GemmEngine::Simdgroup if shape.is_quant() => {
- policy::simdgroup_quant_tile(shape.m, shape.n, shape.b_group_size.unwrap_or(0), apple_gpu_family)
- },
+ GemmEngine::Simdgroup if shape.is_quant() => policy::simdgroup_quant_tile(shape.m, shape.n, apple_gpu_family),
GemmEngine::Simdgroup => policy::simdgroup_fp_tile(shape.m, shape.n, shape.k),
- GemmEngine::Mxu if !shape.a_full_precision || shape.is_quant() => select_mxu_quant_tiling(shape),
+ GemmEngine::Mxu if !shape.a_full_precision || shape.is_quant() => {
+ policy::mxu_mn_tile(!shape.a_full_precision, shape.m, shape.n)
+ },
GemmEngine::Mxu if shape.b_transpose => policy::mxu_fp_tile(shape.m, shape.n, shape.k),
GemmEngine::Mxu => policy::mxu_mn_tile(false, shape.m, shape.n),
}
}
-
-fn select_mxu_quant_tiling(shape: MatmulShape) -> GemmTiling {
- let tiling = policy::mxu_mn_tile(!shape.a_full_precision, shape.m, shape.n);
- if tiling.fits_quant_group_size(shape.b_group_size.unwrap_or(0)) {
- tiling
- } else {
- policy::MXU_DEFAULT_TILE
- }
-}
#[cfg(test)]
#[path = "../../../../../../unit/backends/metal/kernel/matmul/gemm/selection_test.rs"]
mod tests;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
index edc7f9d90..d215a1d41 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -2,7 +2,7 @@ use super::{GemmEngine, GemmPlan, error::GemmSpecializationError};
use crate::{
backends::common::{
gpu_types::gemm::{GemmAPrologueKind, GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},
- kernel::matmul::MatmulShape,
+ kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},
},
data_type::DataType,
};
@@ -54,7 +54,7 @@ impl GemmSpecialization {
pub(super) fn validate(&self) -> Result<(), GemmSpecializationError> {
let valid_activation_group = match self.a_prologue {
GemmAPrologueKind::FullPrecision => self.a_group_size.is_none(),
- GemmAPrologueKind::Int8Symmetric => matches!(self.a_group_size, Some(32 | 64 | 128)),
+ GemmAPrologueKind::Int8Symmetric => self.a_group_size == Some(ACTIVATION_SCALE_GROUP_SIZE),
};
if !valid_activation_group {
return Err(GemmSpecializationError::InvalidAGroupSize {
@@ -67,27 +67,6 @@ impl GemmSpecialization {
use_mxu: self.use_mxu,
});
}
- if self.use_mxu
- && self.b_prologue != GemmBPrologueKind::FullPrecision
- && let Some(group_size) = self.b_group_size
- && !self.tiling.fits_quant_group_size(group_size)
- {
- return Err(GemmSpecializationError::MxuQuantTileTooLarge {
- tiling: self.tiling,
- group_size,
- });
- }
- if !self.use_mxu
- && let Some(group_size) = self.b_group_size
- {
- let simdgroup_block_k = self.tiling.simdgroup_block_k();
- if simdgroup_block_k > group_size {
- return Err(GemmSpecializationError::SimdgroupKExceedsGroupSize {
- simdgroup_k: simdgroup_block_k,
- group_size,
- });
- }
- }
if self.b_prologue != GemmBPrologueKind::FullPrecision && !self.transpose_b {
return Err(GemmSpecializationError::QuantizedRequiresTransposedB);
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
index 597618e85..174d01d48 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -36,7 +36,7 @@ VARIANTS(
GemmBPrologueKind::ScaleBiasDequant,
GemmBPrologueKind::ScaleZeroPointDequant,
GemmBPrologueKind::ScaleSymmetricDequant)
-VARIANTS(GROUP_SIZE, 0, 16, 32, 64, 128)
+VARIANTS(GROUP_SIZE, 0, 32, 64)
VARIANTS(BITS, 0, 4, 8)
VARIANTS(K_SPLIT, 1, 2, 4, 8)
VARIANTS(INPUT_ALIGNED, false, true)
@@ -49,6 +49,8 @@ VARIANTS(NUM_SIMDGROUPS, 2, 4, 8)
CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))
CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))
CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || BT != "float")
+CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleZeroPointDequant || BITS == 4)
+CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant || BITS == 8)
CONSTRAINT(BITS == 0 || K_SPLIT == 1)
CONSTRAINT(BITS != 0 || (INPUT_ROW_TILE == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && GROUP_LANES == 1))
CONSTRAINT(INPUT_ROW_TILE != 1 || REDUCTION_LANES == 32)
@@ -62,16 +64,10 @@ CONSTRAINT(
CONSTRAINT(
INPUT_ROW_TILE == 1 ||
- (((B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant && BITS == 8) ||
- (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant && BITS == 4)) &&
- (GROUP_SIZE == 32 || GROUP_SIZE == 64) &&
+ (B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant && GROUP_SIZE == 64 &&
((GEMV_TILE(16, 8, 2) && INPUT_ROWS(2, 7)) ||
- (GEMV_TILE(16, 16, 4) && INPUT_ROWS(2, 6) &&
- ((GROUP_SIZE == 32 && (BITS == 4 || INPUT_ROW_TILE >= 3)) ||
- (BITS == 4 && GROUP_SIZE == 64 && INPUT_ROW_TILE >= 3))) ||
- (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant && GEMV_TILE(8, 8, 2) &&
- ((GROUP_SIZE == 32 && (INPUT_ROW_TILE == 4 || INPUT_ROW_TILE == 8)) ||
- (GROUP_SIZE == 64 && INPUT_ROW_TILE == 2))))))
+ (BITS == 4 && GEMV_TILE(16, 16, 4) && INPUT_ROWS(3, 6)) ||
+ (BITS == 4 && GEMV_TILE(8, 8, 2) && INPUT_ROW_TILE == 2))))
// Keep only selector-reachable geometry families.
CONSTRAINT(
@@ -80,9 +76,7 @@ CONSTRAINT(
(OUTPUT_ROW_TILE == 2 || OUTPUT_ROW_TILE == 4 || OUTPUT_ROW_TILE == 8 ||
(INPUT_ROW_TILE > 1 && OUTPUT_ROW_TILE == 16))) ||
(NUM_SIMDGROUPS == 4 && (OUTPUT_ROW_TILE == 8 || OUTPUT_ROW_TILE == 16 || OUTPUT_ROW_TILE == 32)) ||
- (NUM_SIMDGROUPS == 8 &&
- (OUTPUT_ROW_TILE == 16 || OUTPUT_ROW_TILE == 32 ||
- (BITS == 4 && INPUT_ALIGNED && OUTPUT_ROW_TILE == 64))))
+ (NUM_SIMDGROUPS == 8 && (OUTPUT_ROW_TILE == 16 || OUTPUT_ROW_TILE == 32)))
CONSTRAINT(
BITS == 0 || (AT == "bfloat" && DT == "bfloat") ||
(NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32))
@@ -103,12 +97,9 @@ CONSTRAINT(BITS != 4 || (GROUP_SIZE / GROUP_LANES) % 16 == 0)
CONSTRAINT(BITS != 8 || (GROUP_SIZE / GROUP_LANES) % 8 == 0)
CONSTRAINT(BITS != 0 || REDUCTION_LANES == 32)
CONSTRAINT(
- BITS == 0 || INPUT_ROW_TILE > 1 || (BITS == 4 &&
- ((GROUP_SIZE == 16 && GROUP_LANES == 1) || (GROUP_SIZE == 32 && GROUP_LANES == 2) ||
- (GROUP_SIZE == 64 && GROUP_LANES == 4) || (GROUP_SIZE == 128 && GROUP_LANES == 8))) ||
- (BITS == 8 &&
- ((GROUP_SIZE == 16 && GROUP_LANES == 2) || (GROUP_SIZE == 32 && GROUP_LANES == 4) ||
- (GROUP_SIZE == 64 && GROUP_LANES == 8) || (GROUP_SIZE == 128 && GROUP_LANES == 16))))
+ BITS == 0 || INPUT_ROW_TILE > 1 ||
+ (BITS == 4 && ((GROUP_SIZE == 32 && GROUP_LANES == 2) || (GROUP_SIZE == 64 && GROUP_LANES == 4))) ||
+ (BITS == 8 && ((GROUP_SIZE == 32 && GROUP_LANES == 4) || (GROUP_SIZE == 64 && GROUP_LANES == 8))))
KERNEL(Gemv)(
const device uint32_t* b,
const device BT* scales
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
index 5420da6c0..43dd010ff 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
@@ -35,7 +35,7 @@ pub fn gathered_tile(
m: u32,
n: u32,
) -> Option<GemvTile> {
- if !matches!(bits, 4 | 8) || !matches!(group, 16 | 32 | 64 | 128) || !(1..=8).contains(&m) {
+ if !matches!(bits, 4 | 8) || !matches!(group, 32 | 64) || !(1..=8).contains(&m) {
return None;
}
let tile = lane_default(bits, group);
@@ -105,7 +105,7 @@ pub fn select(
k: u32,
bf16_io: bool,
) -> Option<GemvTile> {
- if !matches!(bits, 4 | 8) || !matches!(group, 16 | 32 | 64 | 128) || !(1..=8).contains(&m) {
+ if !matches!(bits, 4 | 8) || !matches!(group, 32 | 64) || !(1..=8).contains(&m) {
return None;
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs
index d9f42d111..e0fc85dbc 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs
@@ -1,4 +1,4 @@
-// Qwen3.6-27B W4/ZP and W8/Symmetric, G32/G64.
+// Qwen3.6-27B W4/ZP and W8/Symmetric, G64.
// Tuned on M1, M2, M2 Pro, M3 Max, M4, M4 Pro, M5 Max.
use metal::MTLGPUFamily;
@@ -19,8 +19,7 @@ const GATE: u8 = 1 << 1;
const GATE_UP: u8 = 1 << 2;
const PROJECTION_IN: u8 = 1 << 3;
const PROJECTION_OUT: u8 = 1 << 4;
-const QKV: u8 = 1 << 5;
-const READOUT: u8 = 1 << 6;
+const READOUT: u8 = 1 << 5;
#[derive(Clone, Copy)]
struct RouteRow {
@@ -43,7 +42,6 @@ const fn shape(
(34816, 5120) => GATE_UP,
(16480, 5120) => PROJECTION_IN,
(5120, 6144) => PROJECTION_OUT,
- (8192, 5120) => QKV,
(248320, 5120) => READOUT,
_ => 0,
}
@@ -56,7 +54,7 @@ macro_rules! main_gemv { ($input:literal, $results_per_simdgroup:literal, $lanes
#[rustfmt::skip]
macro_rules! main_gemm { ($engine:ident, $tiling:ident, $split:literal) => { QmvRoute::MainGemm(GemmPlan { engine: GemmEngine::$engine, tiling: GemmTiling::$tiling, split_k: $split }) }; }
#[rustfmt::skip]
-macro_rules! qmv_format { (w4_zp_g32) => { (4, 32) }; (w4_zp_g64) => { (4, 64) }; (w8_sym_g32) => { (8, 32) }; (w8_sym_g64) => { (8, 64) }; }
+macro_rules! qmv_format { (w4_zp_g64) => { (4, 64) }; (w8_sym_g64) => { (8, 64) }; }
#[rustfmt::skip]
macro_rules! measured_device_name { (M1) => { "Apple M1" }; (M2) => { "Apple M2" }; (M2Pro) => { "Apple M2 Pro" }; (M3Max) => { "Apple M3 Max" }; (M4) => { "Apple M4" }; (M4Pro) => { "Apple M4 Pro" }; (M5Max) => { "Apple M5 Max" }; }
#[rustfmt::skip]
@@ -66,310 +64,128 @@ macro_rules! row { ($device:ident, $format:ident, $m:literal, $shapes:expr, $rou
#[rustfmt::skip]
const ROWS: &[RouteRow] = &[
- row!(M1, w4_zp_g32, 2, READOUT, main_gemv!(1, 4, 32, 2, 8)),
- row!(M1, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(2, 16, 8, 1, 2)),
- row!(M1, w4_zp_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M1, w4_zp_g32, 4, DOWN, main_gemv!(1, 4, 32, 2, 8)),
- row!(M1, w4_zp_g32, 4, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M1, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M1, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),
- row!(M1, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M1, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M1, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),
- row!(M1, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 16, 1, 4)),
- row!(M1, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),
- row!(M1, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),
- row!(M1, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M1, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
- row!(M1, w8_sym_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),
- row!(M1, w8_sym_g32, 3, PROJECTION_IN | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M1, w8_sym_g32, 4, GATE_UP, main_gemv!(1, 4, 32, 4, 8)),
- row!(M1, w8_sym_g32, 4, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M1, w8_sym_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M1, w8_sym_g32, 6, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M1, w8_sym_g32, 6, DOWN, tuned!(6, 16, 16, 1, 4)),
- row!(M1, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M1, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M1, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M1, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M1, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M1, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M1, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M2, w4_zp_g32, 2, PROJECTION_IN, main_gemv!(1, 4, 32, 2, 8)),
- row!(M2, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M2, w4_zp_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M2, w4_zp_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M2, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M2, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M2, w4_zp_g32, 7, READOUT, tuned!(4, 8, 8, 1, 2)),
- row!(M2, w4_zp_g32, 7, GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(7, 16, 8, 1, 2)),
- row!(M2, w4_zp_g32, 7, DOWN | GATE, tuned!(8, 8, 8, 1, 2)),
+ row!(M1, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M1, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 16, 1, 4)),
+ row!(M1, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 16, 1, 4)),
+ row!(M1, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 16, 1, 4)),
+ row!(M1, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 16, 1, 4)),
+ row!(M1, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M1, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M1, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M1, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M1, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M1, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M1, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
row!(M2, w4_zp_g64, 2, PROJECTION_IN, tuned!(2, 8, 8, 1, 2)),
- row!(M2, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M2, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M2, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M2, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M2, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M2, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M2, w8_sym_g32, 2, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
- row!(M2, w8_sym_g32, 2, DOWN, tuned!(2, 16, 8, 1, 2)),
- row!(M2, w8_sym_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
- row!(M2, w8_sym_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M2, w8_sym_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M2, w8_sym_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M2, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M2, w8_sym_g64, 2, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M2, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M2, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g64, 2, GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
row!(M2, w8_sym_g64, 2, DOWN, tuned!(2, 16, 8, 1, 2)),
- row!(M2, w8_sym_g64, 3, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M2, w8_sym_g64, 3, GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
row!(M2, w8_sym_g64, 3, DOWN, tuned!(3, 16, 8, 1, 2)),
- row!(M2, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M2, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M2, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M2, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g32, 2, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),
- row!(M2Pro, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g32, 3, DOWN | GATE_UP | READOUT, main_gemv!(1, 4, 32, 2, 8)),
- row!(M2Pro, w4_zp_g32, 3, PROJECTION_IN, tuned!(3, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g32, 3, GATE | PROJECTION_OUT | QKV, tuned!(3, 16, 16, 1, 4)),
- row!(M2Pro, w4_zp_g32, 4, DOWN | GATE_UP | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 2, 8)),
- row!(M2Pro, w4_zp_g32, 4, PROJECTION_IN, tuned!(4, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g32, 4, GATE, tuned!(4, 16, 16, 1, 4)),
- row!(M2Pro, w4_zp_g32, 5, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),
- row!(M2Pro, w4_zp_g32, 5, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
- row!(M2Pro, w4_zp_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
- row!(M2Pro, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g32, 6, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),
- row!(M2Pro, w4_zp_g32, 6, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
- row!(M2Pro, w4_zp_g32, 6, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
- row!(M2Pro, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g32, 7, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),
- row!(M2Pro, w4_zp_g32, 7, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
- row!(M2Pro, w4_zp_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
- row!(M2Pro, w4_zp_g32, 7, PROJECTION_IN, tuned!(7, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M2, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | READOUT, tuned!(3, 16, 8, 1, 2)),
row!(M2Pro, w4_zp_g64, 3, PROJECTION_OUT, tuned!(3, 16, 16, 1, 4)),
row!(M2Pro, w4_zp_g64, 4, GATE_UP | PROJECTION_IN | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT | QKV, tuned!(4, 16, 16, 1, 4)),
+ row!(M2Pro, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT, tuned!(4, 16, 16, 1, 4)),
row!(M2Pro, w4_zp_g64, 5, PROJECTION_IN, tuned!(5, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),
- row!(M2Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M2Pro, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
- row!(M2Pro, w8_sym_g32, 3, DOWN | GATE_UP | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
- row!(M2Pro, w8_sym_g32, 3, PROJECTION_IN, tuned!(3, 16, 8, 1, 2)),
- row!(M2Pro, w8_sym_g32, 3, GATE, tuned!(3, 16, 16, 1, 4)),
- row!(M2Pro, w8_sym_g32, 4, DOWN | GATE_UP | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
- row!(M2Pro, w8_sym_g32, 4, PROJECTION_IN, tuned!(4, 16, 8, 1, 2)),
- row!(M2Pro, w8_sym_g32, 4, GATE | PROJECTION_OUT, tuned!(4, 16, 16, 1, 4)),
- row!(M2Pro, w8_sym_g32, 5, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),
- row!(M2Pro, w8_sym_g32, 5, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
- row!(M2Pro, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
- row!(M2Pro, w8_sym_g32, 5, PROJECTION_IN, tuned!(5, 16, 8, 1, 2)),
- row!(M2Pro, w8_sym_g32, 6, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),
- row!(M2Pro, w8_sym_g32, 6, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
- row!(M2Pro, w8_sym_g32, 6, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
- row!(M2Pro, w8_sym_g32, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),
- row!(M2Pro, w8_sym_g32, 7, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
- row!(M2Pro, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
- row!(M2Pro, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M2Pro, w8_sym_g64, 3, DOWN | GATE | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M2Pro, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(5, 16, 16, 1, 4)),
+ row!(M2Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M2Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M2Pro, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M2Pro, w8_sym_g64, 3, DOWN | PROJECTION_OUT, main_gemv!(1, 4, 32, 8, 8)),
row!(M2Pro, w8_sym_g64, 3, GATE_UP | PROJECTION_IN | READOUT, tuned!(3, 16, 8, 1, 2)),
row!(M2Pro, w8_sym_g64, 4, DOWN | PROJECTION_OUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M2Pro, w8_sym_g64, 4, GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M2Pro, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M2Pro, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M2Pro, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 2, 8)),
- row!(M3Max, w4_zp_g32, 2, READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g32, 3, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),
- row!(M3Max, w4_zp_g32, 3, GATE, tuned!(3, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g32, 3, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),
- row!(M3Max, w4_zp_g32, 4, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),
- row!(M3Max, w4_zp_g32, 4, DOWN | GATE_UP | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g32, 4, GATE | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),
- row!(M3Max, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),
- row!(M3Max, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 16, 1, 4)),
- row!(M3Max, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g64, 2, GATE_UP | PROJECTION_IN | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M2Pro, w8_sym_g64, 4, GATE_UP | PROJECTION_IN | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M2Pro, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M2Pro, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M2Pro, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g64, 2, GATE_UP | PROJECTION_IN | READOUT, main_gemv!(1, 4, 32, 4, 8)),
row!(M3Max, w4_zp_g64, 2, DOWN | GATE | PROJECTION_OUT, tuned!(2, 16, 8, 1, 2)),
row!(M3Max, w4_zp_g64, 3, DOWN | GATE | PROJECTION_OUT, tuned!(3, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g64, 3, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),
- row!(M3Max, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g64, 3, GATE_UP | PROJECTION_IN | READOUT, tuned!(3, 16, 16, 1, 4)),
+ row!(M3Max, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
row!(M3Max, w4_zp_g64, 4, GATE_UP | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),
- row!(M3Max, w4_zp_g64, 5, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g64, 5, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
row!(M3Max, w4_zp_g64, 5, GATE_UP, tuned!(5, 16, 16, 1, 4)),
- row!(M3Max, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV, main_gemv!(1, 4, 32, 4, 8)),
- row!(M3Max, w8_sym_g32, 2, PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g32, 3, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),
- row!(M3Max, w8_sym_g32, 3, GATE_UP | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g32, 4, GATE, main_gemv!(1, 4, 32, 4, 8)),
- row!(M3Max, w8_sym_g32, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g32, 4, READOUT, tuned!(4, 16, 16, 1, 4)),
- row!(M3Max, w8_sym_g32, 5, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
- row!(M3Max, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
- row!(M3Max, w8_sym_g32, 5, DOWN, tuned!(5, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g32, 5, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),
- row!(M3Max, w8_sym_g32, 6, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
- row!(M3Max, w8_sym_g32, 6, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g32, 6, PROJECTION_OUT, tuned!(6, 16, 16, 1, 4)),
- row!(M3Max, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
- row!(M3Max, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M3Max, w8_sym_g64, 3, GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M3Max, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M3Max, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M3Max, w8_sym_g64, 3, GATE_UP | PROJECTION_IN | READOUT, main_gemv!(1, 4, 32, 8, 8)),
row!(M3Max, w8_sym_g64, 3, DOWN | PROJECTION_OUT, tuned!(3, 16, 8, 1, 2)),
row!(M3Max, w8_sym_g64, 4, GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 8, 8)),
- row!(M3Max, w8_sym_g64, 4, DOWN | GATE | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M4, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 2, 8)),
- row!(M4, w4_zp_g32, 2, READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M4, w4_zp_g32, 3, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),
- row!(M4, w4_zp_g32, 3, GATE, tuned!(3, 16, 8, 1, 2)),
- row!(M4, w4_zp_g32, 3, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),
- row!(M4, w4_zp_g32, 4, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),
- row!(M4, w4_zp_g32, 4, DOWN | GATE_UP | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M4, w4_zp_g32, 4, GATE | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),
- row!(M4, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M4, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),
- row!(M4, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M4, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 16, 1, 4)),
- row!(M4, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M4, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M3Max, w8_sym_g64, 4, DOWN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M3Max, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M4, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 4, 8)),
row!(M4, w4_zp_g64, 3, DOWN, tuned!(3, 16, 8, 1, 2)),
- row!(M4, w4_zp_g64, 3, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),
+ row!(M4, w4_zp_g64, 3, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 16, 1, 4)),
row!(M4, w4_zp_g64, 4, DOWN | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M4, w4_zp_g64, 4, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 16, 1, 4)),
+ row!(M4, w4_zp_g64, 4, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT, tuned!(4, 16, 16, 1, 4)),
row!(M4, w4_zp_g64, 5, DOWN | GATE_UP | PROJECTION_IN | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M4, w4_zp_g64, 5, GATE | PROJECTION_OUT | QKV, tuned!(5, 16, 16, 1, 4)),
- row!(M4, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M4, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M4, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV, main_gemv!(1, 4, 32, 4, 8)),
- row!(M4, w8_sym_g32, 2, PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M4, w8_sym_g32, 3, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),
- row!(M4, w8_sym_g32, 3, GATE_UP | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M4, w8_sym_g32, 4, GATE, main_gemv!(1, 4, 32, 4, 8)),
- row!(M4, w8_sym_g32, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 8, 1, 2)),
- row!(M4, w8_sym_g32, 4, READOUT, tuned!(4, 16, 16, 1, 4)),
- row!(M4, w8_sym_g32, 5, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
- row!(M4, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
- row!(M4, w8_sym_g32, 5, DOWN, tuned!(5, 16, 8, 1, 2)),
- row!(M4, w8_sym_g32, 5, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),
- row!(M4, w8_sym_g32, 6, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
- row!(M4, w8_sym_g32, 6, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M4, w8_sym_g32, 6, PROJECTION_OUT, tuned!(6, 16, 16, 1, 4)),
- row!(M4, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
- row!(M4, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M4, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M4, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M4, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M4, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M4, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M4, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M4Pro, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 2, 8)),
- row!(M4Pro, w4_zp_g32, 2, READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M4Pro, w4_zp_g32, 3, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),
- row!(M4Pro, w4_zp_g32, 3, GATE, tuned!(3, 16, 8, 1, 2)),
- row!(M4Pro, w4_zp_g32, 3, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),
- row!(M4Pro, w4_zp_g32, 4, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),
- row!(M4Pro, w4_zp_g32, 4, DOWN | GATE_UP | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M4Pro, w4_zp_g32, 4, GATE | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),
- row!(M4Pro, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M4Pro, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),
- row!(M4Pro, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M4Pro, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 16, 1, 4)),
- row!(M4Pro, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M4Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),
- row!(M4Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),
+ row!(M4, w4_zp_g64, 5, GATE | PROJECTION_OUT, tuned!(5, 16, 16, 1, 4)),
+ row!(M4, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M4, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M4, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M4, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M4, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M4, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M4, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M4, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M4Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 4, 8)),
+ row!(M4Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 16, 1, 4)),
row!(M4Pro, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M4Pro, w4_zp_g64, 4, PROJECTION_IN | QKV, tuned!(4, 16, 16, 1, 4)),
+ row!(M4Pro, w4_zp_g64, 4, PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),
row!(M4Pro, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M4Pro, w4_zp_g64, 5, PROJECTION_IN | QKV, tuned!(5, 16, 16, 1, 4)),
- row!(M4Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M4Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV, main_gemv!(1, 4, 32, 4, 8)),
- row!(M4Pro, w8_sym_g32, 2, PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g32, 3, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),
- row!(M4Pro, w8_sym_g32, 3, GATE_UP | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g32, 4, GATE, main_gemv!(1, 4, 32, 4, 8)),
- row!(M4Pro, w8_sym_g32, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g32, 4, READOUT, tuned!(4, 16, 16, 1, 4)),
- row!(M4Pro, w8_sym_g32, 5, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
- row!(M4Pro, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
- row!(M4Pro, w8_sym_g32, 5, DOWN, tuned!(5, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g32, 5, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),
- row!(M4Pro, w8_sym_g32, 6, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),
- row!(M4Pro, w8_sym_g32, 6, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g32, 6, PROJECTION_OUT, tuned!(6, 16, 16, 1, 4)),
- row!(M4Pro, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),
- row!(M4Pro, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M4Pro, w4_zp_g64, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),
+ row!(M4Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M4Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT, main_gemv!(1, 4, 32, 8, 8)),
row!(M4Pro, w8_sym_g64, 2, READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M4Pro, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M5Max, w4_zp_g32, 2, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(2, 16, 8, 1, 2)),
- row!(M5Max, w4_zp_g32, 2, GATE_UP | READOUT, tuned!(2, 16, 16, 1, 4)),
- row!(M5Max, w4_zp_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M5Max, w4_zp_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M5Max, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M5Max, w4_zp_g32, 6, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
- row!(M5Max, w4_zp_g32, 6, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
- row!(M5Max, w4_zp_g32, 6, GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),
- row!(M5Max, w4_zp_g32, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),
- row!(M5Max, w4_zp_g32, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
- row!(M5Max, w4_zp_g32, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
- row!(M5Max, w4_zp_g32, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
- row!(M5Max, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M5Max, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M5Max, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M5Max, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M4Pro, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
+ row!(M4Pro, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
+ row!(M5Max, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M5Max, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M5Max, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M5Max, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
row!(M5Max, w4_zp_g64, 6, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),
- row!(M5Max, w4_zp_g64, 6, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
+ row!(M5Max, w4_zp_g64, 6, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
row!(M5Max, w4_zp_g64, 6, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
row!(M5Max, w4_zp_g64, 6, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
row!(M5Max, w4_zp_g64, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),
- row!(M5Max, w4_zp_g64, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
+ row!(M5Max, w4_zp_g64, 7, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
row!(M5Max, w4_zp_g64, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
row!(M5Max, w4_zp_g64, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
- row!(M5Max, w8_sym_g32, 2, GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 4, 8)),
- row!(M5Max, w8_sym_g32, 2, DOWN | GATE | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g32, 4, PROJECTION_IN, tuned!(2, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),
- row!(M5Max, w8_sym_g32, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),
- row!(M5Max, w8_sym_g32, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
- row!(M5Max, w8_sym_g32, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
- row!(M5Max, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
- row!(M5Max, w8_sym_g64, 2, GATE | GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 8, 8)),
- row!(M5Max, w8_sym_g64, 2, DOWN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g64, 4, GATE | PROJECTION_IN, tuned!(2, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g64, 6, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
- row!(M5Max, w8_sym_g64, 6, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
+ row!(M5Max, w8_sym_g64, 2, GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 8, 8)),
+ row!(M5Max, w8_sym_g64, 2, DOWN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g64, 4, PROJECTION_IN, tuned!(2, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
+ row!(M5Max, w8_sym_g64, 6, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
row!(M5Max, w8_sym_g64, 6, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
row!(M5Max, w8_sym_g64, 6, GATE_UP | PROJECTION_IN | READOUT, tuned!(6, 16, 8, 1, 2)),
row!(M5Max, w8_sym_g64, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),
- row!(M5Max, w8_sym_g64, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
- row!(M5Max, w8_sym_g64, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
+ row!(M5Max, w8_sym_g64, 7, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
row!(M5Max, w8_sym_g64, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
];
fn qmv_format(shape: &MatmulShape) -> Option<(u32, u32)> {
match (shape.b_prologue, shape.b_bits, shape.b_group_size) {
- (GemmBPrologueKind::ScaleZeroPointDequant, Some(4), Some(group @ (32 | 64))) => Some((4, group)),
- (GemmBPrologueKind::ScaleSymmetricDequant, Some(8), Some(group @ (32 | 64))) => Some((8, group)),
+ (GemmBPrologueKind::ScaleZeroPointDequant, Some(4), Some(64)) => Some((4, 64)),
+ (GemmBPrologueKind::ScaleSymmetricDequant, Some(8), Some(64)) => Some((8, 64)),
_ => None,
}
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
index f56d0836c..ec8d119ac 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
@@ -294,9 +294,7 @@ mod quantize {
}
#[uzu_test]
- fn quantize_scale_g32_and_g64_match_cpu() {
- check_quantize(32, false, None, Int8CodeLayout::Sequential);
- check_quantize(64, false, None, Int8CodeLayout::Sequential);
+ fn quantize_nibble_grouped_matches_cpu() {
check_quantize(128, false, None, Int8CodeLayout::GroupedByNibble);
}
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs
index 657f20156..8f0e8bf0f 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs
@@ -278,19 +278,13 @@ fn bench_a8w(c: &mut Criterion) {
}
let hadamard = ActivationTransform::<Metal>::input_rht(&context, DataType::BF16, true).expect("hadamard kernel");
- for bits in [8u32, 4u32] {
- let prepare = ActivationTransform::<Metal>::quantize(
- &context,
- DataType::BF16,
- ActivationQuantization::new(
- 128,
- 128,
- false,
- Int8CodeLayout::for_right_bits(bits).expect("W4/W8 benchmark"),
- )
+ let bits = 8;
+ let prepare = ActivationTransform::<Metal>::quantize(
+ &context,
+ DataType::BF16,
+ ActivationQuantization::new(128, 128, false, Int8CodeLayout::for_right_bits(bits).expect("W8 benchmark"))
.expect("supported activation quantization"),
- )
- .expect("prepare kernel");
- bench_bits(c, &context, &prepare, &hadamard, bits);
- }
+ )
+ .expect("prepare kernel");
+ bench_bits(c, &context, &prepare, &hadamard, bits);
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
index 2924eacf8..0e7e48bc6 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
@@ -293,8 +293,7 @@ fn gemv_gather() {
(4, QuantizationMethod::ScaleBias, false),
(4, QuantizationMethod::ScaleZeroPoint, false),
(4, QuantizationMethod::ScaleZeroPoint, true),
- (4, QuantizationMethod::ScaleSymmetric, false),
- (8, QuantizationMethod::ScaleZeroPoint, false),
+ (8, QuantizationMethod::ScaleSymmetric, false),
] {
let mut input = QuantInput::new(8, 128, 64, 32, bits, method, 0x5EED);
if signed_codes {
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
index 494469be9..95cbc385f 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
@@ -124,19 +124,14 @@ fn run_parity<T: ArrayElement + Float + Debug + Display>(
#[rstest]
#[test_attr(uzu_test)]
-#[case::gs16_4bit_zp_prefill(64, 256, 64, 16, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::gs32_4bit_mlx_prefill(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
#[case::gs64_4bit_mlx_prefill(64, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]
-#[case::gs128_4bit_mlx_prefill(64, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]
-#[case::gs32_4bit_symmetric_prefill(64, 256, 64, 32, 4, QuantizationMethod::ScaleSymmetric)]
#[case::gs32_4bit_mlx_decode(8, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
-#[case::gs16_4bit_zp_decode(8, 256, 64, 16, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::gs64_4bit_zp_decode(8, 256, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::gs32_unaligned_n(64, 256, 96, 32, 4, QuantizationMethod::ScaleBias)]
#[case::gs32_narrow_n_tail_zp(64, 256, 72, 32, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::gs32_narrow_n_tail_mlx(64, 256, 72, 32, 4, QuantizationMethod::ScaleBias)]
-#[case::gs32_narrow_n_tail_sym(64, 256, 40, 32, 4, QuantizationMethod::ScaleSymmetric)]
-#[case::gs16_k_tail_mlx(64, 272, 64, 16, 4, QuantizationMethod::ScaleBias)]
+#[case::gs32_narrow_n_tail_sym(64, 256, 40, 32, 8, QuantizationMethod::ScaleSymmetric)]
fn parity_bf16(
#[case] m: u32,
#[case] k: u32,
@@ -151,8 +146,6 @@ fn parity_bf16(
#[rstest]
#[test_attr(uzu_test)]
#[case::gs32_8bit_mlx_prefill(64, 256, 64, 32, 8, QuantizationMethod::ScaleBias)]
-#[case::gs64_8bit_zp_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
-#[case::gs128_8bit_zp_prefill(64, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]
#[case::gs64_8bit_symmetric_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]
fn parity_bf16_8bit_splitk(
#[case] m: u32,
@@ -169,11 +162,7 @@ fn parity_bf16_8bit_splitk(
#[test_attr(uzu_test)]
#[case::m1_gs32_4bit_mlx(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
#[case::m1_gs64_4bit_mlx(1, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]
-#[case::m1_gs128_4bit_mlx(1, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]
#[case::m1_gs32_4bit_zp(1, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]
-#[case::m1_gs64_8bit_zp(1, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
-#[case::m1_gs128_8bit_mlx(1, 256, 64, 128, 8, QuantizationMethod::ScaleBias)]
-#[case::m1_gs32_4bit_sym(1, 256, 64, 32, 4, QuantizationMethod::ScaleSymmetric)]
#[case::m1_gs64_8bit_sym(1, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]
#[case::m2_gs32_4bit_mlx(2, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
#[case::m4_gs32_4bit_zp(4, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]
@@ -276,14 +265,13 @@ fn parity_bf16_gemv_qmv_fused_scale_bias() {
#[test_attr(uzu_test)]
#[case::gs64_4bit(1, 96, 64, 64, 4, QuantizationMethod::ScaleBias)]
#[case::gs64_4bit_zp(2, 96, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
-#[case::gs128_8bit(1, 192, 64, 128, 8, QuantizationMethod::ScaleBias)]
// Whole quantization groups with an unaligned K block.
#[case::gs64_4bit_k1152_zp(1, 1152, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
-#[case::gs64_8bit_k1152_zp(1, 1152, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs64_8bit_k1152_sym(1, 1152, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]
// Same, plus a final partial quantization group (1056 = 16 * 64 + 32).
#[case::gs64_4bit_k1056_partial_group(1, 1056, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::gs64_4bit_k514_unaligned_rows(2, 514, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
-#[case::gs64_8bit_k257_unaligned_rows(2, 257, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs64_8bit_k257_unaligned_rows(2, 257, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]
fn parity_gemv_partial_group_bf16(
#[case] m: u32,
#[case] k: u32,
@@ -463,7 +451,6 @@ fn quant_gemm_accumulate_returns_unsupported_dop() {
#[rstest]
#[test_attr(uzu_test)]
#[case::output_group_single_group_gs64(OutputGroup, 64, 64, 64, 4, ScaleZeroPoint, 64)]
-#[case::output_group_single_group_gs128(OutputGroup, 128, 64, 128, 4, ScaleZeroPoint, 64)]
#[case::output_group_odd_groups(OutputGroup, 192, 64, 64, 4, ScaleZeroPoint, 64)]
#[case::group_output_padded_n(GroupOutput, 192, 66, 64, 4, ScaleZeroPoint, 66)]
#[case::output_group_bias_prefix(OutputGroup, 128, 12, 32, 8, ScaleBias, 4)]
@@ -517,13 +504,9 @@ fn quant_gemm_parameter_layout_prefix_matches_cpu(
#[test_attr(uzu_test)]
#[case::gs32_4bit_mlx(128, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
#[case::gs64_4bit_mlx(128, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]
-#[case::gs128_4bit_mlx(128, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]
#[case::small_m_4bit_mlx(24, 256, 512, 64, 4, QuantizationMethod::ScaleBias)]
#[case::gs32_4bit_zp(128, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]
-#[case::gs64_8bit_zp(128, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
-#[case::gs128_8bit_zp(128, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]
#[case::gs64_4bit_zp_g1(128, 64, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
-#[case::gs128_4bit_zp_g1(128, 128, 64, 128, 4, QuantizationMethod::ScaleZeroPoint)]
fn mxu_quant_parity_bf16(
#[case] m: u32,
#[case] k: u32,
@@ -550,18 +533,11 @@ fn mxu_quant_parity_bf16(
#[rstest]
#[test_attr(uzu_test)]
-#[case::sym_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleSymmetric)]
#[case::sym_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]
-#[case::sym_w4_gs128(32u32, 128u32, 4u32, QuantizationMethod::ScaleSymmetric)]
#[case::bias_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleBias)]
#[case::bias_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleBias)]
-#[case::bias_w4_gs128(32u32, 128u32, 4u32, QuantizationMethod::ScaleBias)]
#[case::zp_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleZeroPoint)]
-#[case::zp_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleZeroPoint)]
-#[case::zp_w4_gs128(32u32, 128u32, 4u32, QuantizationMethod::ScaleZeroPoint)]
#[case::m1_bias_w4_gs64(1u32, 64u32, 4u32, QuantizationMethod::ScaleBias)]
-#[case::m1_zp_w8_gs32(1u32, 32u32, 8u32, QuantizationMethod::ScaleZeroPoint)]
-#[case::m1_sym_w4_gs128(1u32, 128u32, 4u32, QuantizationMethod::ScaleSymmetric)]
fn a8w_mxu_parity_bf16(
#[case] m: u32,
#[case] weight_gs: u32,
@@ -600,9 +576,10 @@ fn a8w_independent_activation_group_parity_bf16(#[case] m: u32) {
return;
}
- for (activation_group_size, weight_group_size) in [(32u32, 32u32), (64, 64), (128, 32), (128, 64), (128, 128)] {
+ for (activation_group_size, weight_group_size) in
+ [(ACTIVATION_SCALE_GROUP_SIZE, 32u32), (ACTIVATION_SCALE_GROUP_SIZE, 64)]
+ {
for (bits, method) in [
- (4, QuantizationMethod::ScaleSymmetric),
(8, QuantizationMethod::ScaleSymmetric),
(8, QuantizationMethod::ScaleBias),
(4, QuantizationMethod::ScaleZeroPoint),
@@ -631,27 +608,24 @@ fn a8w_independent_activation_group_parity_bf16(#[case] m: u32) {
#[case::m1(1)]
#[case::m16(16)]
#[case::m33(33)]
-fn a8w8_zero_point_tail_parity(#[case] m: u32) {
+fn a8w4_zero_point_tail_parity(#[case] m: u32) {
let context = MetalContext::new().expect("Metal context");
if !context.supports_mxu {
return;
}
- let input = QuantInput::<bf16>::new(m, 256, 72, 32, 8, QuantizationMethod::ScaleZeroPoint, 0)
+ let input = QuantInput::<bf16>::new(m, 256, 72, 32, 4, QuantizationMethod::ScaleZeroPoint, 0)
.with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE, Some(32));
let reference = run_quant_cpu::<bf16>(&input);
let actual = run_quant_metal(&context, &input.with_group_output(), Some(GemmEngine::Mxu));
- assert_parity::<bf16>(&format!("A8W8 ZP N-tail m={m}"), &reference, &actual, 0.08, 0.8);
+ assert_parity::<bf16>(&format!("A8W4 ZP N-tail m={m}"), &reference, &actual, 0.08, 0.8);
}
#[rstest]
#[test_attr(uzu_test)]
-#[case::w4_sym(4u32, QuantizationMethod::ScaleSymmetric, 256u32)]
#[case::w4_bias(4u32, QuantizationMethod::ScaleBias, 256u32)]
#[case::w4_zp(4u32, QuantizationMethod::ScaleZeroPoint, 256u32)]
#[case::w8_sym(8u32, QuantizationMethod::ScaleSymmetric, 256u32)]
#[case::w8_bias(8u32, QuantizationMethod::ScaleBias, 256u32)]
-#[case::w8_zp(8u32, QuantizationMethod::ScaleZeroPoint, 256u32)]
-#[case::w8_zp_tail(8u32, QuantizationMethod::ScaleZeroPoint, 224u32)]
fn signed_weights_full_precision_activations_parity_bf16(
#[case] bits: u32,
#[case] method: QuantizationMethod,
@@ -677,9 +651,7 @@ fn signed_weights_full_precision_activations_parity_bf16(
#[rstest]
#[test_attr(uzu_test)]
-#[case::w4_bias(4u32, false)]
#[case::w8_bias(8u32, false)]
-#[case::w4_rht_bias(4u32, true)]
#[case::w8_rht_bias(8u32, true)]
fn a8w_mxu_output_bias_parity_bf16(
#[case] bits: u32,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rs
index ae0605033..1dc874b35 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rs
@@ -73,6 +73,4 @@ fn bench_unified_quantized_gemm(c: &mut Criterion) {
let context = crate::tests::util::shared_metal_context();
bench_unified_quant_typed::<bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
bench_unified_quant_typed::<bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
- bench_unified_quant_typed::<bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
- bench_unified_quant_typed::<bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs
index 79be8b9d2..c6ef41e56 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs
@@ -61,8 +61,6 @@ fn bench_gemv(c: &mut Criterion) {
bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);
bench_gemv_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
- bench_gemv_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
- bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);
bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);
});
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/qwen3_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/qwen3_bench.rs
index ddd0f9873..8a2f589c5 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/qwen3_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/qwen3_bench.rs
@@ -56,6 +56,6 @@ fn bench_qwen3_layers_typed<T: ArrayElement + Float>(
#[uzu_bench]
fn bench_qwen3_layers(c: &mut Criterion) {
let context = crate::tests::util::shared_metal_context();
- bench_qwen3_layers_typed::<bf16>(c, &context, "ScaleBias_BF16_gs128_4b", 128, 4, QuantizationMethod::ScaleBias);
- bench_qwen3_layers_typed::<bf16>(c, &context, "ZP_BF16_gs128_4b", 128, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_qwen3_layers_typed::<bf16>(c, &context, "ScaleBias_BF16_gs64_4b", 64, 4, QuantizationMethod::ScaleBias);
+ bench_qwen3_layers_typed::<bf16>(c, &context, "ZP_BF16_gs64_4b", 64, 4, QuantizationMethod::ScaleZeroPoint);
}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
index 563d2d8c2..7b7ff7826 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
@@ -71,18 +71,17 @@ fn policy_boundaries_are_preserved() {
assert_eq!(policy::mxu_fp_tile(m, n, k), expected);
}
- for (m, n, group_size, family, expected) in [
- (16, 4096, 31, MTLGPUFamily::Apple7, Tile64x64x16_Simdgroups2x2),
- (16, 4096, 32, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),
- (8, 4096, 32, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),
- (9, 4096, 32, MTLGPUFamily::Apple8, Tile32x32x32_Simdgroups2x2),
- (9, 4096, 32, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),
- (31, 4096, 32, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),
- (32, 4096, 32, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),
- (64, 6143, 32, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),
- (64, 6144, 32, MTLGPUFamily::Apple9, Tile64x64x32_Simdgroups2x2),
+ for (m, n, family, expected) in [
+ (16, 4096, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),
+ (8, 4096, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),
+ (9, 4096, MTLGPUFamily::Apple8, Tile32x32x32_Simdgroups2x2),
+ (9, 4096, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),
+ (31, 4096, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),
+ (32, 4096, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),
+ (64, 6143, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),
+ (64, 6144, MTLGPUFamily::Apple9, Tile64x64x32_Simdgroups2x2),
] {
- assert_eq!(policy::simdgroup_quant_tile(m, n, group_size, family), expected);
+ assert_eq!(policy::simdgroup_quant_tile(m, n, family), expected);
}
}
@@ -118,11 +117,6 @@ fn selection_fallbacks_and_split_k_are_preserved() {
assert_eq!(problem(invalid_layout, DataType::BF16).select_plan().engine, Mxu);
assert_eq!(problem(quant(shape(64, 4096, 4095)), DataType::BF16).select_plan().engine, Simdgroup);
- let mut large_group = quant(shape(512, 4096, 4096));
- large_group.a_full_precision = false;
- large_group.b_group_size = Some(128);
- assert_eq!(problem(large_group, DataType::BF16).select_plan().tiling, Tile64x64x256_Simdgroups2x2);
-
for (m, n, k, expected_tiling, expected_split_k) in [
(16, 4096, 4096, Tile16x32x256_Simdgroups1x1, 8),
(16, 1024, 4096, Tile16x32x256_Simdgroups1x1, 16),
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
index 9121e6a80..12fa48313 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
@@ -14,7 +14,7 @@ use crate::{
data_type::DataType,
};
-const FROZEN_PLAN_FINGERPRINT: u64 = 5_839_212_743_558_880_136;
+const FROZEN_PLAN_FINGERPRINT: u64 = 10_794_242_881_707_148_570;
const DEVICES: [(&str, &str, u32, MTLGPUFamily, bool); 7] = [
("m1", "Apple M1", 8, MTLGPUFamily::Apple7, false),
@@ -25,19 +25,16 @@ const DEVICES: [(&str, &str, u32, MTLGPUFamily, bool); 7] = [
("m4-pro", "Apple M4 Pro", 20, MTLGPUFamily::Apple9, false),
("m5-max", "Apple M5 Max", 40, MTLGPUFamily::Apple10, true),
];
-const FORMATS: [(&str, u32, u32, GemmBPrologueKind); 4] = [
- ("W4/ZP G32", 4, 32, GemmBPrologueKind::ScaleZeroPointDequant),
+const FORMATS: [(&str, u32, u32, GemmBPrologueKind); 2] = [
("W4/ZP G64", 4, 64, GemmBPrologueKind::ScaleZeroPointDequant),
- ("W8/Symmetric G32", 8, 32, GemmBPrologueKind::ScaleSymmetricDequant),
("W8/Symmetric G64", 8, 64, GemmBPrologueKind::ScaleSymmetricDequant),
];
-const SHAPES: [(&str, u32, u32); 7] = [
+const SHAPES: [(&str, u32, u32); 6] = [
("down", 5120, 17408),
("gate", 6144, 5120),
("gate-up", 34816, 5120),
("projection-in", 16480, 5120),
("projection-out", 5120, 6144),
- ("qkv", 8192, 5120),
("readout", 248320, 5120),
];
@@ -74,7 +71,7 @@ fn table_is_complete_and_fingerprint_is_stable() {
for &(device_label, device_name, gpu_core_count, apple_gpu_family, supports_mxu) in &DEVICES {
for &(format_name, bits, group, prologue) in &FORMATS {
for m in 2..=7 {
- for &(shape_name, n, k) in &SHAPES {
+ for &(shape_name, n, k) in SHAPES.iter().filter(|shape| bits == 4 || shape.0 != "gate") {
let mask = shape(n, k);
let matches: Vec<_> = ROWS
.iter()
@@ -124,7 +121,7 @@ fn table_is_complete_and_fingerprint_is_stable() {
}
assert!(matched_rows.into_iter().all(|matched| matched), "route table contains an orphaned row");
canonical.sort();
- assert_eq!(canonical.len(), 1176);
+ assert_eq!(canonical.len(), 462);
assert_eq!(xxh3_64(canonical.join("\n").as_bytes()), FROZEN_PLAN_FINGERPRINT);
}
@@ -158,7 +155,7 @@ fn exact_lookup_rejects_non_matrix_inputs() {
rht.n -= 1;
assert!(GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).is_none());
- let p = problem(7, 6144, 5120, 8, 64, GemmBPrologueKind::ScaleSymmetricDequant);
+ let p = problem(7, 5120, 17408, 8, 64, GemmBPrologueKind::ScaleSymmetricDequant);
assert!(matches!(route("Apple M5 Max", MTLGPUFamily::Apple10, true, &p, true), Some(QmvRoute::MainGemm(_))));
assert_eq!(route("Apple M5 Max", MTLGPUFamily::Apple10, false, &p, true), None);
}
@@ -212,7 +209,7 @@ fn normal_routing_handles_inputs_outside_the_frozen_matrix() {
#[uzu_test]
fn family_lookup_requires_one_unanimous_route() {
- let m1_route = problem(4, 8192, 5120, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
+ let m1_route = problem(4, 34816, 5120, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
let unanimous = problem(6, 5120, 17408, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
let families: [(&MatmulShape, MTLGPUFamily, bool, &str, &[&str]); 4] = [
(&m1_route, MTLGPUFamily::Apple7, false, "Apple M1", &["Apple M1 Pro", "Apple M1 Max", "Apple M1 Ultra"]),