A high-performance inference engine for AI models
delete unused matmul shapes and variants (#905)
eugene6c4fa4f04e83
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/common/gpu_types/gemm.rsindex 92e1b7877..ebdf056d2 100644--- a/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs+++ b/crates/uzu-engine/src/backends/common/gpu_types/gemm.rs@@ -46,7 +46,7 @@ pub enum GemmTiling {Tile128x128x256_Simdgroups4x4,}-const MXU_SIMDGROUP_BLOCK_K: u32 = 32;+pub const MXU_SIMDGROUP_BLOCK_K: u32 = 32;impl GemmTiling {pub const fn block_m(self) -> u32 {@@ -140,24 +140,6 @@ impl GemmTiling {| Self::Tile128x128x256_Simdgroups4x4)}-- pub const fn simdgroup_block_k(self) -> u32 {- if self.is_mxu_variant() {- MXU_SIMDGROUP_BLOCK_K- } else {- self.block_k()- }- }-- pub const fn fits_quant_group_size(- self,- group_size: u32,- ) -> bool {- match self {- Self::Tile128x128x256_Simdgroups4x4 => group_size <= 64,- _ => true,- }- }}pub const fn gemm_tiling_simdgroups_per_row(t: GemmTiling) -> u32 {diff --git a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rsindex 07dc5b311..cf81fce78 100644--- a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs+++ b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs@@ -30,9 +30,7 @@ impl ActivationQuantization {emit_group_sums: bool,code_layout: Int8CodeLayout,) -> Option<Self> {- (matches!(scale_group_size, 32 | 64 | 128)- && matches!(weight_group_size, 32 | 64 | 128)- && scale_group_size >= weight_group_size)+ (scale_group_size == ACTIVATION_SCALE_GROUP_SIZE && (!emit_group_sums || matches!(weight_group_size, 32 | 64))).then_some(Self {scale_group_size,sum_group_size: emit_group_sums.then_some(weight_group_size),diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.hindex 71eaa9816..326dc2c7d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h@@ -432,19 +432,5 @@ METAL_FUNC void fragment_mma(thread OutputFragment& output, thread LeftFragment&);}-template <class OutputFragment, class LeftFragment, class RightFragment>-METAL_FUNC void fragment_mm(thread OutputFragment& output, thread LeftFragment& left, thread RightFragment& right) {- static_assert(- metal::is_same_v<typename OutputFragment::FragmentOpsType, typename LeftFragment::FragmentOpsType> &&- metal::is_same_v<typename OutputFragment::FragmentOpsType, typename RightFragment::FragmentOpsType>,- "fragment_mm requires output, left, and right fragments to use the same FragmentOps"- );- OutputFragment::FragmentOpsType::template fragment_mm<LeftFragment::MMA_TRANSPOSE, RightFragment::MMA_TRANSPOSE>(- output,- left,- right- );-}-} // namespace matmul} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.hindex 77e4f7a3d..b9fd94728 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h@@ -131,12 +131,8 @@ METAL_FUNC static void fragment_matmul(for (ushort row = 0; row < rows; row += output_row_step) {METAL_PRAGMA_UNROLLfor (ushort col = 0; col < output_col_count; col += output_col_step) {- if constexpr (OUTPUT_MODE == MatmulMode::multiply) {- matmul_paired_outputs(row, col, 0, uzu::integral_constant<MatmulMode, MatmulMode::multiply>{});- }METAL_PRAGMA_UNROLL- for (ushort depth_index = OUTPUT_MODE == MatmulMode::multiply_accumulate ? 0 : 1; depth_index < depth;- ++depth_index) {+ for (ushort depth_index = 0; depth_index < depth; ++depth_index) {matmul_paired_outputs(row,col,@@ -156,13 +152,3 @@ METAL_FUNC static void fragment_mma() {fragment_matmul<MatmulMode::multiply_accumulate, transpose_left, transpose_right>(output, left, right);}--template <bool transpose_left, bool transpose_right, class OutputFragment, class LeftFragment, class RightFragment>-METAL_FUNC static void fragment_mm(- thread OutputFragment& output,- thread LeftFragment& left,- thread RightFragment& right-) {- // MXU relaxed multiply is slightly faster than multiply_accumulate for pure matmul.- fragment_matmul<MatmulMode::multiply, transpose_left, transpose_right>(output, left, right);-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_pack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_pack.hindex b1a960ca9..c324073aa 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_pack.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_pack.h@@ -7,13 +7,12 @@ namespace gemm {template <int bits, int word_size_bits = 8>METAL_FUNC constexpr short get_pack_factor() {- return (bits == 3 || bits == 5) ? 8 : (bits == 6 ? 4 : word_size_bits / bits);+ return word_size_bits / bits;}template <int bits, int word_size_bits = 8>METAL_FUNC constexpr short get_bytes_per_pack() {- constexpr int is_power_of_two_bits = (bits & (bits - 1)) == 0;- return is_power_of_two_bits ? (word_size_bits / 8) : (bits == 5 ? 5 : 3);+ return word_size_bits / 8;}} // namespace gemmdiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.hindex 759bf3e09..71532ffd6 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h@@ -10,47 +10,6 @@ using namespace metal;namespace uzu {namespace gemm {-template <typename U>-METAL_FUNC U uint_to_fp(uint32_t x) {- return static_cast<U>(as_type<float>(x | 0x4B000000u) - 8388608.0f);-}--template <>-METAL_FUNC bfloat uint_to_fp<bfloat>(uint32_t x) {- return as_type<bfloat>(uint16_t(x | 0x4300u)) - bfloat(128.0f);-}--template <int BITS>-METAL_FUNC float4 _uint4_to_fp4_float(uint4 n) {- static_assert(BITS > 0 && BITS <= 23, "BITS must fit in float23 mantissa");- constexpr uint mask = (1u << BITS) - 1u;- n &= uint4(mask);- return as_type<float4>(n | uint4(0x4B000000u)) - float4(8388608.0f);-}--template <typename U, int BITS>-METAL_FUNC vec<U, 4> uint4_to_fp4(uint4 n);--template <>-METAL_FUNC float4 uint4_to_fp4<float, 4>(uint4 n) {- return _uint4_to_fp4_float<4>(n);-}--template <>-METAL_FUNC float4 uint4_to_fp4<float, 8>(uint4 n) {- return _uint4_to_fp4_float<8>(n);-}--template <>-METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 4>(uint4 n) {- return bfloat4(_uint4_to_fp4_float<4>(n));-}--template <>-METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 8>(uint4 n) {- return bfloat4(_uint4_to_fp4_float<8>(n));-}-template <ushort BITS>METAL_FUNC constexpr uint symmetric_zero_point() {return 1u << (BITS - 1);@@ -58,21 +17,14 @@ METAL_FUNC constexpr uint symmetric_zero_point() {template <ushort BITS>METAL_FUNC uint decode_zero_point(uint8_t packed, uint group_index) {- static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 zero points supported");- if constexpr (BITS == 4) {- return read_packed<4>(packed, group_index);- } else {- return uint(packed);- }+ static_assert(BITS == 4, "zero points are 4-bit");+ return read_packed<4>(packed, group_index);}template <ushort BITS>METAL_FUNC uint decode_zero_point(const device uint8_t* zero_points_row, uint group_index) {- if constexpr (BITS == 4) {- return read_packed<4>(zero_points_row, group_index);- } else {- return decode_zero_point<BITS>(zero_points_row[group_index], group_index);- }+ static_assert(BITS == 4, "zero points are 4-bit");+ return read_packed<4>(zero_points_row, group_index);}UZU_CONST ushort W4_BITS = 4;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.hindex 27a6fed7d..f20405cc7 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h@@ -71,17 +71,6 @@ struct SimdgroupFragmentOps {}}}-- template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>- METAL_FUNC static void fragment_mm(- thread OutputFragment& output,- thread LeftFragment& left,- thread RightFragment& right- ) {- // simdgroup_multiply is slower here.- output.clear();- fragment_mma<transpose_a, transpose_b>(output, left, right);- }};template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y, int STRIDE_X, int STRIDE_Y, typename FragT>diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.hindex 23a023f29..376cd8200 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h@@ -65,38 +65,6 @@ struct SimdgroupMMA<AccumulatorT, SIMDGROUP_MMA_ROWS, SIMDGROUP_MMA_COLS, LeftT,}}- template <- typename SourcePointerType,- typename RowStride,- typename ColStride,- typename RowLimit,- typename ColLimit,- typename RowOffset,- typename ColOffset>- METAL_FUNC static constexpr void load_safe(- thread ThreadDataType& destination,- SourcePointerType source,- RowStride row_stride,- ColStride col_stride,- RowLimit row_limit,- ColLimit col_limit,- RowOffset row_offset = 0,- ColOffset col_offset = 0- ) {- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < THREAD_ELEMENT_ROWS; i++) {- METAL_PRAGMA_UNROLL- for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {- if ((row_offset + i) < row_limit && (col_offset + j) < col_limit) {- destination[i * THREAD_ELEMENT_COLS + j] =- static_cast<AccumulatorT>(source[(row_offset + i) * row_stride + (col_offset + j) * col_stride]);- } else {- destination[i * THREAD_ELEMENT_COLS + j] = AccumulatorT(0);- }- }- }- }-template <typename DestinationPointerType, typename RowStride, typename ColStride>METAL_FUNC static constexpr void store(const thread ThreadDataType& source,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.hindex 271d99673..487a90a54 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h@@ -171,16 +171,12 @@ struct IntegerSchedule {metadata_context.right_zero_point_group_stride);}- if constexpr (RightOperand::BITS == 4) {- const ushort packed =- *reinterpret_cast<const device ushort*>(zero_point_row + (right_zero_point_column_start >> 1));- uint spread = (uint(packed) | (uint(packed) << 8)) & 0x00FF00FFu;- spread = (spread | (spread << 4)) & 0x0F0F0F0Fu;- zero_points[tile_n] = as_type<ZeroPointVector>(spread);- } else {- zero_points[tile_n] =- *reinterpret_cast<const device ZeroPointVector*>(zero_point_row + right_zero_point_column_start);- }+ static_assert(RightOperand::BITS == 4, "zero points are 4-bit");+ const ushort packed =+ *reinterpret_cast<const device ushort*>(zero_point_row + (right_zero_point_column_start >> 1));+ uint spread = (uint(packed) | (uint(packed) << 8)) & 0x00FF00FFu;+ spread = (spread | (spread << 4)) & 0x0F0F0F0Fu;+ zero_points[tile_n] = as_type<ZeroPointVector>(spread);}if constexpr (!ALIGNED_N) {const bool4 live =diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.hindex 2ef2001b0..4adb2da49 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h@@ -5,7 +5,6 @@#include "../../../../common/thread_context.h"#include "../../../../generated/gemm.h"#include "../../../common/fragment.h"-#include "../../../common/mxu_gemm_loop.h"#include "../gemm_alignment.h"#include "../operands.h"#include "../quant_scale_bias.h"diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rsindex 59455cf65..bb635d08c 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/error.rs@@ -4,15 +4,10 @@ use crate::backends::common::gpu_types::gemm::GemmTiling;#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]pub enum GemmSpecializationError {- #[error("unsupported A group size {a_group_size:?}; expected None or 32/64/128")]+ #[error("unsupported A group size {a_group_size:?}; expected None or 128")]InvalidAGroupSize {a_group_size: Option<u32>,},- #[error("simdgroup K={simdgroup_k} exceeds group size {group_size}")]- SimdgroupKExceedsGroupSize {- simdgroup_k: u32,- group_size: u32,- },#[error("quantized B requires transposed layout")]QuantizedRequiresTransposedB,#[error("tiling {tiling} does not match use_mxu={use_mxu}")]@@ -20,11 +15,4 @@ pub enum GemmSpecializationError {tiling: GemmTiling,use_mxu: bool,},- #[error(- "MXU quantized GEMM with tile {tiling} requires group_size <= 64 (got {group_size}) due to threadgroup memory budget"- )]- MxuQuantTileTooLarge {- tiling: GemmTiling,- group_size: u32,- },}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metalindex 91e9d5cd3..5905cdb15 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -59,12 +59,12 @@ VARIANTS(GemmBPrologueKind::ScaleZeroPointDequant,GemmBPrologueKind::ScaleSymmetricDequant)VARIANTS(BITS, 0, 4, 8)-VARIANTS(GROUP_SIZE, 0, 16, 32, 64, 128)+VARIANTS(GROUP_SIZE, 0, 32, 64)VARIANTS(A_PROLOGUE,GemmAPrologueKind::FullPrecision,GemmAPrologueKind::Int8Symmetric)-VARIANTS(A_GROUP_SIZE, 0, 32, 64, 128)+VARIANTS(A_GROUP_SIZE, 0, 128)CONSTRAINT(USE_MXU ==(GEMM_TILING == GemmTiling::Tile16x32x256_Simdgroups1x1 ||@@ -76,37 +76,31 @@ CONSTRAINT(CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || BT != "float")-CONSTRAINT(- GROUP_SIZE != 16 ||- GEMM_TILING == GemmTiling::Tile64x64x16_Simdgroups2x2)+CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleZeroPointDequant || BITS == 4)+CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant || BITS == 8)CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision ||(TRANSPOSE_B &&- (GEMM_TILING != GemmTiling::Tile64x64x16_Simdgroups2x2 ||- GROUP_SIZE == 16)))+ GEMM_TILING != GemmTiling::Tile64x64x16_Simdgroups2x2 &&+ GEMM_TILING != GemmTiling::Tile64x32x32_Simdgroups2x2))CONSTRAINT(- B_PROLOGUE == GemmBPrologueKind::FullPrecision ||- GEMM_TILING != GemmTiling::Tile128x128x256_Simdgroups4x4 ||- GROUP_SIZE <= 64)+ B_PROLOGUE != GemmBPrologueKind::FullPrecision || USE_MXU ||+ GEMM_TILING == GemmTiling::Tile64x64x16_Simdgroups2x2 ||+ GEMM_TILING == GemmTiling::Tile64x32x32_Simdgroups2x2)CONSTRAINT(- !(GEMM_TILING == GemmTiling::Tile16x32x256_Simdgroups1x1 ||- GEMM_TILING == GemmTiling::Tile16x128x256_Simdgroups1x4) ||+ GEMM_TILING != GemmTiling::Tile16x32x256_Simdgroups1x1 ||(TRANSPOSE_B &&(B_PROLOGUE == GemmBPrologueKind::FullPrecision ||A_PROLOGUE == GemmAPrologueKind::Int8Symmetric)))-CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || USE_MXU)-CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || BITS == 4 || BITS == 8)CONSTRAINT(- A_PROLOGUE == GemmAPrologueKind::FullPrecision ||- (GROUP_SIZE % METAL_SIMD_SIZE == 0 && GROUP_SIZE != 0))+ GEMM_TILING != GemmTiling::Tile16x128x256_Simdgroups1x4 ||+ (TRANSPOSE_B && B_PROLOGUE == GemmBPrologueKind::FullPrecision))+CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || USE_MXU)CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision ||(TRANSPOSE_B && B_PROLOGUE != GemmBPrologueKind::FullPrecision))CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || (AT == "bfloat" && DT == "bfloat"))CONSTRAINT((A_PROLOGUE == GemmAPrologueKind::FullPrecision) == (A_GROUP_SIZE == 0))-// The integer schedule drains one weight group at a time, so an activation-// group narrower than the weight group has no kernel.-CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || A_GROUP_SIZE >= GROUP_SIZE)KERNEL(Gemm)(const device AT* a OPTIONAL(A_PROLOGUE == GemmAPrologueKind::FullPrecision),const device BT* b,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metalindex ef20f04a3..7faab6ed8 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal@@ -6,7 +6,7 @@ using namespace metal;using namespace uzu::gemm;template <typename T>-VARIANTS(T, float, half, bfloat)+VARIANTS(T, float, bfloat)KERNEL(GemmSplitKReduce)(const device T* partial_sums,device T* output,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rsindex 9e5ba1d04..b7beec182 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs@@ -8,7 +8,7 @@ use metal::MTLGPUFamily;use crate::backends::common::gpu_types::gemm::GemmTiling;-pub(super) const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;+const MXU_DEFAULT_TILE: GemmTiling = GemmTiling::Tile64x64x256_Simdgroups2x2;const MXU_SKINNY_M_MAX: u32 = 16;const SKINNY_SQUARE_K_MAX: u32 = 2560;@@ -108,12 +108,9 @@ fn prefers_wide_partial_m_tile(apple_gpu_family: MTLGPUFamily) -> bool {pub(super) fn simdgroup_quant_tile(m: u32,n: u32,- group_size: u32,apple_gpu_family: MTLGPUFamily,) -> GemmTiling {- if group_size < 32 {- GemmTiling::Tile64x64x16_Simdgroups2x2- } else if m < SIMDGROUP_QUANT_SMALL_M_MAX {+ if m < SIMDGROUP_QUANT_SMALL_M_MAX {if !prefers_wide_partial_m_tile(apple_gpu_family)|| m <= SIMDGROUP_QUANT_NARROW_BLOCK_M|| m.is_multiple_of(SIMDGROUP_QUANT_NARROW_BLOCK_M)diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rsindex b5c49acd9..4f8921c8e 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs@@ -117,16 +117,7 @@ impl GemmProblem {let Some(step) = outer_block_k(shape, engine, tiling) else {return 1;};- let group_size = shape.b_group_size.unwrap_or(0);- let mut align = if engine == GemmEngine::Mxu || !shape.is_quant() {- step- } else {- step.max(group_size)- };- if shape.b_prologue == GemmBPrologueKind::ScaleZeroPointDequant && shape.b_bits == Some(4) {- align = align.max(2_u32.saturating_mul(group_size));- }- let align = align.max(ACTIVATION_SCALE_GROUP_SIZE).max(group_size);+ let align = step.max(ACTIVATION_SCALE_GROUP_SIZE);let target_tiles = policy::split_k_target_tiles(!shape.a_full_precision, tiling, shape.b_bits);let mut split_k = (target_tiles / base_tiles).max(1).min((shape.k / align).max(1));if !shape.a_full_precision && engine == GemmEngine::Mxu && tiling.block_k() != 0 {@@ -201,7 +192,7 @@ fn mxu_is_eligible(shape: MatmulShape) -> bool {}shape.b_transpose&& shape.b_leading_dimension.is_none()- && shape.k.is_multiple_of(select_mxu_quant_tiling(shape).block_k())+ && shape.k.is_multiple_of(policy::mxu_mn_tile(!shape.a_full_precision, shape.m, shape.n).block_k())}fn select_tiling(@@ -210,24 +201,15 @@ fn select_tiling(apple_gpu_family: MTLGPUFamily,) -> GemmTiling {match engine {- GemmEngine::Simdgroup if shape.is_quant() => {- policy::simdgroup_quant_tile(shape.m, shape.n, shape.b_group_size.unwrap_or(0), apple_gpu_family)- },+ GemmEngine::Simdgroup if shape.is_quant() => policy::simdgroup_quant_tile(shape.m, shape.n, apple_gpu_family),GemmEngine::Simdgroup => policy::simdgroup_fp_tile(shape.m, shape.n, shape.k),- GemmEngine::Mxu if !shape.a_full_precision || shape.is_quant() => select_mxu_quant_tiling(shape),+ GemmEngine::Mxu if !shape.a_full_precision || shape.is_quant() => {+ policy::mxu_mn_tile(!shape.a_full_precision, shape.m, shape.n)+ },GemmEngine::Mxu if shape.b_transpose => policy::mxu_fp_tile(shape.m, shape.n, shape.k),GemmEngine::Mxu => policy::mxu_mn_tile(false, shape.m, shape.n),}}--fn select_mxu_quant_tiling(shape: MatmulShape) -> GemmTiling {- let tiling = policy::mxu_mn_tile(!shape.a_full_precision, shape.m, shape.n);- if tiling.fits_quant_group_size(shape.b_group_size.unwrap_or(0)) {- tiling- } else {- policy::MXU_DEFAULT_TILE- }-}#[cfg(test)]#[path = "../../../../../../unit/backends/metal/kernel/matmul/gemm/selection_test.rs"]mod tests;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rsindex edc7f9d90..d215a1d41 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -2,7 +2,7 @@ use super::{GemmEngine, GemmPlan, error::GemmSpecializationError};use crate::{backends::common::{gpu_types::gemm::{GemmAPrologueKind, GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},- kernel::matmul::MatmulShape,+ kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},},data_type::DataType,};@@ -54,7 +54,7 @@ impl GemmSpecialization {pub(super) fn validate(&self) -> Result<(), GemmSpecializationError> {let valid_activation_group = match self.a_prologue {GemmAPrologueKind::FullPrecision => self.a_group_size.is_none(),- GemmAPrologueKind::Int8Symmetric => matches!(self.a_group_size, Some(32 | 64 | 128)),+ GemmAPrologueKind::Int8Symmetric => self.a_group_size == Some(ACTIVATION_SCALE_GROUP_SIZE),};if !valid_activation_group {return Err(GemmSpecializationError::InvalidAGroupSize {@@ -67,27 +67,6 @@ impl GemmSpecialization {use_mxu: self.use_mxu,});}- if self.use_mxu- && self.b_prologue != GemmBPrologueKind::FullPrecision- && let Some(group_size) = self.b_group_size- && !self.tiling.fits_quant_group_size(group_size)- {- return Err(GemmSpecializationError::MxuQuantTileTooLarge {- tiling: self.tiling,- group_size,- });- }- if !self.use_mxu- && let Some(group_size) = self.b_group_size- {- let simdgroup_block_k = self.tiling.simdgroup_block_k();- if simdgroup_block_k > group_size {- return Err(GemmSpecializationError::SimdgroupKExceedsGroupSize {- simdgroup_k: simdgroup_block_k,- group_size,- });- }- }if self.b_prologue != GemmBPrologueKind::FullPrecision && !self.transpose_b {return Err(GemmSpecializationError::QuantizedRequiresTransposedB);}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metalindex 597618e85..174d01d48 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -36,7 +36,7 @@ VARIANTS(GemmBPrologueKind::ScaleBiasDequant,GemmBPrologueKind::ScaleZeroPointDequant,GemmBPrologueKind::ScaleSymmetricDequant)-VARIANTS(GROUP_SIZE, 0, 16, 32, 64, 128)+VARIANTS(GROUP_SIZE, 0, 32, 64)VARIANTS(BITS, 0, 4, 8)VARIANTS(K_SPLIT, 1, 2, 4, 8)VARIANTS(INPUT_ALIGNED, false, true)@@ -49,6 +49,8 @@ VARIANTS(NUM_SIMDGROUPS, 2, 4, 8)CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || BT != "float")+CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleZeroPointDequant || BITS == 4)+CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant || BITS == 8)CONSTRAINT(BITS == 0 || K_SPLIT == 1)CONSTRAINT(BITS != 0 || (INPUT_ROW_TILE == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && GROUP_LANES == 1))CONSTRAINT(INPUT_ROW_TILE != 1 || REDUCTION_LANES == 32)@@ -62,16 +64,10 @@ CONSTRAINT(CONSTRAINT(INPUT_ROW_TILE == 1 ||- (((B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant && BITS == 8) ||- (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant && BITS == 4)) &&- (GROUP_SIZE == 32 || GROUP_SIZE == 64) &&+ (B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant && GROUP_SIZE == 64 &&((GEMV_TILE(16, 8, 2) && INPUT_ROWS(2, 7)) ||- (GEMV_TILE(16, 16, 4) && INPUT_ROWS(2, 6) &&- ((GROUP_SIZE == 32 && (BITS == 4 || INPUT_ROW_TILE >= 3)) ||- (BITS == 4 && GROUP_SIZE == 64 && INPUT_ROW_TILE >= 3))) ||- (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant && GEMV_TILE(8, 8, 2) &&- ((GROUP_SIZE == 32 && (INPUT_ROW_TILE == 4 || INPUT_ROW_TILE == 8)) ||- (GROUP_SIZE == 64 && INPUT_ROW_TILE == 2))))))+ (BITS == 4 && GEMV_TILE(16, 16, 4) && INPUT_ROWS(3, 6)) ||+ (BITS == 4 && GEMV_TILE(8, 8, 2) && INPUT_ROW_TILE == 2))))// Keep only selector-reachable geometry families.CONSTRAINT(@@ -80,9 +76,7 @@ CONSTRAINT((OUTPUT_ROW_TILE == 2 || OUTPUT_ROW_TILE == 4 || OUTPUT_ROW_TILE == 8 ||(INPUT_ROW_TILE > 1 && OUTPUT_ROW_TILE == 16))) ||(NUM_SIMDGROUPS == 4 && (OUTPUT_ROW_TILE == 8 || OUTPUT_ROW_TILE == 16 || OUTPUT_ROW_TILE == 32)) ||- (NUM_SIMDGROUPS == 8 &&- (OUTPUT_ROW_TILE == 16 || OUTPUT_ROW_TILE == 32 ||- (BITS == 4 && INPUT_ALIGNED && OUTPUT_ROW_TILE == 64))))+ (NUM_SIMDGROUPS == 8 && (OUTPUT_ROW_TILE == 16 || OUTPUT_ROW_TILE == 32)))CONSTRAINT(BITS == 0 || (AT == "bfloat" && DT == "bfloat") ||(NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32))@@ -103,12 +97,9 @@ CONSTRAINT(BITS != 4 || (GROUP_SIZE / GROUP_LANES) % 16 == 0)CONSTRAINT(BITS != 8 || (GROUP_SIZE / GROUP_LANES) % 8 == 0)CONSTRAINT(BITS != 0 || REDUCTION_LANES == 32)CONSTRAINT(- BITS == 0 || INPUT_ROW_TILE > 1 || (BITS == 4 &&- ((GROUP_SIZE == 16 && GROUP_LANES == 1) || (GROUP_SIZE == 32 && GROUP_LANES == 2) ||- (GROUP_SIZE == 64 && GROUP_LANES == 4) || (GROUP_SIZE == 128 && GROUP_LANES == 8))) ||- (BITS == 8 &&- ((GROUP_SIZE == 16 && GROUP_LANES == 2) || (GROUP_SIZE == 32 && GROUP_LANES == 4) ||- (GROUP_SIZE == 64 && GROUP_LANES == 8) || (GROUP_SIZE == 128 && GROUP_LANES == 16))))+ BITS == 0 || INPUT_ROW_TILE > 1 ||+ (BITS == 4 && ((GROUP_SIZE == 32 && GROUP_LANES == 2) || (GROUP_SIZE == 64 && GROUP_LANES == 4))) ||+ (BITS == 8 && ((GROUP_SIZE == 32 && GROUP_LANES == 4) || (GROUP_SIZE == 64 && GROUP_LANES == 8))))KERNEL(Gemv)(const device uint32_t* b,const device BT* scalesdiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rsindex 5420da6c0..43dd010ff 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs@@ -35,7 +35,7 @@ pub fn gathered_tile(m: u32,n: u32,) -> Option<GemvTile> {- if !matches!(bits, 4 | 8) || !matches!(group, 16 | 32 | 64 | 128) || !(1..=8).contains(&m) {+ if !matches!(bits, 4 | 8) || !matches!(group, 32 | 64) || !(1..=8).contains(&m) {return None;}let tile = lane_default(bits, group);@@ -105,7 +105,7 @@ pub fn select(k: u32,bf16_io: bool,) -> Option<GemvTile> {- if !matches!(bits, 4 | 8) || !matches!(group, 16 | 32 | 64 | 128) || !(1..=8).contains(&m) {+ if !matches!(bits, 4 | 8) || !matches!(group, 32 | 64) || !(1..=8).contains(&m) {return None;}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rsindex d9f42d111..e0fc85dbc 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs@@ -1,4 +1,4 @@-// Qwen3.6-27B W4/ZP and W8/Symmetric, G32/G64.+// Qwen3.6-27B W4/ZP and W8/Symmetric, G64.// Tuned on M1, M2, M2 Pro, M3 Max, M4, M4 Pro, M5 Max.use metal::MTLGPUFamily;@@ -19,8 +19,7 @@ const GATE: u8 = 1 << 1;const GATE_UP: u8 = 1 << 2;const PROJECTION_IN: u8 = 1 << 3;const PROJECTION_OUT: u8 = 1 << 4;-const QKV: u8 = 1 << 5;-const READOUT: u8 = 1 << 6;+const READOUT: u8 = 1 << 5;#[derive(Clone, Copy)]struct RouteRow {@@ -43,7 +42,6 @@ const fn shape((34816, 5120) => GATE_UP,(16480, 5120) => PROJECTION_IN,(5120, 6144) => PROJECTION_OUT,- (8192, 5120) => QKV,(248320, 5120) => READOUT,_ => 0,}@@ -56,7 +54,7 @@ macro_rules! main_gemv { ($input:literal, $results_per_simdgroup:literal, $lanes#[rustfmt::skip]macro_rules! main_gemm { ($engine:ident, $tiling:ident, $split:literal) => { QmvRoute::MainGemm(GemmPlan { engine: GemmEngine::$engine, tiling: GemmTiling::$tiling, split_k: $split }) }; }#[rustfmt::skip]-macro_rules! qmv_format { (w4_zp_g32) => { (4, 32) }; (w4_zp_g64) => { (4, 64) }; (w8_sym_g32) => { (8, 32) }; (w8_sym_g64) => { (8, 64) }; }+macro_rules! qmv_format { (w4_zp_g64) => { (4, 64) }; (w8_sym_g64) => { (8, 64) }; }#[rustfmt::skip]macro_rules! measured_device_name { (M1) => { "Apple M1" }; (M2) => { "Apple M2" }; (M2Pro) => { "Apple M2 Pro" }; (M3Max) => { "Apple M3 Max" }; (M4) => { "Apple M4" }; (M4Pro) => { "Apple M4 Pro" }; (M5Max) => { "Apple M5 Max" }; }#[rustfmt::skip]@@ -66,310 +64,128 @@ macro_rules! row { ($device:ident, $format:ident, $m:literal, $shapes:expr, $rou#[rustfmt::skip]const ROWS: &[RouteRow] = &[- row!(M1, w4_zp_g32, 2, READOUT, main_gemv!(1, 4, 32, 2, 8)),- row!(M1, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(2, 16, 8, 1, 2)),- row!(M1, w4_zp_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M1, w4_zp_g32, 4, DOWN, main_gemv!(1, 4, 32, 2, 8)),- row!(M1, w4_zp_g32, 4, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M1, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M1, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),- row!(M1, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M1, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M1, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),- row!(M1, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 16, 1, 4)),- row!(M1, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),- row!(M1, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),- row!(M1, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M1, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),- row!(M1, w8_sym_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),- row!(M1, w8_sym_g32, 3, PROJECTION_IN | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M1, w8_sym_g32, 4, GATE_UP, main_gemv!(1, 4, 32, 4, 8)),- row!(M1, w8_sym_g32, 4, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M1, w8_sym_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M1, w8_sym_g32, 6, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M1, w8_sym_g32, 6, DOWN, tuned!(6, 16, 16, 1, 4)),- row!(M1, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M1, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M1, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M1, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M1, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M1, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M1, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M2, w4_zp_g32, 2, PROJECTION_IN, main_gemv!(1, 4, 32, 2, 8)),- row!(M2, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M2, w4_zp_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M2, w4_zp_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M2, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M2, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M2, w4_zp_g32, 7, READOUT, tuned!(4, 8, 8, 1, 2)),- row!(M2, w4_zp_g32, 7, GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(7, 16, 8, 1, 2)),- row!(M2, w4_zp_g32, 7, DOWN | GATE, tuned!(8, 8, 8, 1, 2)),+ row!(M1, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M1, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 16, 1, 4)),+ row!(M1, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 16, 1, 4)),+ row!(M1, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 16, 1, 4)),+ row!(M1, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 16, 1, 4)),+ row!(M1, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M1, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M1, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M1, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M1, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M1, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M1, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),row!(M2, w4_zp_g64, 2, PROJECTION_IN, tuned!(2, 8, 8, 1, 2)),- row!(M2, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M2, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M2, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M2, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M2, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M2, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M2, w8_sym_g32, 2, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),- row!(M2, w8_sym_g32, 2, DOWN, tuned!(2, 16, 8, 1, 2)),- row!(M2, w8_sym_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),- row!(M2, w8_sym_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M2, w8_sym_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M2, w8_sym_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M2, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M2, w8_sym_g64, 2, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M2, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M2, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M2, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M2, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M2, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M2, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M2, w8_sym_g64, 2, GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),row!(M2, w8_sym_g64, 2, DOWN, tuned!(2, 16, 8, 1, 2)),- row!(M2, w8_sym_g64, 3, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M2, w8_sym_g64, 3, GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),row!(M2, w8_sym_g64, 3, DOWN, tuned!(3, 16, 8, 1, 2)),- row!(M2, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M2, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M2, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M2, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g32, 2, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),- row!(M2Pro, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g32, 3, DOWN | GATE_UP | READOUT, main_gemv!(1, 4, 32, 2, 8)),- row!(M2Pro, w4_zp_g32, 3, PROJECTION_IN, tuned!(3, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g32, 3, GATE | PROJECTION_OUT | QKV, tuned!(3, 16, 16, 1, 4)),- row!(M2Pro, w4_zp_g32, 4, DOWN | GATE_UP | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 2, 8)),- row!(M2Pro, w4_zp_g32, 4, PROJECTION_IN, tuned!(4, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g32, 4, GATE, tuned!(4, 16, 16, 1, 4)),- row!(M2Pro, w4_zp_g32, 5, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),- row!(M2Pro, w4_zp_g32, 5, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),- row!(M2Pro, w4_zp_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),- row!(M2Pro, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g32, 6, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),- row!(M2Pro, w4_zp_g32, 6, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),- row!(M2Pro, w4_zp_g32, 6, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),- row!(M2Pro, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g32, 7, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),- row!(M2Pro, w4_zp_g32, 7, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),- row!(M2Pro, w4_zp_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),- row!(M2Pro, w4_zp_g32, 7, PROJECTION_IN, tuned!(7, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M2, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M2, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M2, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M2, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | READOUT, tuned!(3, 16, 8, 1, 2)),row!(M2Pro, w4_zp_g64, 3, PROJECTION_OUT, tuned!(3, 16, 16, 1, 4)),row!(M2Pro, w4_zp_g64, 4, GATE_UP | PROJECTION_IN | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT | QKV, tuned!(4, 16, 16, 1, 4)),+ row!(M2Pro, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT, tuned!(4, 16, 16, 1, 4)),row!(M2Pro, w4_zp_g64, 5, PROJECTION_IN, tuned!(5, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),- row!(M2Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M2Pro, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),- row!(M2Pro, w8_sym_g32, 3, DOWN | GATE_UP | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),- row!(M2Pro, w8_sym_g32, 3, PROJECTION_IN, tuned!(3, 16, 8, 1, 2)),- row!(M2Pro, w8_sym_g32, 3, GATE, tuned!(3, 16, 16, 1, 4)),- row!(M2Pro, w8_sym_g32, 4, DOWN | GATE_UP | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),- row!(M2Pro, w8_sym_g32, 4, PROJECTION_IN, tuned!(4, 16, 8, 1, 2)),- row!(M2Pro, w8_sym_g32, 4, GATE | PROJECTION_OUT, tuned!(4, 16, 16, 1, 4)),- row!(M2Pro, w8_sym_g32, 5, GATE_UP | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),- row!(M2Pro, w8_sym_g32, 5, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),- row!(M2Pro, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),- row!(M2Pro, w8_sym_g32, 5, PROJECTION_IN, tuned!(5, 16, 8, 1, 2)),- row!(M2Pro, w8_sym_g32, 6, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),- row!(M2Pro, w8_sym_g32, 6, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),- row!(M2Pro, w8_sym_g32, 6, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),- row!(M2Pro, w8_sym_g32, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 1)),- row!(M2Pro, w8_sym_g32, 7, DOWN | GATE | QKV, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),- row!(M2Pro, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),- row!(M2Pro, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M2Pro, w8_sym_g64, 3, DOWN | GATE | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 8, 8)),+ row!(M2Pro, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(5, 16, 16, 1, 4)),+ row!(M2Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M2Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M2Pro, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M2Pro, w8_sym_g64, 3, DOWN | PROJECTION_OUT, main_gemv!(1, 4, 32, 8, 8)),row!(M2Pro, w8_sym_g64, 3, GATE_UP | PROJECTION_IN | READOUT, tuned!(3, 16, 8, 1, 2)),row!(M2Pro, w8_sym_g64, 4, DOWN | PROJECTION_OUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M2Pro, w8_sym_g64, 4, GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M2Pro, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M2Pro, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M2Pro, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 2, 8)),- row!(M3Max, w4_zp_g32, 2, READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g32, 3, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),- row!(M3Max, w4_zp_g32, 3, GATE, tuned!(3, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g32, 3, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),- row!(M3Max, w4_zp_g32, 4, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),- row!(M3Max, w4_zp_g32, 4, DOWN | GATE_UP | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g32, 4, GATE | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),- row!(M3Max, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),- row!(M3Max, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 16, 1, 4)),- row!(M3Max, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g64, 2, GATE_UP | PROJECTION_IN | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),+ row!(M2Pro, w8_sym_g64, 4, GATE_UP | PROJECTION_IN | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M2Pro, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M2Pro, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M2Pro, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g64, 2, GATE_UP | PROJECTION_IN | READOUT, main_gemv!(1, 4, 32, 4, 8)),row!(M3Max, w4_zp_g64, 2, DOWN | GATE | PROJECTION_OUT, tuned!(2, 16, 8, 1, 2)),row!(M3Max, w4_zp_g64, 3, DOWN | GATE | PROJECTION_OUT, tuned!(3, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g64, 3, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),- row!(M3Max, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g64, 3, GATE_UP | PROJECTION_IN | READOUT, tuned!(3, 16, 16, 1, 4)),+ row!(M3Max, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),row!(M3Max, w4_zp_g64, 4, GATE_UP | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),- row!(M3Max, w4_zp_g64, 5, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g64, 5, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),row!(M3Max, w4_zp_g64, 5, GATE_UP, tuned!(5, 16, 16, 1, 4)),- row!(M3Max, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV, main_gemv!(1, 4, 32, 4, 8)),- row!(M3Max, w8_sym_g32, 2, PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g32, 3, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),- row!(M3Max, w8_sym_g32, 3, GATE_UP | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g32, 4, GATE, main_gemv!(1, 4, 32, 4, 8)),- row!(M3Max, w8_sym_g32, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g32, 4, READOUT, tuned!(4, 16, 16, 1, 4)),- row!(M3Max, w8_sym_g32, 5, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),- row!(M3Max, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),- row!(M3Max, w8_sym_g32, 5, DOWN, tuned!(5, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g32, 5, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),- row!(M3Max, w8_sym_g32, 6, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),- row!(M3Max, w8_sym_g32, 6, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g32, 6, PROJECTION_OUT, tuned!(6, 16, 16, 1, 4)),- row!(M3Max, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),- row!(M3Max, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M3Max, w8_sym_g64, 3, GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M3Max, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M3Max, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M3Max, w8_sym_g64, 3, GATE_UP | PROJECTION_IN | READOUT, main_gemv!(1, 4, 32, 8, 8)),row!(M3Max, w8_sym_g64, 3, DOWN | PROJECTION_OUT, tuned!(3, 16, 8, 1, 2)),row!(M3Max, w8_sym_g64, 4, GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 8, 8)),- row!(M3Max, w8_sym_g64, 4, DOWN | GATE | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M4, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 2, 8)),- row!(M4, w4_zp_g32, 2, READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M4, w4_zp_g32, 3, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),- row!(M4, w4_zp_g32, 3, GATE, tuned!(3, 16, 8, 1, 2)),- row!(M4, w4_zp_g32, 3, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),- row!(M4, w4_zp_g32, 4, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),- row!(M4, w4_zp_g32, 4, DOWN | GATE_UP | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M4, w4_zp_g32, 4, GATE | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),- row!(M4, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M4, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),- row!(M4, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M4, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 16, 1, 4)),- row!(M4, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M4, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),+ row!(M3Max, w8_sym_g64, 4, DOWN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M3Max, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M4, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 4, 8)),row!(M4, w4_zp_g64, 3, DOWN, tuned!(3, 16, 8, 1, 2)),- row!(M4, w4_zp_g64, 3, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),+ row!(M4, w4_zp_g64, 3, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 16, 1, 4)),row!(M4, w4_zp_g64, 4, DOWN | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M4, w4_zp_g64, 4, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 16, 1, 4)),+ row!(M4, w4_zp_g64, 4, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT, tuned!(4, 16, 16, 1, 4)),row!(M4, w4_zp_g64, 5, DOWN | GATE_UP | PROJECTION_IN | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M4, w4_zp_g64, 5, GATE | PROJECTION_OUT | QKV, tuned!(5, 16, 16, 1, 4)),- row!(M4, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M4, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M4, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV, main_gemv!(1, 4, 32, 4, 8)),- row!(M4, w8_sym_g32, 2, PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M4, w8_sym_g32, 3, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),- row!(M4, w8_sym_g32, 3, GATE_UP | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M4, w8_sym_g32, 4, GATE, main_gemv!(1, 4, 32, 4, 8)),- row!(M4, w8_sym_g32, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 8, 1, 2)),- row!(M4, w8_sym_g32, 4, READOUT, tuned!(4, 16, 16, 1, 4)),- row!(M4, w8_sym_g32, 5, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),- row!(M4, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),- row!(M4, w8_sym_g32, 5, DOWN, tuned!(5, 16, 8, 1, 2)),- row!(M4, w8_sym_g32, 5, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),- row!(M4, w8_sym_g32, 6, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),- row!(M4, w8_sym_g32, 6, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M4, w8_sym_g32, 6, PROJECTION_OUT, tuned!(6, 16, 16, 1, 4)),- row!(M4, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),- row!(M4, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M4, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M4, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M4, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M4, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M4, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M4, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M4Pro, w4_zp_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 2, 8)),- row!(M4Pro, w4_zp_g32, 2, READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M4Pro, w4_zp_g32, 3, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),- row!(M4Pro, w4_zp_g32, 3, GATE, tuned!(3, 16, 8, 1, 2)),- row!(M4Pro, w4_zp_g32, 3, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),- row!(M4Pro, w4_zp_g32, 4, PROJECTION_OUT, main_gemv!(1, 4, 32, 2, 8)),- row!(M4Pro, w4_zp_g32, 4, DOWN | GATE_UP | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M4Pro, w4_zp_g32, 4, GATE | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),- row!(M4Pro, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M4Pro, w4_zp_g32, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),- row!(M4Pro, w4_zp_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M4Pro, w4_zp_g32, 6, PROJECTION_IN, tuned!(6, 16, 16, 1, 4)),- row!(M4Pro, w4_zp_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M4Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 4, 8)),- row!(M4Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 16, 1, 4)),+ row!(M4, w4_zp_g64, 5, GATE | PROJECTION_OUT, tuned!(5, 16, 16, 1, 4)),+ row!(M4, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M4, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M4, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M4, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M4, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M4, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M4, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M4, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M4Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 4, 8)),+ row!(M4Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 16, 1, 4)),row!(M4Pro, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M4Pro, w4_zp_g64, 4, PROJECTION_IN | QKV, tuned!(4, 16, 16, 1, 4)),+ row!(M4Pro, w4_zp_g64, 4, PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),row!(M4Pro, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M4Pro, w4_zp_g64, 5, PROJECTION_IN | QKV, tuned!(5, 16, 16, 1, 4)),- row!(M4Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M4Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g32, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV, main_gemv!(1, 4, 32, 4, 8)),- row!(M4Pro, w8_sym_g32, 2, PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g32, 3, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 4, 8)),- row!(M4Pro, w8_sym_g32, 3, GATE_UP | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g32, 4, GATE, main_gemv!(1, 4, 32, 4, 8)),- row!(M4Pro, w8_sym_g32, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(4, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g32, 4, READOUT, tuned!(4, 16, 16, 1, 4)),- row!(M4Pro, w8_sym_g32, 5, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),- row!(M4Pro, w8_sym_g32, 5, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),- row!(M4Pro, w8_sym_g32, 5, DOWN, tuned!(5, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g32, 5, GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(5, 16, 16, 1, 4)),- row!(M4Pro, w8_sym_g32, 6, GATE, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 2)),- row!(M4Pro, w8_sym_g32, 6, DOWN | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g32, 6, PROJECTION_OUT, tuned!(6, 16, 16, 1, 4)),- row!(M4Pro, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Simdgroup, Tile8x32x32_Simdgroups1x1, 3)),- row!(M4Pro, w8_sym_g32, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV, main_gemv!(1, 4, 32, 8, 8)),+ row!(M4Pro, w4_zp_g64, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),+ row!(M4Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M4Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT, main_gemv!(1, 4, 32, 8, 8)),row!(M4Pro, w8_sym_g64, 2, READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M4Pro, w8_sym_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M5Max, w4_zp_g32, 2, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | QKV, tuned!(2, 16, 8, 1, 2)),- row!(M5Max, w4_zp_g32, 2, GATE_UP | READOUT, tuned!(2, 16, 16, 1, 4)),- row!(M5Max, w4_zp_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M5Max, w4_zp_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M5Max, w4_zp_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M5Max, w4_zp_g32, 6, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),- row!(M5Max, w4_zp_g32, 6, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),- row!(M5Max, w4_zp_g32, 6, GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),- row!(M5Max, w4_zp_g32, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),- row!(M5Max, w4_zp_g32, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),- row!(M5Max, w4_zp_g32, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),- row!(M5Max, w4_zp_g32, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),- row!(M5Max, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M5Max, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M5Max, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M5Max, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),+ row!(M4Pro, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),+ row!(M4Pro, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),+ row!(M5Max, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M5Max, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M5Max, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M5Max, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),row!(M5Max, w4_zp_g64, 6, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),- row!(M5Max, w4_zp_g64, 6, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),+ row!(M5Max, w4_zp_g64, 6, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),row!(M5Max, w4_zp_g64, 6, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),row!(M5Max, w4_zp_g64, 6, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),row!(M5Max, w4_zp_g64, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),- row!(M5Max, w4_zp_g64, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),+ row!(M5Max, w4_zp_g64, 7, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),row!(M5Max, w4_zp_g64, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),row!(M5Max, w4_zp_g64, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),- row!(M5Max, w8_sym_g32, 2, GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 4, 8)),- row!(M5Max, w8_sym_g32, 2, DOWN | GATE | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g32, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g32, 4, PROJECTION_IN, tuned!(2, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g32, 4, DOWN | GATE | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g32, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g32, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(6, 16, 16, 1, 4)),- row!(M5Max, w8_sym_g32, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),- row!(M5Max, w8_sym_g32, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),- row!(M5Max, w8_sym_g32, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),- row!(M5Max, w8_sym_g32, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),- row!(M5Max, w8_sym_g64, 2, GATE | GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 8, 8)),- row!(M5Max, w8_sym_g64, 2, DOWN | PROJECTION_OUT | QKV | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g64, 4, GATE | PROJECTION_IN, tuned!(2, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_OUT | QKV | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | QKV | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g64, 6, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),- row!(M5Max, w8_sym_g64, 6, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),+ row!(M5Max, w8_sym_g64, 2, GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 8, 8)),+ row!(M5Max, w8_sym_g64, 2, DOWN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g64, 4, PROJECTION_IN, tuned!(2, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),+ row!(M5Max, w8_sym_g64, 6, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),row!(M5Max, w8_sym_g64, 6, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),row!(M5Max, w8_sym_g64, 6, GATE_UP | PROJECTION_IN | READOUT, tuned!(6, 16, 8, 1, 2)),row!(M5Max, w8_sym_g64, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),- row!(M5Max, w8_sym_g64, 7, DOWN | QKV, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),- row!(M5Max, w8_sym_g64, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),+ row!(M5Max, w8_sym_g64, 7, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),row!(M5Max, w8_sym_g64, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),];fn qmv_format(shape: &MatmulShape) -> Option<(u32, u32)> {match (shape.b_prologue, shape.b_bits, shape.b_group_size) {- (GemmBPrologueKind::ScaleZeroPointDequant, Some(4), Some(group @ (32 | 64))) => Some((4, group)),- (GemmBPrologueKind::ScaleSymmetricDequant, Some(8), Some(group @ (32 | 64))) => Some((8, group)),+ (GemmBPrologueKind::ScaleZeroPointDequant, Some(4), Some(64)) => Some((4, 64)),+ (GemmBPrologueKind::ScaleSymmetricDequant, Some(8), Some(64)) => Some((8, 64)),_ => None,}}diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rsindex f56d0836c..ec8d119ac 100644--- a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs@@ -294,9 +294,7 @@ mod quantize {}#[uzu_test]- fn quantize_scale_g32_and_g64_match_cpu() {- check_quantize(32, false, None, Int8CodeLayout::Sequential);- check_quantize(64, false, None, Int8CodeLayout::Sequential);+ fn quantize_nibble_grouped_matches_cpu() {check_quantize(128, false, None, Int8CodeLayout::GroupedByNibble);}}diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rsindex 657f20156..8f0e8bf0f 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs@@ -278,19 +278,13 @@ fn bench_a8w(c: &mut Criterion) {}let hadamard = ActivationTransform::<Metal>::input_rht(&context, DataType::BF16, true).expect("hadamard kernel");- for bits in [8u32, 4u32] {- let prepare = ActivationTransform::<Metal>::quantize(- &context,- DataType::BF16,- ActivationQuantization::new(- 128,- 128,- false,- Int8CodeLayout::for_right_bits(bits).expect("W4/W8 benchmark"),- )+ let bits = 8;+ let prepare = ActivationTransform::<Metal>::quantize(+ &context,+ DataType::BF16,+ ActivationQuantization::new(128, 128, false, Int8CodeLayout::for_right_bits(bits).expect("W8 benchmark")).expect("supported activation quantization"),- )- .expect("prepare kernel");- bench_bits(c, &context, &prepare, &hadamard, bits);- }+ )+ .expect("prepare kernel");+ bench_bits(c, &context, &prepare, &hadamard, bits);}diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rsindex 2924eacf8..0e7e48bc6 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs@@ -293,8 +293,7 @@ fn gemv_gather() {(4, QuantizationMethod::ScaleBias, false),(4, QuantizationMethod::ScaleZeroPoint, false),(4, QuantizationMethod::ScaleZeroPoint, true),- (4, QuantizationMethod::ScaleSymmetric, false),- (8, QuantizationMethod::ScaleZeroPoint, false),+ (8, QuantizationMethod::ScaleSymmetric, false),] {let mut input = QuantInput::new(8, 128, 64, 32, bits, method, 0x5EED);if signed_codes {diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rsindex 494469be9..95cbc385f 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs@@ -124,19 +124,14 @@ fn run_parity<T: ArrayElement + Float + Debug + Display>(#[rstest]#[test_attr(uzu_test)]-#[case::gs16_4bit_zp_prefill(64, 256, 64, 16, 4, QuantizationMethod::ScaleZeroPoint)]#[case::gs32_4bit_mlx_prefill(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]#[case::gs64_4bit_mlx_prefill(64, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]-#[case::gs128_4bit_mlx_prefill(64, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]-#[case::gs32_4bit_symmetric_prefill(64, 256, 64, 32, 4, QuantizationMethod::ScaleSymmetric)]#[case::gs32_4bit_mlx_decode(8, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]-#[case::gs16_4bit_zp_decode(8, 256, 64, 16, 4, QuantizationMethod::ScaleZeroPoint)]#[case::gs64_4bit_zp_decode(8, 256, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]#[case::gs32_unaligned_n(64, 256, 96, 32, 4, QuantizationMethod::ScaleBias)]#[case::gs32_narrow_n_tail_zp(64, 256, 72, 32, 4, QuantizationMethod::ScaleZeroPoint)]#[case::gs32_narrow_n_tail_mlx(64, 256, 72, 32, 4, QuantizationMethod::ScaleBias)]-#[case::gs32_narrow_n_tail_sym(64, 256, 40, 32, 4, QuantizationMethod::ScaleSymmetric)]-#[case::gs16_k_tail_mlx(64, 272, 64, 16, 4, QuantizationMethod::ScaleBias)]+#[case::gs32_narrow_n_tail_sym(64, 256, 40, 32, 8, QuantizationMethod::ScaleSymmetric)]fn parity_bf16(#[case] m: u32,#[case] k: u32,@@ -151,8 +146,6 @@ fn parity_bf16(#[rstest]#[test_attr(uzu_test)]#[case::gs32_8bit_mlx_prefill(64, 256, 64, 32, 8, QuantizationMethod::ScaleBias)]-#[case::gs64_8bit_zp_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]-#[case::gs128_8bit_zp_prefill(64, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]#[case::gs64_8bit_symmetric_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]fn parity_bf16_8bit_splitk(#[case] m: u32,@@ -169,11 +162,7 @@ fn parity_bf16_8bit_splitk(#[test_attr(uzu_test)]#[case::m1_gs32_4bit_mlx(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]#[case::m1_gs64_4bit_mlx(1, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]-#[case::m1_gs128_4bit_mlx(1, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]#[case::m1_gs32_4bit_zp(1, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]-#[case::m1_gs64_8bit_zp(1, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]-#[case::m1_gs128_8bit_mlx(1, 256, 64, 128, 8, QuantizationMethod::ScaleBias)]-#[case::m1_gs32_4bit_sym(1, 256, 64, 32, 4, QuantizationMethod::ScaleSymmetric)]#[case::m1_gs64_8bit_sym(1, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]#[case::m2_gs32_4bit_mlx(2, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]#[case::m4_gs32_4bit_zp(4, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]@@ -276,14 +265,13 @@ fn parity_bf16_gemv_qmv_fused_scale_bias() {#[test_attr(uzu_test)]#[case::gs64_4bit(1, 96, 64, 64, 4, QuantizationMethod::ScaleBias)]#[case::gs64_4bit_zp(2, 96, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]-#[case::gs128_8bit(1, 192, 64, 128, 8, QuantizationMethod::ScaleBias)]// Whole quantization groups with an unaligned K block.#[case::gs64_4bit_k1152_zp(1, 1152, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]-#[case::gs64_8bit_k1152_zp(1, 1152, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]+#[case::gs64_8bit_k1152_sym(1, 1152, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]// Same, plus a final partial quantization group (1056 = 16 * 64 + 32).#[case::gs64_4bit_k1056_partial_group(1, 1056, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]#[case::gs64_4bit_k514_unaligned_rows(2, 514, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]-#[case::gs64_8bit_k257_unaligned_rows(2, 257, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]+#[case::gs64_8bit_k257_unaligned_rows(2, 257, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]fn parity_gemv_partial_group_bf16(#[case] m: u32,#[case] k: u32,@@ -463,7 +451,6 @@ fn quant_gemm_accumulate_returns_unsupported_dop() {#[rstest]#[test_attr(uzu_test)]#[case::output_group_single_group_gs64(OutputGroup, 64, 64, 64, 4, ScaleZeroPoint, 64)]-#[case::output_group_single_group_gs128(OutputGroup, 128, 64, 128, 4, ScaleZeroPoint, 64)]#[case::output_group_odd_groups(OutputGroup, 192, 64, 64, 4, ScaleZeroPoint, 64)]#[case::group_output_padded_n(GroupOutput, 192, 66, 64, 4, ScaleZeroPoint, 66)]#[case::output_group_bias_prefix(OutputGroup, 128, 12, 32, 8, ScaleBias, 4)]@@ -517,13 +504,9 @@ fn quant_gemm_parameter_layout_prefix_matches_cpu(#[test_attr(uzu_test)]#[case::gs32_4bit_mlx(128, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]#[case::gs64_4bit_mlx(128, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]-#[case::gs128_4bit_mlx(128, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]#[case::small_m_4bit_mlx(24, 256, 512, 64, 4, QuantizationMethod::ScaleBias)]#[case::gs32_4bit_zp(128, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]-#[case::gs64_8bit_zp(128, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]-#[case::gs128_8bit_zp(128, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]#[case::gs64_4bit_zp_g1(128, 64, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]-#[case::gs128_4bit_zp_g1(128, 128, 64, 128, 4, QuantizationMethod::ScaleZeroPoint)]fn mxu_quant_parity_bf16(#[case] m: u32,#[case] k: u32,@@ -550,18 +533,11 @@ fn mxu_quant_parity_bf16(#[rstest]#[test_attr(uzu_test)]-#[case::sym_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleSymmetric)]#[case::sym_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]-#[case::sym_w4_gs128(32u32, 128u32, 4u32, QuantizationMethod::ScaleSymmetric)]#[case::bias_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleBias)]#[case::bias_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleBias)]-#[case::bias_w4_gs128(32u32, 128u32, 4u32, QuantizationMethod::ScaleBias)]#[case::zp_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleZeroPoint)]-#[case::zp_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleZeroPoint)]-#[case::zp_w4_gs128(32u32, 128u32, 4u32, QuantizationMethod::ScaleZeroPoint)]#[case::m1_bias_w4_gs64(1u32, 64u32, 4u32, QuantizationMethod::ScaleBias)]-#[case::m1_zp_w8_gs32(1u32, 32u32, 8u32, QuantizationMethod::ScaleZeroPoint)]-#[case::m1_sym_w4_gs128(1u32, 128u32, 4u32, QuantizationMethod::ScaleSymmetric)]fn a8w_mxu_parity_bf16(#[case] m: u32,#[case] weight_gs: u32,@@ -600,9 +576,10 @@ fn a8w_independent_activation_group_parity_bf16(#[case] m: u32) {return;}- for (activation_group_size, weight_group_size) in [(32u32, 32u32), (64, 64), (128, 32), (128, 64), (128, 128)] {+ for (activation_group_size, weight_group_size) in+ [(ACTIVATION_SCALE_GROUP_SIZE, 32u32), (ACTIVATION_SCALE_GROUP_SIZE, 64)]+ {for (bits, method) in [- (4, QuantizationMethod::ScaleSymmetric),(8, QuantizationMethod::ScaleSymmetric),(8, QuantizationMethod::ScaleBias),(4, QuantizationMethod::ScaleZeroPoint),@@ -631,27 +608,24 @@ fn a8w_independent_activation_group_parity_bf16(#[case] m: u32) {#[case::m1(1)]#[case::m16(16)]#[case::m33(33)]-fn a8w8_zero_point_tail_parity(#[case] m: u32) {+fn a8w4_zero_point_tail_parity(#[case] m: u32) {let context = MetalContext::new().expect("Metal context");if !context.supports_mxu {return;}- let input = QuantInput::<bf16>::new(m, 256, 72, 32, 8, QuantizationMethod::ScaleZeroPoint, 0)+ let input = QuantInput::<bf16>::new(m, 256, 72, 32, 4, QuantizationMethod::ScaleZeroPoint, 0).with_prepared_a(ACTIVATION_SCALE_GROUP_SIZE, Some(32));let reference = run_quant_cpu::<bf16>(&input);let actual = run_quant_metal(&context, &input.with_group_output(), Some(GemmEngine::Mxu));- assert_parity::<bf16>(&format!("A8W8 ZP N-tail m={m}"), &reference, &actual, 0.08, 0.8);+ assert_parity::<bf16>(&format!("A8W4 ZP N-tail m={m}"), &reference, &actual, 0.08, 0.8);}#[rstest]#[test_attr(uzu_test)]-#[case::w4_sym(4u32, QuantizationMethod::ScaleSymmetric, 256u32)]#[case::w4_bias(4u32, QuantizationMethod::ScaleBias, 256u32)]#[case::w4_zp(4u32, QuantizationMethod::ScaleZeroPoint, 256u32)]#[case::w8_sym(8u32, QuantizationMethod::ScaleSymmetric, 256u32)]#[case::w8_bias(8u32, QuantizationMethod::ScaleBias, 256u32)]-#[case::w8_zp(8u32, QuantizationMethod::ScaleZeroPoint, 256u32)]-#[case::w8_zp_tail(8u32, QuantizationMethod::ScaleZeroPoint, 224u32)]fn signed_weights_full_precision_activations_parity_bf16(#[case] bits: u32,#[case] method: QuantizationMethod,@@ -677,9 +651,7 @@ fn signed_weights_full_precision_activations_parity_bf16(#[rstest]#[test_attr(uzu_test)]-#[case::w4_bias(4u32, false)]#[case::w8_bias(8u32, false)]-#[case::w4_rht_bias(4u32, true)]#[case::w8_rht_bias(8u32, true)]fn a8w_mxu_output_bias_parity_bf16(#[case] bits: u32,diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rsindex ae0605033..1dc874b35 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemm_bench.rs@@ -73,6 +73,4 @@ fn bench_unified_quantized_gemm(c: &mut Criterion) {let context = crate::tests::util::shared_metal_context();bench_unified_quant_typed::<bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);bench_unified_quant_typed::<bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);- bench_unified_quant_typed::<bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);- bench_unified_quant_typed::<bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);}diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rsindex 79be8b9d2..c6ef41e56 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs@@ -61,8 +61,6 @@ fn bench_gemv(c: &mut Criterion) {bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);bench_gemv_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);- bench_gemv_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);- bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);});}diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/qwen3_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/qwen3_bench.rsindex ddd0f9873..8a2f589c5 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/qwen3_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/qwen3_bench.rs@@ -56,6 +56,6 @@ fn bench_qwen3_layers_typed<T: ArrayElement + Float>(#[uzu_bench]fn bench_qwen3_layers(c: &mut Criterion) {let context = crate::tests::util::shared_metal_context();- bench_qwen3_layers_typed::<bf16>(c, &context, "ScaleBias_BF16_gs128_4b", 128, 4, QuantizationMethod::ScaleBias);- bench_qwen3_layers_typed::<bf16>(c, &context, "ZP_BF16_gs128_4b", 128, 4, QuantizationMethod::ScaleZeroPoint);+ bench_qwen3_layers_typed::<bf16>(c, &context, "ScaleBias_BF16_gs64_4b", 64, 4, QuantizationMethod::ScaleBias);+ bench_qwen3_layers_typed::<bf16>(c, &context, "ZP_BF16_gs64_4b", 64, 4, QuantizationMethod::ScaleZeroPoint);}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rsindex 563d2d8c2..7b7ff7826 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs@@ -71,18 +71,17 @@ fn policy_boundaries_are_preserved() {assert_eq!(policy::mxu_fp_tile(m, n, k), expected);}- for (m, n, group_size, family, expected) in [- (16, 4096, 31, MTLGPUFamily::Apple7, Tile64x64x16_Simdgroups2x2),- (16, 4096, 32, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),- (8, 4096, 32, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),- (9, 4096, 32, MTLGPUFamily::Apple8, Tile32x32x32_Simdgroups2x2),- (9, 4096, 32, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),- (31, 4096, 32, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),- (32, 4096, 32, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),- (64, 6143, 32, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),- (64, 6144, 32, MTLGPUFamily::Apple9, Tile64x64x32_Simdgroups2x2),+ for (m, n, family, expected) in [+ (16, 4096, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),+ (8, 4096, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),+ (9, 4096, MTLGPUFamily::Apple8, Tile32x32x32_Simdgroups2x2),+ (9, 4096, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),+ (31, 4096, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),+ (32, 4096, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),+ (64, 6143, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),+ (64, 6144, MTLGPUFamily::Apple9, Tile64x64x32_Simdgroups2x2),] {- assert_eq!(policy::simdgroup_quant_tile(m, n, group_size, family), expected);+ assert_eq!(policy::simdgroup_quant_tile(m, n, family), expected);}}@@ -118,11 +117,6 @@ fn selection_fallbacks_and_split_k_are_preserved() {assert_eq!(problem(invalid_layout, DataType::BF16).select_plan().engine, Mxu);assert_eq!(problem(quant(shape(64, 4096, 4095)), DataType::BF16).select_plan().engine, Simdgroup);- let mut large_group = quant(shape(512, 4096, 4096));- large_group.a_full_precision = false;- large_group.b_group_size = Some(128);- assert_eq!(problem(large_group, DataType::BF16).select_plan().tiling, Tile64x64x256_Simdgroups2x2);-for (m, n, k, expected_tiling, expected_split_k) in [(16, 4096, 4096, Tile16x32x256_Simdgroups1x1, 8),(16, 1024, 4096, Tile16x32x256_Simdgroups1x1, 16),diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rsindex 9121e6a80..12fa48313 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs@@ -14,7 +14,7 @@ use crate::{data_type::DataType,};-const FROZEN_PLAN_FINGERPRINT: u64 = 5_839_212_743_558_880_136;+const FROZEN_PLAN_FINGERPRINT: u64 = 10_794_242_881_707_148_570;const DEVICES: [(&str, &str, u32, MTLGPUFamily, bool); 7] = [("m1", "Apple M1", 8, MTLGPUFamily::Apple7, false),@@ -25,19 +25,16 @@ const DEVICES: [(&str, &str, u32, MTLGPUFamily, bool); 7] = [("m4-pro", "Apple M4 Pro", 20, MTLGPUFamily::Apple9, false),("m5-max", "Apple M5 Max", 40, MTLGPUFamily::Apple10, true),];-const FORMATS: [(&str, u32, u32, GemmBPrologueKind); 4] = [- ("W4/ZP G32", 4, 32, GemmBPrologueKind::ScaleZeroPointDequant),+const FORMATS: [(&str, u32, u32, GemmBPrologueKind); 2] = [("W4/ZP G64", 4, 64, GemmBPrologueKind::ScaleZeroPointDequant),- ("W8/Symmetric G32", 8, 32, GemmBPrologueKind::ScaleSymmetricDequant),("W8/Symmetric G64", 8, 64, GemmBPrologueKind::ScaleSymmetricDequant),];-const SHAPES: [(&str, u32, u32); 7] = [+const SHAPES: [(&str, u32, u32); 6] = [("down", 5120, 17408),("gate", 6144, 5120),("gate-up", 34816, 5120),("projection-in", 16480, 5120),("projection-out", 5120, 6144),- ("qkv", 8192, 5120),("readout", 248320, 5120),];@@ -74,7 +71,7 @@ fn table_is_complete_and_fingerprint_is_stable() {for &(device_label, device_name, gpu_core_count, apple_gpu_family, supports_mxu) in &DEVICES {for &(format_name, bits, group, prologue) in &FORMATS {for m in 2..=7 {- for &(shape_name, n, k) in &SHAPES {+ for &(shape_name, n, k) in SHAPES.iter().filter(|shape| bits == 4 || shape.0 != "gate") {let mask = shape(n, k);let matches: Vec<_> = ROWS.iter()@@ -124,7 +121,7 @@ fn table_is_complete_and_fingerprint_is_stable() {}assert!(matched_rows.into_iter().all(|matched| matched), "route table contains an orphaned row");canonical.sort();- assert_eq!(canonical.len(), 1176);+ assert_eq!(canonical.len(), 462);assert_eq!(xxh3_64(canonical.join("\n").as_bytes()), FROZEN_PLAN_FINGERPRINT);}@@ -158,7 +155,7 @@ fn exact_lookup_rejects_non_matrix_inputs() {rht.n -= 1;assert!(GemvSpecialization::select_tile(&rht, DataType::BF16, DataType::BF16, DataType::BF16, tile).is_none());- let p = problem(7, 6144, 5120, 8, 64, GemmBPrologueKind::ScaleSymmetricDequant);+ let p = problem(7, 5120, 17408, 8, 64, GemmBPrologueKind::ScaleSymmetricDequant);assert!(matches!(route("Apple M5 Max", MTLGPUFamily::Apple10, true, &p, true), Some(QmvRoute::MainGemm(_))));assert_eq!(route("Apple M5 Max", MTLGPUFamily::Apple10, false, &p, true), None);}@@ -212,7 +209,7 @@ fn normal_routing_handles_inputs_outside_the_frozen_matrix() {#[uzu_test]fn family_lookup_requires_one_unanimous_route() {- let m1_route = problem(4, 8192, 5120, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);+ let m1_route = problem(4, 34816, 5120, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);let unanimous = problem(6, 5120, 17408, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);let families: [(&MatmulShape, MTLGPUFamily, bool, &str, &[&str]); 4] = [(&m1_route, MTLGPUFamily::Apple7, false, "Apple M1", &["Apple M1 Pro", "Apple M1 Max", "Apple M1 Ultra"]),