A high-performance inference engine for AI models
Fix U2 group size and decouple block_k and group_size (#1137)
CCYeh1b0eff2c95db
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rsindex 72cb91c4f..1f7bdae3f 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs@@ -128,9 +128,9 @@ impl MatmulKernel for MatmulCpuKernel {let code_layout = quantization.code_layout();let compatible = matches!(a_group_size, 32 | 64 | 128)&& k.is_multiple_of(a_group_size)- && matches!(b.group_size(), Some(32 | 64 | 128))&& b.quantized().is_some_and(|quantized| {matches!(quantized.mode, QuantizationMode::U2 | QuantizationMode::U4 | QuantizationMode::U8)+ && (quantized.params.is_per_row() || matches!(quantized.group_size, 32 | 64 | 128))});if !compatible {return Err(MatmulCpuError::IncompatibleA {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metalindex b29e37139..d2e7434cd 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -30,7 +30,7 @@ using namespace uzu::gemm;// The A8 MXU route reads its scales into registers, so it asks for no// threadgroup B memory.#define GEMM_TGB_ELEMENTS \- ((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * (int(GROUP_SIZE) + 16 / int(sizeof(BT)))) \+ ((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * (int(BLOCK_K) + 16 / int(sizeof(BT)))) \: GEMM_K_PARTIAL_ELEMENTS) \: (GEMM_SMALL_BATCH \? GEMM_K_PARTIAL_ELEMENTS \@@ -45,7 +45,7 @@ template <bool USE_MXU,GemmBPrologueKind B_PROLOGUE,uint BITS,- uint GROUP_SIZE,+ uint BLOCK_K,GemmAPrologueKind A_PROLOGUE,uint A_GROUP_SIZE>VARIANTS(AT, bfloat)@@ -76,7 +76,7 @@ VARIANTS(GemmBPrologueKind::ScaleZeroPointDequant,GemmBPrologueKind::ScaleSymmetricDequant)VARIANTS(BITS, 0, 2, 4, 8)-VARIANTS(GROUP_SIZE, 0, 32, 64, 128)+VARIANTS(BLOCK_K, 0, 32, 64, 128)VARIANTS(A_PROLOGUE,GemmAPrologueKind::FullPrecision,@@ -93,12 +93,12 @@ CONSTRAINT(GEMM_TILING == GemmTiling::Tile64x64x256_Simdgroups2x2 ||GEMM_TILING == GemmTiling::Tile128x128x256_Simdgroups4x4))CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))-CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))+CONSTRAINT((BITS == 0) == (BLOCK_K == 0))+CONSTRAINT((BITS == 2) == (BLOCK_K == 128))CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleZeroPointDequant || BITS == 4)CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant || BITS == 8)CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant || BITS == 2 || BITS == 4)CONSTRAINT(- (BITS == 2) == (GROUP_SIZE == 128) &&(BITS != 2 || A_IS_INT8 ||(DT == "bfloat" &&(A_PROLOGUE == GemmAPrologueKind::PackedHalf || (!USE_MXU && !GEMM_SMALL_BATCH)))))@@ -185,7 +185,7 @@ KERNEL(Gemm)(using LeftOperand = operands::LeftOperandFor<A_PROLOGUE, AT, ushort(A_GROUP_SIZE), A_PROLOGUE == GemmAPrologueKind::Int8Symmetric && BITS == 4>;- using RightOperand = operands::RightOperandFor<B_PROLOGUE, ushort(BITS), ushort(GROUP_SIZE), BT>;+ using RightOperand = operands::RightOperandFor<B_PROLOGUE, ushort(BITS), ushort(BLOCK_K), BT>;static_assert(NEEDS_ASYMMETRIC_WEIGHT_CORRECTION == (A_IS_INT8 && RightOperand::NEEDS_CORRECTION),"kernel bindings and operand correction policy must agree"diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rsindex f713ab311..1fee7fae9 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -29,7 +29,7 @@ use crate::{specialization::GemmSpecialization,trellis::TrellisGemm,},- supports_integer_right_operand,+ supports_integer_right_operand, weight_block_k,},},},@@ -85,7 +85,7 @@ impl GemmKernel {specialization.use_mxu,specialization.b_prologue,specialization.bits_per_b.unwrap_or(0),- specialization.b_group_size.unwrap_or(0),+ specialization.b_block_k.unwrap_or(0),specialization.a_prologue,specialization.a_group_size.unwrap_or(0),specialization.output_transform,@@ -566,8 +566,7 @@ fn validate_int8_left_operand()&& matches!(a_group_size, 32 | 64 | 128)&& shape.k.is_multiple_of(a_group_size)- && shape- .b_group_size+ && weight_block_k(shape.b_bits, shape.b_group_size).is_some_and(|gs| matches!(gs, 32 | 64 | 128) && shape.k.is_multiple_of(gs) && a_group_size >= gs);if !compatible {return Err(MatmulMetalError::IncompatibleA {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rsindex 5342771c7..20d174bfb 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs@@ -14,7 +14,10 @@ use crate::{},metal::{gpu_types::gemm::{GemmBLayout, GemmTiling},- kernel::matmul::gemm::{GemmEngine, GemmPlan, policy},+ kernel::matmul::{+ gemm::{GemmEngine, GemmPlan, policy},+ weight_block_k,+ },},},data_type::DataType,@@ -41,7 +44,7 @@ pub enum GemmPlanError {/// Activation format, tiling and K alignment of the small-M GEMM for W4 and W8 (group 32 or 64) and W2 (group 128).pub fn small_m_weights(shape: &MatmulShape) -> Option<(ActivationFormat, GemmTiling, u32)> {let bits = shape.b_bits?;- let group_size = shape.b_group_size?;+ let group_size = weight_block_k(shape.b_bits, shape.b_group_size)?;let packed_half = ActivationFormat::PackedHalf {block_k: group_size,bits,@@ -261,7 +264,7 @@ pub(super) fn outer_block_k(tiling: GemmTiling,) -> Option<u32> {if engine == GemmEngine::Mxu && shape.is_quant() {- shape.b_group_size.filter(|&group_size| group_size != 0)+ weight_block_k(shape.b_bits, shape.b_group_size).filter(|&block_k| block_k != 0)} else {Some(tiling.block_k()).filter(|&block_k| block_k != 0)}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rsindex 7eafc463a..242b5d665 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -21,7 +21,7 @@ pub(super) struct GemmSpecialization {pub(super) a_prologue: GemmAPrologueKind,pub(super) b_prologue: GemmBPrologueKind,pub(super) bits_per_b: Option<u32>,- pub(super) b_group_size: Option<u32>,+ pub(super) b_block_k: Option<u32>,pub(super) signed_codes: bool,pub(super) gemm_b_layout: GemmBLayout,pub(super) hoist_row_metadata: bool,@@ -50,7 +50,7 @@ impl GemmSpecialization {a_prologue,b_prologue: shape.b_prologue,bits_per_b: shape.b_bits,- b_group_size: shape.b_group_size,+ b_block_k: super::super::weight_block_k(shape.b_bits, shape.b_group_size),signed_codes: shape.signed_codes,gemm_b_layout,hoist_row_metadata: row_scaled_weightsdiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metalindex 6be69842c..682282502 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -18,11 +18,11 @@ using namespace uzu::gemm;(OUTPUT_ROW_TILE == 8 / K_SPLIT || OUTPUT_ROW_TILE == 32 / K_SPLIT) && INPUT_ROW_TILE == 1)#define GEMV_QUANTIZED_ROW_TILE \(K_SPLIT == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE == 1 && \- ((BITS == 2 && GROUP_SIZE == 32 * GROUP_LANES) || (BITS == 4 && GROUP_SIZE == 16 * GROUP_LANES) || \- (BITS == 8 && GROUP_SIZE == 8 * GROUP_LANES)))+ ((BITS == 2 && BLOCK_K == 32 * GROUP_LANES) || (BITS == 4 && BLOCK_K == 16 * GROUP_LANES) || \+ (BITS == 8 && BLOCK_K == 8 * GROUP_LANES)))#define GEMV_INTERLEAVED_ROWS_TILE \(K_SPLIT == 8 && REDUCTION_LANES == 4 && NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE == 1 && \- BITS == 4 && GROUP_SIZE == 16 * GROUP_LANES)+ BITS == 4 && BLOCK_K == 16 * GROUP_LANES)#define GEMV_BATCHED_FMA_32_TILE \(K_SPLIT == 1 && REDUCTION_LANES == 8 && NUM_SIMDGROUPS == 4 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE >= 2 && \INPUT_ROW_TILE <= 4)@@ -34,7 +34,7 @@ template <typename BT,typename DT,GemmBPrologueKind B_PROLOGUE,- uint GROUP_SIZE,+ uint BLOCK_K,uint BITS,uint K_SPLIT,bool INPUT_ALIGNED,@@ -54,7 +54,7 @@ VARIANTS(GemmBPrologueKind::ScaleBiasDequant,GemmBPrologueKind::ScaleZeroPointDequant,GemmBPrologueKind::ScaleSymmetricDequant)-VARIANTS(GROUP_SIZE, 0, 32, 64, 128)+VARIANTS(BLOCK_K, 0, 32, 64, 128)VARIANTS(BITS, 0, 2, 4, 8)VARIANTS(K_SPLIT, 1, 2, 4, 8)VARIANTS(INPUT_ALIGNED, false, true)@@ -65,13 +65,13 @@ VARIANTS(GROUP_LANES, 1, 2, 4, 8)VARIANTS(NUM_SIMDGROUPS, 2, 4, 8)CONSTRAINT(- (B_PROLOGUE == GemmBPrologueKind::FullPrecision && BITS == 0 && GROUP_SIZE == 0) ||+ (B_PROLOGUE == GemmBPrologueKind::FullPrecision && BITS == 0 && BLOCK_K == 0) ||((B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant ||B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) &&- BITS == 4 && GROUP_SIZE != 0) ||- (B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant && BITS == 8 && GROUP_SIZE != 0) ||+ BITS == 4 && BLOCK_K != 0) ||+ (B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant && BITS == 8 && BLOCK_K != 0) ||(B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant && BITS == 2 && GEMV_QUANTIZED_ROW_TILE))-CONSTRAINT((BITS == 2) == (GROUP_SIZE == 128))+CONSTRAINT((BITS == 2) == (BLOCK_K == 128))CONSTRAINT(INPUT_ROW_TILE == 1 || GEMV_BATCHED_INPUT)CONSTRAINT(GEMV_FULL_PRECISION_TILE ||@@ -143,7 +143,7 @@ KERNEL(Gemv)(if constexpr (BITS == 0) {FullPrecisionBSource<Tile, AT, BT, DT, INPUT_ALIGNED, FullTile>::accumulate(result, ops, params, tile);} else {- QuantBSource<Tile, AT, BT, DT, B_PROLOGUE, GROUP_SIZE, BITS, INPUT_ALIGNED, FullTile>::accumulate(+ QuantBSource<Tile, AT, BT, DT, B_PROLOGUE, BLOCK_K, BITS, INPUT_ALIGNED, FullTile>::accumulate(result,ops,params,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 096060e7f..70a647a59 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -18,6 +18,7 @@ use crate::{matmul::{MatmulOutputWork,gemv::policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK},+ weight_block_k,},},},@@ -32,7 +33,7 @@ const MXU_TILED_KERNEL_MIN_K: u32 = 2048;#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct GemvSpecialization {b_prologue: GemmBPrologueKind,- group_size: u32,+ block_k: u32,bits: u32,output_transform: GemmDTransform,input_aligned: bool,@@ -60,6 +61,7 @@ impl GemvSpecialization {) -> Option<Self> {let is_quant = shape.is_quant();let bits = shape.b_bits.unwrap_or(0);+ let block_k = weight_block_k(shape.b_bits, shape.b_group_size).unwrap_or(0);let bf16_io = input_data_type == DataType::BF16 && output_data_type == DataType::BF16;let mxu_tiled_kernel = gemm_b_layout == GemmBLayout::MxuTiled&& shape.b_prologue == GemmBPrologueKind::ScaleSymmetricDequant@@ -71,11 +73,11 @@ impl GemvSpecialization {&& shape.k >= MXU_TILED_KERNEL_MIN_K&& shape.n >= 32 * gpu_core_count;let tile = if is_quant && !shape.gathered && gemm_b_layout == GemmBLayout::InterleavedRows && shape.m == 1 {- policy::interleaved_rows_tile(shape.b_group_size.unwrap_or(0), shape.k)+ policy::interleaved_rows_tile(block_k, shape.k)} else if is_quant && shape.gathered {- policy::gathered_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n)+ policy::gathered_tile(bits, block_k, shape.m, shape.n)} else if is_quant {- policy::quantized_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n, bf16_io)+ policy::quantized_tile(bits, block_k, shape.m, shape.n, bf16_io)} else {let input_aligned = shape.k.is_multiple_of(FP_K_BLOCK);Some(policy::fp_tile(shape.m, shape.n, shape.k, input_aligned))@@ -123,7 +125,7 @@ impl GemvSpecialization {let output_transform = output_transform_for_tile(shape, tile)?;let specialization = Self {b_prologue: shape.b_prologue,- group_size: shape.b_group_size.unwrap_or(0),+ block_k: weight_block_k(shape.b_bits, shape.b_group_size).unwrap_or(0),bits,output_transform,input_aligned,@@ -164,7 +166,7 @@ impl GemvSpecialization {weights_data_type,output_data_type,self.b_prologue,- self.group_size,+ self.block_k,self.bits,self.k_split,self.input_aligned,@@ -340,7 +342,7 @@ impl GemvKernel {Entry::Occupied(entry) => entry.into_mut(),Entry::Vacant(entry) => entry.insert(GemvMxuTiledMetalKernel::new(context,- specialization.group_size,+ specialization.block_k,specialization.output_transform,specialization.signed_codes,)?),diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rsindex cba7c0612..45875630a 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs@@ -23,6 +23,7 @@ use crate::{layout,matmul_dispatch::MatmulDispatch,matmul_output_work::MatmulOutputWork,+ weight_block_k,},},},@@ -193,7 +194,7 @@ impl MatmulKernel for MatmulMetalKernel {shape: &MatmulShape,context: &MetalContext,) -> Option<ActivationQuantization> {- let weight_group_size = shape.b_group_size?;+ let weight_group_size = weight_block_k(shape.b_bits, shape.b_group_size)?;let emit_group_sums = GemmKernel::requires_activation_group_sums(shape);let code_layout = shape.b_bits.and_then(Int8CodeLayout::for_right_bits)?;let quantization =diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rsindex 88210c211..83cb0c998 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs@@ -6,6 +6,21 @@ mod matmul_dispatch;mod matmul_metal_kernel;mod matmul_output_work;+const U2_BLOCK_K: u32 = 128;++fn weight_block_k(+ bits: Option<u32>,+ group_size: Option<u32>,+) -> Option<u32> {+ group_size.map(|group_size| {+ if bits == Some(2) {+ U2_BLOCK_K+ } else {+ group_size+ }+ })+}+pub use error::MatmulMetalError;pub use gemm::GemmKernel;pub use matmul_metal_kernel::MatmulMetalKernel;diff --git a/crates/uzu-engine/src/encodable_block/weight_matrix.rs b/crates/uzu-engine/src/encodable_block/weight_matrix.rsindex 3b22ef13c..9bbe22fd5 100644--- a/crates/uzu-engine/src/encodable_block/weight_matrix.rs+++ b/crates/uzu-engine/src/encodable_block/weight_matrix.rs@@ -5,10 +5,7 @@ use crate::{backends::common::{Backend,gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},- kernel::{- activation_transform::ACTIVATION_SCALE_GROUP_SIZE,- matmul::{MatmulB, QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection, WeightsLayout},- },+ kernel::matmul::{MatmulB, QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection, WeightsLayout},},config::weight_matrix::{AnyWeightMatrixSpec, Layout},data_type::DataType,@@ -118,7 +115,7 @@ impl<B: Backend> WeightMatrix<B> {}let (rows, columns) = physical_shape(&layout, output_dim, input_dim);- let Some(mut info) = quantization_info else {+ let Some(info) = quantization_info else {let values = tree.read_buffer("weights", &[rows, columns], data_type).await?;return Ok(Self {values,@@ -140,9 +137,6 @@ impl<B: Backend> WeightMatrix<B> {)));}let groups = columns.div_ceil(group_size);- if info.mode == QuantizationMode::U2 {- info.group_size = ACTIVATION_SCALE_GROUP_SIZE;- }let values = tree.read_buffer("weights", &[rows, columns / packing_divisor], storage_data_type).await?;let params = QuantParams::new(params_layout, rows, groups);diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rsindex cf6d30a2d..11ef1a923 100644--- a/crates/uzu-engine/unit/common/matmul/quant.rs+++ b/crates/uzu-engine/unit/common/matmul/quant.rs@@ -418,7 +418,11 @@ fn quant_b_variant<TB: BufferRef, T: DataTypeRepr + Float>(correction,params,mode: input.mode,- group_size: input.group_size,+ group_size: if input.one_group_per_row {+ input.k+ } else {+ input.group_size+ },signed_codes: input.signed_codes,weights_layout,})