Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Fix U2 group size and decouple block_k and group_size (#1137)

CCYeh1b0eff2c95db
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
index 72cb91c4f..1f7bdae3f 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
@@ -128,9 +128,9 @@ impl MatmulKernel for MatmulCpuKernel {
let code_layout = quantization.code_layout();
let compatible = matches!(a_group_size, 32 | 64 | 128)
&& k.is_multiple_of(a_group_size)
- && matches!(b.group_size(), Some(32 | 64 | 128))
&& b.quantized().is_some_and(|quantized| {
matches!(quantized.mode, QuantizationMode::U2 | QuantizationMode::U4 | QuantizationMode::U8)
+ && (quantized.params.is_per_row() || matches!(quantized.group_size, 32 | 64 | 128))
});
if !compatible {
return Err(MatmulCpuError::IncompatibleA {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
index b29e37139..d2e7434cd 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -30,7 +30,7 @@ using namespace uzu::gemm;
// The A8 MXU route reads its scales into registers, so it asks for no
// threadgroup B memory.
#define GEMM_TGB_ELEMENTS \
- ((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * (int(GROUP_SIZE) + 16 / int(sizeof(BT)))) \
+ ((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * (int(BLOCK_K) + 16 / int(sizeof(BT)))) \
: GEMM_K_PARTIAL_ELEMENTS) \
: (GEMM_SMALL_BATCH \
? GEMM_K_PARTIAL_ELEMENTS \
@@ -45,7 +45,7 @@ template <
bool USE_MXU,
GemmBPrologueKind B_PROLOGUE,
uint BITS,
- uint GROUP_SIZE,
+ uint BLOCK_K,
GemmAPrologueKind A_PROLOGUE,
uint A_GROUP_SIZE>
VARIANTS(AT, bfloat)
@@ -76,7 +76,7 @@ VARIANTS(
GemmBPrologueKind::ScaleZeroPointDequant,
GemmBPrologueKind::ScaleSymmetricDequant)
VARIANTS(BITS, 0, 2, 4, 8)
-VARIANTS(GROUP_SIZE, 0, 32, 64, 128)
+VARIANTS(BLOCK_K, 0, 32, 64, 128)
VARIANTS(
A_PROLOGUE,
GemmAPrologueKind::FullPrecision,
@@ -93,12 +93,12 @@ CONSTRAINT(
GEMM_TILING == GemmTiling::Tile64x64x256_Simdgroups2x2 ||
GEMM_TILING == GemmTiling::Tile128x128x256_Simdgroups4x4))
CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))
-CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))
+CONSTRAINT((BITS == 0) == (BLOCK_K == 0))
+CONSTRAINT((BITS == 2) == (BLOCK_K == 128))
CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleZeroPointDequant || BITS == 4)
CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant || BITS == 8)
CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant || BITS == 2 || BITS == 4)
CONSTRAINT(
- (BITS == 2) == (GROUP_SIZE == 128) &&
(BITS != 2 || A_IS_INT8 ||
(DT == "bfloat" &&
(A_PROLOGUE == GemmAPrologueKind::PackedHalf || (!USE_MXU && !GEMM_SMALL_BATCH)))))
@@ -185,7 +185,7 @@ KERNEL(Gemm)(
using LeftOperand = operands::
LeftOperandFor<A_PROLOGUE, AT, ushort(A_GROUP_SIZE), A_PROLOGUE == GemmAPrologueKind::Int8Symmetric && BITS == 4>;
- using RightOperand = operands::RightOperandFor<B_PROLOGUE, ushort(BITS), ushort(GROUP_SIZE), BT>;
+ using RightOperand = operands::RightOperandFor<B_PROLOGUE, ushort(BITS), ushort(BLOCK_K), BT>;
static_assert(
NEEDS_ASYMMETRIC_WEIGHT_CORRECTION == (A_IS_INT8 && RightOperand::NEEDS_CORRECTION),
"kernel bindings and operand correction policy must agree"
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
index f713ab311..1fee7fae9 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -29,7 +29,7 @@ use crate::{
specialization::GemmSpecialization,
trellis::TrellisGemm,
},
- supports_integer_right_operand,
+ supports_integer_right_operand, weight_block_k,
},
},
},
@@ -85,7 +85,7 @@ impl GemmKernel {
specialization.use_mxu,
specialization.b_prologue,
specialization.bits_per_b.unwrap_or(0),
- specialization.b_group_size.unwrap_or(0),
+ specialization.b_block_k.unwrap_or(0),
specialization.a_prologue,
specialization.a_group_size.unwrap_or(0),
specialization.output_transform,
@@ -566,8 +566,7 @@ fn validate_int8_left_operand(
)
&& matches!(a_group_size, 32 | 64 | 128)
&& shape.k.is_multiple_of(a_group_size)
- && shape
- .b_group_size
+ && weight_block_k(shape.b_bits, shape.b_group_size)
.is_some_and(|gs| matches!(gs, 32 | 64 | 128) && shape.k.is_multiple_of(gs) && a_group_size >= gs);
if !compatible {
return Err(MatmulMetalError::IncompatibleA {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
index 5342771c7..20d174bfb 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
@@ -14,7 +14,10 @@ use crate::{
},
metal::{
gpu_types::gemm::{GemmBLayout, GemmTiling},
- kernel::matmul::gemm::{GemmEngine, GemmPlan, policy},
+ kernel::matmul::{
+ gemm::{GemmEngine, GemmPlan, policy},
+ weight_block_k,
+ },
},
},
data_type::DataType,
@@ -41,7 +44,7 @@ pub enum GemmPlanError {
/// Activation format, tiling and K alignment of the small-M GEMM for W4 and W8 (group 32 or 64) and W2 (group 128).
pub fn small_m_weights(shape: &MatmulShape) -> Option<(ActivationFormat, GemmTiling, u32)> {
let bits = shape.b_bits?;
- let group_size = shape.b_group_size?;
+ let group_size = weight_block_k(shape.b_bits, shape.b_group_size)?;
let packed_half = ActivationFormat::PackedHalf {
block_k: group_size,
bits,
@@ -261,7 +264,7 @@ pub(super) fn outer_block_k(
tiling: GemmTiling,
) -> Option<u32> {
if engine == GemmEngine::Mxu && shape.is_quant() {
- shape.b_group_size.filter(|&group_size| group_size != 0)
+ weight_block_k(shape.b_bits, shape.b_group_size).filter(|&block_k| block_k != 0)
} else {
Some(tiling.block_k()).filter(|&block_k| block_k != 0)
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
index 7eafc463a..242b5d665 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -21,7 +21,7 @@ pub(super) struct GemmSpecialization {
pub(super) a_prologue: GemmAPrologueKind,
pub(super) b_prologue: GemmBPrologueKind,
pub(super) bits_per_b: Option<u32>,
- pub(super) b_group_size: Option<u32>,
+ pub(super) b_block_k: Option<u32>,
pub(super) signed_codes: bool,
pub(super) gemm_b_layout: GemmBLayout,
pub(super) hoist_row_metadata: bool,
@@ -50,7 +50,7 @@ impl GemmSpecialization {
a_prologue,
b_prologue: shape.b_prologue,
bits_per_b: shape.b_bits,
- b_group_size: shape.b_group_size,
+ b_block_k: super::super::weight_block_k(shape.b_bits, shape.b_group_size),
signed_codes: shape.signed_codes,
gemm_b_layout,
hoist_row_metadata: row_scaled_weights
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
index 6be69842c..682282502 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -18,11 +18,11 @@ using namespace uzu::gemm;
(OUTPUT_ROW_TILE == 8 / K_SPLIT || OUTPUT_ROW_TILE == 32 / K_SPLIT) && INPUT_ROW_TILE == 1)
#define GEMV_QUANTIZED_ROW_TILE \
(K_SPLIT == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE == 1 && \
- ((BITS == 2 && GROUP_SIZE == 32 * GROUP_LANES) || (BITS == 4 && GROUP_SIZE == 16 * GROUP_LANES) || \
- (BITS == 8 && GROUP_SIZE == 8 * GROUP_LANES)))
+ ((BITS == 2 && BLOCK_K == 32 * GROUP_LANES) || (BITS == 4 && BLOCK_K == 16 * GROUP_LANES) || \
+ (BITS == 8 && BLOCK_K == 8 * GROUP_LANES)))
#define GEMV_INTERLEAVED_ROWS_TILE \
(K_SPLIT == 8 && REDUCTION_LANES == 4 && NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE == 1 && \
- BITS == 4 && GROUP_SIZE == 16 * GROUP_LANES)
+ BITS == 4 && BLOCK_K == 16 * GROUP_LANES)
#define GEMV_BATCHED_FMA_32_TILE \
(K_SPLIT == 1 && REDUCTION_LANES == 8 && NUM_SIMDGROUPS == 4 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE >= 2 && \
INPUT_ROW_TILE <= 4)
@@ -34,7 +34,7 @@ template <
typename BT,
typename DT,
GemmBPrologueKind B_PROLOGUE,
- uint GROUP_SIZE,
+ uint BLOCK_K,
uint BITS,
uint K_SPLIT,
bool INPUT_ALIGNED,
@@ -54,7 +54,7 @@ VARIANTS(
GemmBPrologueKind::ScaleBiasDequant,
GemmBPrologueKind::ScaleZeroPointDequant,
GemmBPrologueKind::ScaleSymmetricDequant)
-VARIANTS(GROUP_SIZE, 0, 32, 64, 128)
+VARIANTS(BLOCK_K, 0, 32, 64, 128)
VARIANTS(BITS, 0, 2, 4, 8)
VARIANTS(K_SPLIT, 1, 2, 4, 8)
VARIANTS(INPUT_ALIGNED, false, true)
@@ -65,13 +65,13 @@ VARIANTS(GROUP_LANES, 1, 2, 4, 8)
VARIANTS(NUM_SIMDGROUPS, 2, 4, 8)
CONSTRAINT(
- (B_PROLOGUE == GemmBPrologueKind::FullPrecision && BITS == 0 && GROUP_SIZE == 0) ||
+ (B_PROLOGUE == GemmBPrologueKind::FullPrecision && BITS == 0 && BLOCK_K == 0) ||
((B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant ||
B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) &&
- BITS == 4 && GROUP_SIZE != 0) ||
- (B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant && BITS == 8 && GROUP_SIZE != 0) ||
+ BITS == 4 && BLOCK_K != 0) ||
+ (B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant && BITS == 8 && BLOCK_K != 0) ||
(B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant && BITS == 2 && GEMV_QUANTIZED_ROW_TILE))
-CONSTRAINT((BITS == 2) == (GROUP_SIZE == 128))
+CONSTRAINT((BITS == 2) == (BLOCK_K == 128))
CONSTRAINT(INPUT_ROW_TILE == 1 || GEMV_BATCHED_INPUT)
CONSTRAINT(
GEMV_FULL_PRECISION_TILE ||
@@ -143,7 +143,7 @@ KERNEL(Gemv)(
if constexpr (BITS == 0) {
FullPrecisionBSource<Tile, AT, BT, DT, INPUT_ALIGNED, FullTile>::accumulate(result, ops, params, tile);
} else {
- QuantBSource<Tile, AT, BT, DT, B_PROLOGUE, GROUP_SIZE, BITS, INPUT_ALIGNED, FullTile>::accumulate(
+ QuantBSource<Tile, AT, BT, DT, B_PROLOGUE, BLOCK_K, BITS, INPUT_ALIGNED, FullTile>::accumulate(
result,
ops,
params,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 096060e7f..70a647a59 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -18,6 +18,7 @@ use crate::{
matmul::{
MatmulOutputWork,
gemv::policy::{self, DEFAULT_RESULTS_PER_SIMDGROUP, FP_K_BLOCK},
+ weight_block_k,
},
},
},
@@ -32,7 +33,7 @@ const MXU_TILED_KERNEL_MIN_K: u32 = 2048;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct GemvSpecialization {
b_prologue: GemmBPrologueKind,
- group_size: u32,
+ block_k: u32,
bits: u32,
output_transform: GemmDTransform,
input_aligned: bool,
@@ -60,6 +61,7 @@ impl GemvSpecialization {
) -> Option<Self> {
let is_quant = shape.is_quant();
let bits = shape.b_bits.unwrap_or(0);
+ let block_k = weight_block_k(shape.b_bits, shape.b_group_size).unwrap_or(0);
let bf16_io = input_data_type == DataType::BF16 && output_data_type == DataType::BF16;
let mxu_tiled_kernel = gemm_b_layout == GemmBLayout::MxuTiled
&& shape.b_prologue == GemmBPrologueKind::ScaleSymmetricDequant
@@ -71,11 +73,11 @@ impl GemvSpecialization {
&& shape.k >= MXU_TILED_KERNEL_MIN_K
&& shape.n >= 32 * gpu_core_count;
let tile = if is_quant && !shape.gathered && gemm_b_layout == GemmBLayout::InterleavedRows && shape.m == 1 {
- policy::interleaved_rows_tile(shape.b_group_size.unwrap_or(0), shape.k)
+ policy::interleaved_rows_tile(block_k, shape.k)
} else if is_quant && shape.gathered {
- policy::gathered_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n)
+ policy::gathered_tile(bits, block_k, shape.m, shape.n)
} else if is_quant {
- policy::quantized_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n, bf16_io)
+ policy::quantized_tile(bits, block_k, shape.m, shape.n, bf16_io)
} else {
let input_aligned = shape.k.is_multiple_of(FP_K_BLOCK);
Some(policy::fp_tile(shape.m, shape.n, shape.k, input_aligned))
@@ -123,7 +125,7 @@ impl GemvSpecialization {
let output_transform = output_transform_for_tile(shape, tile)?;
let specialization = Self {
b_prologue: shape.b_prologue,
- group_size: shape.b_group_size.unwrap_or(0),
+ block_k: weight_block_k(shape.b_bits, shape.b_group_size).unwrap_or(0),
bits,
output_transform,
input_aligned,
@@ -164,7 +166,7 @@ impl GemvSpecialization {
weights_data_type,
output_data_type,
self.b_prologue,
- self.group_size,
+ self.block_k,
self.bits,
self.k_split,
self.input_aligned,
@@ -340,7 +342,7 @@ impl GemvKernel {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => entry.insert(GemvMxuTiledMetalKernel::new(
context,
- specialization.group_size,
+ specialization.block_k,
specialization.output_transform,
specialization.signed_codes,
)?),
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
index cba7c0612..45875630a 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
@@ -23,6 +23,7 @@ use crate::{
layout,
matmul_dispatch::MatmulDispatch,
matmul_output_work::MatmulOutputWork,
+ weight_block_k,
},
},
},
@@ -193,7 +194,7 @@ impl MatmulKernel for MatmulMetalKernel {
shape: &MatmulShape,
context: &MetalContext,
) -> Option<ActivationQuantization> {
- let weight_group_size = shape.b_group_size?;
+ let weight_group_size = weight_block_k(shape.b_bits, shape.b_group_size)?;
let emit_group_sums = GemmKernel::requires_activation_group_sums(shape);
let code_layout = shape.b_bits.and_then(Int8CodeLayout::for_right_bits)?;
let quantization =
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
index 88210c211..83cb0c998 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
@@ -6,6 +6,21 @@ mod matmul_dispatch;
mod matmul_metal_kernel;
mod matmul_output_work;
+const U2_BLOCK_K: u32 = 128;
+
+fn weight_block_k(
+ bits: Option<u32>,
+ group_size: Option<u32>,
+) -> Option<u32> {
+ group_size.map(|group_size| {
+ if bits == Some(2) {
+ U2_BLOCK_K
+ } else {
+ group_size
+ }
+ })
+}
+
pub use error::MatmulMetalError;
pub use gemm::GemmKernel;
pub use matmul_metal_kernel::MatmulMetalKernel;
diff --git a/crates/uzu-engine/src/encodable_block/weight_matrix.rs b/crates/uzu-engine/src/encodable_block/weight_matrix.rs
index 3b22ef13c..9bbe22fd5 100644
--- a/crates/uzu-engine/src/encodable_block/weight_matrix.rs
+++ b/crates/uzu-engine/src/encodable_block/weight_matrix.rs
@@ -5,10 +5,7 @@ use crate::{
backends::common::{
Backend,
gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},
- kernel::{
- activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
- matmul::{MatmulB, QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection, WeightsLayout},
- },
+ kernel::matmul::{MatmulB, QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection, WeightsLayout},
},
config::weight_matrix::{AnyWeightMatrixSpec, Layout},
data_type::DataType,
@@ -118,7 +115,7 @@ impl<B: Backend> WeightMatrix<B> {
}
let (rows, columns) = physical_shape(&layout, output_dim, input_dim);
- let Some(mut info) = quantization_info else {
+ let Some(info) = quantization_info else {
let values = tree.read_buffer("weights", &[rows, columns], data_type).await?;
return Ok(Self {
values,
@@ -140,9 +137,6 @@ impl<B: Backend> WeightMatrix<B> {
)));
}
let groups = columns.div_ceil(group_size);
- if info.mode == QuantizationMode::U2 {
- info.group_size = ACTIVATION_SCALE_GROUP_SIZE;
- }
let values = tree.read_buffer("weights", &[rows, columns / packing_divisor], storage_data_type).await?;
let params = QuantParams::new(params_layout, rows, groups);
diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rs
index cf6d30a2d..11ef1a923 100644
--- a/crates/uzu-engine/unit/common/matmul/quant.rs
+++ b/crates/uzu-engine/unit/common/matmul/quant.rs
@@ -418,7 +418,11 @@ fn quant_b_variant<TB: BufferRef, T: DataTypeRepr + Float>(
correction,
params,
mode: input.mode,
- group_size: input.group_size,
+ group_size: if input.one_group_per_row {
+ input.k
+ } else {
+ input.group_size
+ },
signed_codes: input.signed_codes,
weights_layout,
})