A high-performance inference engine for AI models
Support 2 bit symmetric row scale (Dash, MXU) (#1054)
CCYeh69c64cbe2aa9
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/matmul.rs b/crates/uzu-engine/src/backends/common/gpu_types/matmul.rsindex ce4714f04..9d338c70b 100644--- a/crates/uzu-engine/src/backends/common/gpu_types/matmul.rs+++ b/crates/uzu-engine/src/backends/common/gpu_types/matmul.rs@@ -1 +1,4 @@pub const QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT: u32 = 4;+pub const MXU_TILED_BLOCK_ROWS: u32 = 32;+pub const MXU_TILED_W2_BLOCK_ROWS: u32 = 16;+pub const MXU_TILED_PREPARE_THREADS: u32 = 1024;diff --git a/crates/uzu-engine/src/backends/common/gpu_types/quantization.rs b/crates/uzu-engine/src/backends/common/gpu_types/quantization.rsindex be0cce87c..d51b53b7c 100644--- a/crates/uzu-engine/src/backends/common/gpu_types/quantization.rs+++ b/crates/uzu-engine/src/backends/common/gpu_types/quantization.rs@@ -13,6 +13,8 @@ pub enum QuantizationMode {I8,#[serde(rename = "uint8")]U8,+ #[serde(rename = "uint2")]+ U2,}impl QuantizationMode {@@ -24,7 +26,7 @@ impl QuantizationMode {pub fn storage_type(&self) -> DataType {match self {- QuantizationMode::U4 => DataType::U8,+ QuantizationMode::U2 | QuantizationMode::U4 => DataType::U8,QuantizationMode::I8 => DataType::I8,QuantizationMode::U8 => DataType::U8,}@@ -32,6 +34,7 @@ impl QuantizationMode {pub fn weight_codes_sign_flip_mask(&self) -> Option<u8> {match self {+ QuantizationMode::U2 => None,QuantizationMode::U4 => Some(0x88),QuantizationMode::I8 => None,QuantizationMode::U8 => Some(0x80),@@ -42,6 +45,7 @@ impl QuantizationMode {impl From<QuantizationMode> for DataType {fn from(val: QuantizationMode) -> Self {match val {+ QuantizationMode::U2 => DataType::U2,QuantizationMode::U4 => DataType::U4,QuantizationMode::I8 => DataType::I8,QuantizationMode::U8 => DataType::U8,diff --git a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rsindex d5ff4cc24..462d291e1 100644--- a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs+++ b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs@@ -33,12 +33,13 @@ impl ActivationQuantization {emit_group_sums: bool,code_layout: Int8CodeLayout,) -> Option<Self> {- (scale_group_size == ACTIVATION_SCALE_GROUP_SIZE && (!emit_group_sums || matches!(weight_group_size, 32 | 64)))- .then_some(Self {- scale_group_size,- sum_group_size: emit_group_sums.then_some(weight_group_size),- code_layout,- })+ (scale_group_size == ACTIVATION_SCALE_GROUP_SIZE+ && (!emit_group_sums || matches!(weight_group_size, 32 | 64 | 128)))+ .then_some(Self {+ scale_group_size,+ sum_group_size: emit_group_sums.then_some(weight_group_size),+ code_layout,+ })}pub const fn scale_group_size(self) -> u32 {diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rsindex dc69c1203..14462b617 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs@@ -18,8 +18,8 @@ impl Int8CodeLayout {pub const fn for_right_bits(bits: u32) -> Option<Self> {match bits {+ 2 | 8 => Some(Self::Sequential),4 => Some(Self::GroupedByNibble),- 8 => Some(Self::Sequential),_ => None,}}diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rsindex 1d2030d4a..b611a716d 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs@@ -59,6 +59,10 @@ impl QuantParams {}}+ pub const fn is_per_row(self) -> bool {+ self.group_count == 1+ }+pub const fn layout(self) -> QuantParamsLayout {self.layout}@@ -101,15 +105,16 @@ impl QuantParams {packing_divisor: u32,) -> QuantParamsStrides {let row_values = self.padded_values_per_row(packing_divisor);- match self.layout {- QuantParamsLayout::OutputGroup => QuantParamsStrides {- output_stride: row_values,- group_stride: 1,- },- QuantParamsLayout::GroupOutput => QuantParamsStrides {- output_stride: 1,- group_stride: row_values,- },+ let (output_stride, mut group_stride) = match self.layout {+ QuantParamsLayout::OutputGroup => (row_values, 1),+ QuantParamsLayout::GroupOutput => (1, row_values),+ };+ if self.is_per_row() {+ group_stride = 0;+ }+ QuantParamsStrides {+ output_stride,+ group_stride,}}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs b/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rsindex 21b36d47c..df79d2fa7 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs@@ -79,6 +79,7 @@ pub fn input_embedding_lookup<T: DataTypeRepr + Float>(let scale = unsafe { (*scales.add(index)).to_f32().unwrap() };let offset = token * weights_stride;let code = match quantization_mode {+ QuantizationMode::U2 => unreachable!("2-bit weights are not embedding tables"),QuantizationMode::U4 => read_u4(values, 2 * offset + column) as f32,QuantizationMode::I8 => (unsafe { *(values as *const i8).add(offset + column) }) as f32,QuantizationMode::U8 => (unsafe { *values.add(offset + column) }) as f32,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rsindex 7716791dc..72cb91c4f 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs@@ -129,8 +129,9 @@ impl MatmulKernel for MatmulCpuKernel {let compatible = matches!(a_group_size, 32 | 64 | 128)&& k.is_multiple_of(a_group_size)&& matches!(b.group_size(), Some(32 | 64 | 128))- && b.quantized()- .is_some_and(|quantized| matches!(quantized.mode, QuantizationMode::U4 | QuantizationMode::U8));+ && b.quantized().is_some_and(|quantized| {+ matches!(quantized.mode, QuantizationMode::U2 | QuantizationMode::U4 | QuantizationMode::U8)+ });if !compatible {return Err(MatmulCpuError::IncompatibleA {path: "CpuMatmul",diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.h b/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.hindex 7b0cd7bbe..33993c836 100644--- a/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.h+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.h@@ -7,6 +7,12 @@ using namespace metal;namespace uzu::matmul {static constant constexpr uint32_t QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT = 4;+static constant constexpr uint32_t MXU_TILED_BLOCK_ROWS = 32;++static constant constexpr uint32_t MXU_TILED_W2_BLOCK_ROWS = 16;++static constant constexpr uint32_t MXU_TILED_PREPARE_THREADS = 1024;+typedef struct {uint32_t M;uint32_t N;diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/quantization.h b/crates/uzu-engine/src/backends/metal/kernel/generated/quantization.hindex 01c1ffba7..c832a2a93 100644--- a/crates/uzu-engine/src/backends/metal/kernel/generated/quantization.h+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/quantization.h@@ -9,5 +9,6 @@ enum class QuantizationMode : uint32_t {U4 = 0,I8 = 1,U8 = 2,+ U2 = 3,};} // namespace uzu::quantizationdiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/block_geometry.hindex 19ba79a27..5f2c9ddf5 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h@@ -23,9 +23,12 @@ static METAL_FUNC uint2 morton_tile_id(uint2 threadgroup_position) {return uint2(morton_expand_bits(threadgroup_position.x), morton_expand_bits(threadgroup_position.x >> 1));}+template <bool MAY_USE_MORTON = true>static METAL_FUNC uint2 tile_id(uint2 threadgroup_position, const constant uzu::matmul::GemmParams* params) {- if (params->use_morton) {- return morton_tile_id(threadgroup_position);+ if constexpr (MAY_USE_MORTON) {+ if (params->use_morton) {+ return morton_tile_id(threadgroup_position);+ }}return threadgroup_position;}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.hindex 23efd27b3..440c479d4 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h@@ -5,8 +5,10 @@using namespace metal;-template <typename T, typename BiasT, bool SINGLE_SIMDGROUP = false>+template <typename T, typename BiasT, bool SINGLE_SIMDGROUP = false, typename Source>static METAL_FUNC void apply_output_random_hadamard_transform(+ Source source,+ uint source_stride,device T* output_block,const device int32_t* rht_factors_block,const device BiasT* bias_block,@@ -25,12 +27,14 @@ static METAL_FUNC void apply_output_random_hadamard_transform(const ushort row_local = ushort(cell / stripes_per_row);const ushort stripe = ushort(cell % stripes_per_row);const ushort col_local = stripe * ushort(HADAMARD_TRANSFORM_BLOCK_SIZE) + simd_lane;- const size_t output_index = size_t(row_local) * size_t(leading_dimension_d) + size_t(col_local);- T transformed =- simdgroup_output_random_hadamard_transform(simd_lane, output_block[output_index], rht_factors_block[col_local]);+ T transformed = simdgroup_output_random_hadamard_transform(+ simd_lane,+ source[size_t(row_local) * size_t(source_stride) + size_t(col_local)],+ rht_factors_block[col_local]+ );if (apply_bias) {transformed = T(float(transformed) + float(bias_block[col_local]));}- output_block[output_index] = transformed;+ output_block[size_t(row_local) * size_t(leading_dimension_d) + size_t(col_local)] = transformed;}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 2776903b2..11d3eca06 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -52,7 +52,7 @@ struct MxuMmaCore {static_assert(SIMDGROUPS_PER_K == 1 || SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN == 1, "K slices share one tile");UZU_CONST ushort SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;UZU_CONST ushort SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;- UZU_CONST ushort SIMDGROUP_BLOCK_K = static_cast<ushort>(MXU_SIMDGROUP_BLOCK_K);+ UZU_CONST ushort SIMDGROUP_BLOCK_K = Right::MMA_BLOCK_K;UZU_CONST bool TRANSPOSE_RIGHT = TRANSPOSE_B;UZU_CONST ushort THREADGROUP_BLOCK_K_FP = gemm_tiling_block_k(GEMM_TILING);static_assert(@@ -86,9 +86,10 @@ struct MxuMmaCore {const uint partition = thread_context.threadgroup_position.z;const uint tile_y = thread_context.threadgroup_position.y;- const uint2 tile = tile_id(uint2(thread_context.threadgroup_position.x, tile_y), params);+ constexpr bool MAY_USE_MORTON = !Right::QUANTIZED;+ const uint2 tile = tile_id<MAY_USE_MORTON>(uint2(thread_context.threadgroup_position.x, tile_y), params);const auto geometry = ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::compute(tile, params);- if (geometry.out_of_bounds) {+ if (MAY_USE_MORTON && geometry.out_of_bounds) {return;}@@ -126,69 +127,108 @@ struct MxuMmaCore {.abs_row_base = uint(geometry.block_row_start) + tile_row_offset,};+ const bool simdgroup_rows_complete =+ (alignment.contains(GemmAlignment::M) || (simdgroup_limit_m == SIMDGROUP_BLOCK_M)) &&+ ((Right::QUANTIZED && SIMDGROUPS_PER_ROW == 1) || simdgroup_limit_m > 0);+ const bool simdgroup_columns_complete =+ (alignment.contains(GemmAlignment::N) || (simdgroup_limit_n == SIMDGROUP_BLOCK_N)) &&+ ((Right::QUANTIZED && SIMDGROUPS_PER_COLUMN == 1) || simdgroup_limit_n > 0);+const bool apply_scale = output_transform.contains(GemmDTransform::SCALE);const bool apply_bias = output_transform.contains(GemmDTransform::BIAS);const bool apply_rht = output_transform.contains(GemmDTransform::RHT);const device RightElementType* bias_simdgroup = output_bias + size_t(block_col) + size_t(tile_col_offset);- dispatch_bool(- (alignment.contains(GemmAlignment::M) || (simdgroup_limit_m == SIMDGROUP_BLOCK_M)) && simdgroup_limit_m > 0,- [&](auto aligned_m) {- dispatch_bool(- (alignment.contains(GemmAlignment::N) || (simdgroup_limit_n == SIMDGROUP_BLOCK_N)) &&- simdgroup_limit_n > 0,- [&](auto aligned_n) {- AccumFragment accumulator_tile =- Schedule::template launch<MxuMmaCore, aligned_m.value, aligned_n.value>(- left,- right,- b_shared,- params,- tile_context,- alignment,- thread_context- );-- if (!accumulator_tile.template reduce_k_slices<SIMDGROUPS_PER_K, 1>(- reinterpret_cast<threadgroup float*>(b_shared),- thread_context- )) {- return;- }-- if (apply_scale) {- const AccumulatorType scale = AccumulatorType(params->ab_scale);- accumulator_tile.map([&](auto value) { return value * scale; });- }-- if (apply_bias && !apply_rht) {- accumulator_tile.map_coords(thread_context.simd_lane_id, [&](short, short col, auto value) {- if constexpr (aligned_n.value) {- return value + AccumulatorType(bias_simdgroup[col]);- } else {- if (col < simdgroup_limit_n) {- return value + AccumulatorType(bias_simdgroup[col]);- }- return value;- }- });- }-- if constexpr (aligned_m.value && aligned_n.value) {- accumulator_tile.store(thread_context.simd_lane_id, d_simdgroup, int(params->leading_dimension_d));- } else {- accumulator_tile.template store_safe<aligned_m.value, aligned_n.value>(- thread_context.simd_lane_id,- d_simdgroup,- int(params->leading_dimension_d),- short2(simdgroup_limit_n, simdgroup_limit_m)- );- }+ threadgroup OutputElementType* tile_stage = reinterpret_cast<threadgroup OutputElementType*>(b_shared);+ if constexpr (SIMDGROUPS_PER_K > 1) {+ static_assert(Left::QUANTIZED, "K slices park their partials in b_shared, which int8 activations leave free");+ static_assert(THREADGROUP_BLOCK_N == HADAMARD_TRANSFORM_BLOCK_SIZE, "the staged RHT runs one stripe per row");+ static_assert(sizeof(OutputElementType) <= sizeof(AccumulatorType), "the tile must fit the partials");+ }++ dispatch_bool(simdgroup_rows_complete, [&](auto aligned_m) {+ dispatch_bool(simdgroup_columns_complete, [&](auto aligned_n) {+ AccumFragment accumulator_tile = Schedule::template launch<MxuMmaCore, aligned_m.value, aligned_n.value>(+ left,+ right,+ b_shared,+ params,+ tile_context,+ alignment,+ thread_context+ );++ if (!accumulator_tile.template reduce_k_slices<SIMDGROUPS_PER_K, 1>(+ reinterpret_cast<threadgroup float*>(b_shared),+ thread_context+ )) {+ return;+ }++ if (apply_scale) {+ const AccumulatorType scale = AccumulatorType(params->ab_scale);+ accumulator_tile.map([&](auto value) { return value * scale; });+ }++ if constexpr (SIMDGROUPS_PER_K > 1) {+ if (apply_rht) {+ simdgroup_barrier(mem_flags::mem_threadgroup);+ accumulator_tile.template store_safe<aligned_m.value, aligned_n.value>(+ thread_context.simd_lane_id,+ tile_stage,+ int(THREADGROUP_BLOCK_N),+ short2(simdgroup_limit_n, simdgroup_limit_m)+ );+ return;+ }+ }++ if (apply_bias && !apply_rht) {+ accumulator_tile.map_coords(thread_context.simd_lane_id, [&](short, short col, auto value) {+ if constexpr (aligned_n.value) {+ return value + AccumulatorType(bias_simdgroup[col]);+ } else {+ if (col < simdgroup_limit_n) {+ return value + AccumulatorType(bias_simdgroup[col]);}+ return value;+ }+ });+ }++ if constexpr (aligned_m.value && aligned_n.value) {+ accumulator_tile.store(thread_context.simd_lane_id, d_simdgroup, int(params->leading_dimension_d));+ } else {+ accumulator_tile.template store_safe<aligned_m.value, aligned_n.value>(+ thread_context.simd_lane_id,+ d_simdgroup,+ int(params->leading_dimension_d),+ short2(simdgroup_limit_n, simdgroup_limit_m));}- );+ });+ });++ if constexpr (SIMDGROUPS_PER_K > 1) {+ if (apply_rht) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ apply_output_random_hadamard_transform<OutputElementType, RightElementType>(+ tile_stage,+ uint(THREADGROUP_BLOCK_N),+ d + block_row * params->leading_dimension_d + block_col,+ rht_factors + block_col,+ output_bias + block_col,+ apply_bias,+ ushort(min(int(THREADGROUP_BLOCK_M), int(params->M) - int(block_row))),+ ushort(THREADGROUP_BLOCK_N),+ params->leading_dimension_d,+ SIMDGROUPS_PER_K,+ thread_context+ );+ return;+ }+ }if (k_slice != 0) {return;@@ -203,6 +243,8 @@ struct MxuMmaCore {}simdgroup_barrier(mem_flags::mem_device);apply_output_random_hadamard_transform<OutputElementType, RightElementType, true>(+ d_simdgroup,+ params->leading_dimension_d,d_simdgroup,rht_factors + block_col + tile_col_offset,bias_simdgroup,@@ -225,7 +267,9 @@ struct MxuMmaCore {device OutputElementType* d_block = d + block_row * params->leading_dimension_d + block_col;const ushort tile_block_rows = ushort(min(int(THREADGROUP_BLOCK_M), int(params->M) - int(block_row)));const ushort tile_block_cols = ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(block_col)));- apply_output_random_hadamard_transform(+ apply_output_random_hadamard_transform<OutputElementType, RightElementType>(+ d_block,+ params->leading_dimension_d,d_block,rht_factors + block_col,output_bias + block_col,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_tiled_layout.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_tiled_layout.hnew file mode 100644index 000000000..82c84900d--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_tiled_layout.h@@ -0,0 +1,23 @@+#pragma once++#include "../../../common/defines.h"+#include "../../../generated/matmul.h"+#include "../../common/mxu_fragment/ops.h"++namespace uzu {+namespace gemm {++template <uint BITS>+struct MxuTiledLayout {+ using Ops = uzu::matmul::MxuFragmentOps<>;+ UZU_CONST uint BLOCK_ROWS = BITS == 2 ? uzu::matmul::MXU_TILED_W2_BLOCK_ROWS : uzu::matmul::MXU_TILED_BLOCK_ROWS;+ UZU_CONST uint LANE_VECTOR_CHUNKS = uzu::matmul::MXU_TILED_BLOCK_ROWS / BLOCK_ROWS;+ UZU_CONST uint ROW_FRAGMENTS = BLOCK_ROWS / Ops::FRAGMENT_ROWS;+ UZU_CONST uint COL_FRAGMENTS = BITS == 4 ? 1 : 2;+ UZU_CONST uint LANE_WORDS = ROW_FRAGMENTS * Ops::THREAD_ELEMENT_ROWS * COL_FRAGMENTS;+ UZU_CONST uint LANE_BYTES = LANE_WORDS * sizeof(uint);+ UZU_CONST uint BLOCK_CHUNK_BYTES = METAL_SIMD_SIZE * LANE_BYTES;+};++} // namespace gemm+} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.hindex 2c4106a84..24d36f84d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h@@ -47,11 +47,16 @@ struct RightOperand {UZU_CONST GemmBPrologueKind SCHEME = PROLOGUE;UZU_CONST bool NEEDS_CORRECTION = PROLOGUE == GemmBPrologueKind::ScaleBiasDequant;- UZU_CONST ushort CODE_ORIGIN = (!QUANTIZED || (BITS_ == 4 && PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant))- ? 0- : ushort(1u << (BITS - 1));+ UZU_CONST ushort CODE_ORIGIN =+ (!QUANTIZED || ((BITS_ == 2 || BITS_ == 4) && PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant))+ ? 0+ : ushort(1u << (BITS - 1));+ UZU_CONST ushort MMA_BLOCK_K = BITS_ == 2 ? 2 * MXU_SIMDGROUP_BLOCK_K : MXU_SIMDGROUP_BLOCK_K;- static_assert(!QUANTIZED || BITS_ == 4 || BITS_ == 8, "quantized integer weights must use 4 or 8 bits");+ static_assert(+ !QUANTIZED || BITS_ == 2 || BITS_ == 4 || BITS_ == 8,+ "quantized integer weights must use 2, 4 or 8 bits"+ );static_assert(!QUANTIZED || PROLOGUE != GemmBPrologueKind::FullPrecision, "quantized weights need a scheme");using CodeElement = int8_t;@@ -61,7 +66,7 @@ struct RightOperand {template <ushort BLOCK_K>static constexpr ushort outer_block_k() {if constexpr (QUANTIZED) {- static_assert(GROUP_SIZE_ % MXU_SIMDGROUP_BLOCK_K == 0, "weight groups must contain complete MMA chunks");+ static_assert(GROUP_SIZE_ % MMA_BLOCK_K == 0, "weight groups must contain complete MMA chunks");static_assert(BLOCK_K % GROUP_SIZE_ == 0, "tile block K must contain complete weight groups");return GROUP_SIZE_;} else {@@ -92,6 +97,7 @@ struct RightStorage {const device typename Right::ScaleElement* biases;const device uint8_t* zero_points;bool signed_codes;+ bool hoist_row_metadata;GemmBLayout gemm_b_layout;METAL_FUNC uint sign_mask() const thread {@@ -170,10 +176,11 @@ METAL_FUNC RightStorage<Right> pack_right(const device Element* biases,const device uint8_t* zero_points,const bool signed_codes,+ const bool hoist_row_metadata,const GemmBLayout gemm_b_layout) {if constexpr (!Right::QUANTIZED) {- return {dense, nullptr, nullptr, nullptr, nullptr, false, GemmBLayout::RowMajor};+ return {dense, nullptr, nullptr, nullptr, nullptr, false, false, GemmBLayout::RowMajor};} else {return {nullptr,@@ -182,6 +189,7 @@ METAL_FUNC RightStorage<Right> pack_right(Right::SCHEME == GemmBPrologueKind::ScaleBiasDequant ? biases : nullptr,Right::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant ? zero_points : nullptr,signed_codes,+ hoist_row_metadata,gemm_b_layout};}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.hindex c9c163c11..937ee0302 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h@@ -7,6 +7,7 @@#include "../../../common/fragment.h"#include "../../../common/quant_pack.h"#include "../../../common/quant_unpack.h"+#include "../mxu_tiled_layout.h"#include "../schedules/tile_context.h"using namespace metal;@@ -30,6 +31,11 @@ static METAL_FUNC void for_each_fragment_row(Visitor visitor) {}}}++template <bool ALIGNED>+static METAL_FUNC short clamp_to_live_row(const short row_offset, const short row_limit) {+ return ALIGNED ? row_offset : min(row_offset, short(row_limit - 1));+}} // namespacetemplate <typename Fragment, bool ALIGNED, bool CODES_GROUPED_BY_NIBBLE>@@ -42,11 +48,10 @@ load_int8_tile(const device int8_t* src, const int row_stride, const short simdgconst device int8_t* base = src + int(position.y) * row_stride + int(get_pack_factor<W4_BITS>()) * int(position.x);const short row_limit = simdgroup_limit - position.y;for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short row_offset) {- vec<uint, Fragment::COL_FRAGMENTS> packed_chunk(0u);- if (ALIGNED || row_offset < row_limit) {- packed_chunk =- *reinterpret_cast<const device vec<uint, Fragment::COL_FRAGMENTS>*>(base + int(row_offset) * row_stride);- }+ const vec<uint, Fragment::COL_FRAGMENTS> packed_chunk =+ *reinterpret_cast<const device vec<uint, Fragment::COL_FRAGMENTS>*>(+ base + int(clamp_to_live_row<ALIGNED>(row_offset, row_limit)) * row_stride+ );METAL_PRAGMA_UNROLLfor (ushort fragment_column = 0; fragment_column < Fragment::COL_FRAGMENTS; ++fragment_column) {reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, fragment_column))[row_slot] =@@ -65,12 +70,10 @@ load_int8_tile(const device int8_t* src, const int row_stride, const short simdgfor_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short row_offset) {METAL_PRAGMA_UNROLLfor (ushort fragment_column = 0; fragment_column < Fragment::COL_FRAGMENTS; ++fragment_column) {- char4 codes(0);- if (ALIGNED || row_offset < row_limit) {- codes = *reinterpret_cast<const device packed_char4*>(- base + int(row_offset) * row_stride + int(fragment_column) * Ops::FRAGMENT_COLS- );- }+ const char4 codes = *reinterpret_cast<const device packed_char4*>(+ base + int(clamp_to_live_row<ALIGNED>(row_offset, row_limit)) * row_stride ++ int(fragment_column) * Ops::FRAGMENT_COLS+ );reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, fragment_column))[row_slot] =as_type<uint>(codes);}@@ -93,11 +96,12 @@ struct Int8Cursor {GemmBLayout gemm_b_layout;uint chunk_stride;- METAL_FUNC Fragment load(const uint) const thread {+ METAL_FUNC Fragment load(const uint chunk) const thread {+ const device int8_t* chunk_address = address + chunk * chunk_stride;if (gemm_b_layout == GemmBLayout::MxuTiled) {Fragment tile;- const uint4 low = *reinterpret_cast<const device uint4*>(address);- const uint4 high = *reinterpret_cast<const device uint4*>(address + sizeof(uint4));+ const uint4 low = *reinterpret_cast<const device uint4*>(chunk_address);+ const uint4 high = *reinterpret_cast<const device uint4*>(chunk_address + sizeof(uint4));const uint words[8] = {low.x, low.y, low.z, low.w, high.x, high.y, high.z, high.w};for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short) {METAL_PRAGMA_UNROLL@@ -109,25 +113,27 @@ struct Int8Cursor {return tile;}return load_int8_tile<Fragment, ALIGNED, CODES_GROUPED_BY_NIBBLE>(- address,+ chunk_address,row_stride,simdgroup_limit,simd_lane_id);}- METAL_FUNC void advance() thread { address += chunk_stride; }+ METAL_FUNC void advance(const uint chunks) thread { address += chunks * chunk_stride; }METAL_FUNC void begin_k_group(const uint) thread {}};template <typename Fragment, typename Operand, bool ALIGNED>-struct W4Cursor {+struct PackedCursor {using Ops = typename Fragment::FragmentOpsType;+ UZU_CONST ushort BITS = Operand::BITS;+ UZU_CONST uint CODE_MASK = ((1u << BITS) - 1) * 0x01010101u;UZU_CONST short BLOCK_K = short(Fragment::COL_FRAGMENTS * Ops::FRAGMENT_ROWS);UZU_CONST bool HAS_ZERO_POINTS = Operand::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant;- UZU_CONST uint CHUNK_BYTES = uint(BLOCK_K) * uint(get_bytes_per_pack<W4_BITS>()) / uint(get_pack_factor<W4_BITS>());+ UZU_CONST uint CHUNK_BYTES = uint(BLOCK_K) * uint(get_bytes_per_pack<BITS>()) / uint(get_pack_factor<BITS>());struct PackedChunk {UZU_CONST ushort WORD_COUNT = Fragment::ROW_FRAGMENTS * Ops::THREAD_ELEMENT_ROWS;@@ -150,6 +156,19 @@ struct W4Cursor {METAL_FUNC PackedChunk fetch(const uint chunk_index) const thread {if (gemm_b_layout == GemmBLayout::MxuTiled) {+ if constexpr (BITS == 2) {+ const device uint8_t* lane_vector = current + (chunk_index & ~1u) * chunk_stride;+ const uint block_stride = MxuTiledLayout<BITS>::BLOCK_ROWS * uint(row_stride_bytes);+ PackedChunk packed_chunk;+ METAL_PRAGMA_UNROLL+ for (ushort fragment_row = 0; fragment_row < Fragment::ROW_FRAGMENTS; ++fragment_row) {+ const uint4 vector = *reinterpret_cast<const device uint4*>(lane_vector + fragment_row * block_stride);+ const uint2 words = chunk_index % 2 ? vector.zw : vector.xy;+ packed_chunk.words[PackedChunk::word_index(fragment_row, 0)] = words.x;+ packed_chunk.words[PackedChunk::word_index(fragment_row, 1)] = words.y;+ }+ return packed_chunk;+ }const uint4 words = *reinterpret_cast<const device uint4*>(current + chunk_index * chunk_stride);return {{words.x, words.y, words.z, words.w}};}@@ -174,6 +193,14 @@ struct W4Cursor {for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short row_offset) {const uint word =packed_chunk.words[PackedChunk::word_index(fragment_row, row_slot)] ^ (signed_codes ? W4_SIGN_MASK : 0u);+ if constexpr (BITS == 2) {+ METAL_PRAGMA_UNROLL+ for (ushort fragment_column = 0; fragment_column < Fragment::COL_FRAGMENTS; ++fragment_column) {+ reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, fragment_column))[row_slot] =+ (word >> (BITS * fragment_column)) & CODE_MASK;+ }+ return;+ }uint low = word & W4_NIBBLE_MASK;uint high = (word >> W4_BITS) & W4_NIBBLE_MASK;if constexpr (HAS_ZERO_POINTS) {@@ -197,9 +224,9 @@ struct W4Cursor {return tile;}- METAL_FUNC Fragment load(const uint) const thread { return decode(fetch(0)); }+ METAL_FUNC Fragment load(const uint chunk) const thread { return decode(fetch(chunk)); }- METAL_FUNC void advance() thread { current += chunk_stride; }+ METAL_FUNC void advance(const uint chunks) thread { current += chunks * chunk_stride; }METAL_FUNC void begin_k_group(const uint k_offset) thread {if constexpr (HAS_ZERO_POINTS) {@@ -243,21 +270,18 @@ static METAL_FUNC auto make_right_cursor(const device uint8_t* current =source.codes + size_t(tile.block_col + tile.tile_col_offset) * size_t(row_stride_bytes) +size_t(tile.k_offset) * size_t(get_bytes_per_pack<Operand::BITS>()) / size_t(get_pack_factor<Operand::BITS>());- static_assert(Core::SIMDGROUP_BLOCK_N == 2 * Ops::FRAGMENT_ROWS, "the MxuTiled layout holds 32-row blocks");- const uint mxu_tiled_lane_bytes = Operand::BITS == 4 ? 16 : 32;- const uint mxu_tiled_chunk_bytes = METAL_SIMD_SIZE * mxu_tiled_lane_bytes;+ using Layout = MxuTiledLayout<Operand::BITS>;+ static_assert(Core::SIMDGROUP_BLOCK_N % Layout::BLOCK_ROWS == 0, "a simdgroup tile covers whole MxuTiled blocks");if (source.gemm_b_layout == GemmBLayout::MxuTiled) {- const uint chunk_count = uint(params->K) / uint(Core::SIMDGROUP_BLOCK_K);- const uint row_block = uint(tile.block_col + tile.tile_col_offset) / uint(Core::SIMDGROUP_BLOCK_N);- current =- source.codes +- (size_t(row_block) * chunk_count + tile.k_offset / uint(Core::SIMDGROUP_BLOCK_K)) * mxu_tiled_chunk_bytes +- thread_context.simd_lane_id * mxu_tiled_lane_bytes;+ constexpr uint LANE_VECTOR_K = Layout::LANE_VECTOR_CHUNKS * Operand::MMA_BLOCK_K;+ current = source.codes + size_t(tile.block_col + tile.tile_col_offset) * size_t(row_stride_bytes) ++ size_t(tile.k_offset / LANE_VECTOR_K) * Layout::BLOCK_CHUNK_BYTES ++ thread_context.simd_lane_id * Layout::LANE_BYTES;}- if constexpr (Operand::BITS == 4) {- static_assert(Core::TILES_K == get_pack_factor<Operand::BITS>(), "W4 requires two K fragments");- using Cursor = W4Cursor<Fragment, Operand, ALIGNED>;+ if constexpr (Operand::BITS == 2 || Operand::BITS == 4) {+ static_assert(Core::TILES_K == get_pack_factor<Operand::BITS>(), "packed codes need one K fragment per code");+ using Cursor = PackedCursor<Fragment, Operand, ALIGNED>;const device uint8_t* zero_points = nullptr;const uint zero_point_group_stride_bytes = params->zero_point_group_stride / get_pack_factor<W4_BITS>();if constexpr (Cursor::HAS_ZERO_POINTS) {@@ -273,10 +297,11 @@ static METAL_FUNC auto make_right_cursor(zero_points,zero_point_group_stride_bytes,source.gemm_b_layout,- source.gemm_b_layout == GemmBLayout::MxuTiled ? mxu_tiled_chunk_bytes : Cursor::CHUNK_BYTES+ source.gemm_b_layout == GemmBLayout::MxuTiled ? Layout::BLOCK_CHUNK_BYTES / Layout::LANE_VECTOR_CHUNKS+ : Cursor::CHUNK_BYTES};} else {- static_assert(Operand::BITS == 8, "integer tile cursors support 4-bit and 8-bit codes");+ static_assert(Operand::BITS == 8, "integer tile cursors support 2-bit, 4-bit and 8-bit codes");const device int8_t* origin_int8 = reinterpret_cast<const device int8_t*>(current);return Int8Cursor<Fragment, ALIGNED, false>{origin_int8,@@ -284,7 +309,7 @@ static METAL_FUNC auto make_right_cursor(tile.simdgroup_limit_n,ushort(thread_context.simd_lane_id),source.gemm_b_layout,- source.gemm_b_layout == GemmBLayout::MxuTiled ? mxu_tiled_chunk_bytes+ source.gemm_b_layout == GemmBLayout::MxuTiled ? Layout::BLOCK_CHUNK_BYTES: uint(Int8Cursor<Fragment, ALIGNED, false>::BLOCK_K)};}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.hindex acd2b8e62..0446b8cf0 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h@@ -55,6 +55,7 @@ struct MetadataContext {uint left_group_count;uint right_group_count;uint right_scale_group_stride;+ uint padded_output_count;};template <typename LeftOperand, typename RightOperand>@@ -143,7 +144,7 @@ struct IntegerSchedule {uint right_scale_column_start = metadata_context.right_column_base + uint(right_column_offset);if constexpr (!ALIGNED_N) {right_scale_column_start =- clamp_group_output_column(right_scale_column_start, metadata_context.right_scale_group_stride);+ clamp_group_output_column(right_scale_column_start, metadata_context.padded_output_count);}scales[tile_n] = *reinterpret_cast<const device ScaleVector*>(right.scales + right_group_index * metadata_context.right_scale_group_stride + right_scale_column_start@@ -175,9 +176,9 @@ struct IntegerSchedule {}template <typename Core>- static constexpr bool prefetches_int4_chunks() {+ static constexpr bool prefetches_packed_chunks() {return (Core::TILES_M == 1 || (Core::TILES_M == 2 && Core::TILES_N == 2 && CENTER_RIGHT_ZERO_POINTS)) &&- chunks_per_k_group<Core>() > 1 && RightOperand::BITS == 4;+ chunks_per_k_group<Core>() > 1 && (RightOperand::BITS == 2 || RightOperand::BITS == 4);}template <typename Core, bool ALIGNED_M, bool ALIGNED_N>@@ -238,26 +239,24 @@ struct IntegerSchedule {) {GroupProducts<Core> group_product;group_product.clear();- if constexpr (prefetches_int4_chunks<Core>()) {+ if constexpr (prefetches_packed_chunks<Core>()) {for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {auto left_tile = left_cursor.load(uint(chunk));auto right_tile = right_cursor.decode(packed_right_chunks[chunk]);uzu::matmul::fragment_mma(group_product, left_tile, right_tile);- left_cursor.advance();- right_cursor.advance();});} else {decltype(right_cursor.load(0u)) right_tiles[CHUNKS_PER_K_GROUP];for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {right_tiles[chunk] = right_cursor.load(uint(chunk));- right_cursor.advance();});for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {auto left_tile = left_cursor.load(uint(chunk));uzu::matmul::fragment_mma(group_product, left_tile, right_tiles[chunk]);- left_cursor.advance();});}+ left_cursor.advance(CHUNKS_PER_K_GROUP);+ right_cursor.advance(CHUNKS_PER_K_GROUP);return group_product;}@@ -266,10 +265,19 @@ struct IntegerSchedule {thread typename Core::AccumFragment& accumulator,thread GroupProducts<Core>& group_product,const thread LeftMetadata<Core, ALIGNED_M>& left_metadata,- const thread RightMetadata<Core, ALIGNED_N>& right_metadata+ const thread RightMetadata<Core, ALIGNED_N>& right_metadata,+ thread float (&row_correction_sums)[LeftMetadata<Core, ALIGNED_M>::LEFT_VALUE_COUNT],+ const bool hoist_row_metadata) {using Ops = typename Core::FragmentOps;+ if constexpr (HAS_BIAS) {+ if (hoist_row_metadata) {+ for_each_static_index<int(LeftMetadata<Core, ALIGNED_M>::LEFT_VALUE_COUNT)>([&](const ushort index) {+ row_correction_sums[index] += left_metadata.group_corrections[index];+ });+ }+ }for_each_fragment(accumulator,group_product,@@ -278,6 +286,11 @@ struct IntegerSchedule {[&](const ushort element) {const ushort right_column = element % Ops::THREAD_ELEMENT_COLS;const ushort left_row = tile_m * Ops::THREAD_ELEMENT_ROWS + element / Ops::THREAD_ELEMENT_COLS;+ if (hoist_row_metadata) {+ accumulated[element] =+ fma(left_metadata.scales[left_row], float(group_product_value[element]), accumulated[element]);+ return;+ }const float right_scale = float(right_metadata.scales[tile_n][right_column]);accumulated[element] =fma(left_metadata.scales[left_row] * right_scale,@@ -295,6 +308,28 @@ struct IntegerSchedule {);}+ template <typename Core, bool ALIGNED_N>+ static METAL_FUNC UZU_ALWAYS_INLINE void apply_row_metadata(+ thread typename Core::AccumFragment& accumulator,+ const thread RightMetadata<Core, ALIGNED_N>& right_metadata,+ const thread float (&row_correction_sums)[Core::TILES_M * Core::FragmentOps::THREAD_ELEMENT_ROWS]+ ) {+ using Ops = typename Core::FragmentOps;+ for_each_static_index<int(Core::TILES_N)>([&](const ushort tile_n) {+ for_each_static_index<int(Core::TILES_M)>([&](const ushort tile_m) {+ thread auto& accumulated = accumulator.fragment_at(tile_m, tile_n);+ for_each_static_index<int(Ops::THREAD_ELEMENT_ROWS) * int(Ops::THREAD_ELEMENT_COLS)>([&](const ushort element) {+ const ushort right_column = element % Ops::THREAD_ELEMENT_COLS;+ const ushort left_row = tile_m * Ops::THREAD_ELEMENT_ROWS + element / Ops::THREAD_ELEMENT_COLS;+ accumulated[element] =+ fma(float(right_metadata.bias_offsets[tile_n][right_column]),+ row_correction_sums[left_row],+ float(right_metadata.scales[tile_n][right_column]) * accumulated[element]);+ });+ });+ });+ }+template <typename Core, bool ALIGNED_M, bool ALIGNED_N>static METAL_FUNC typename Core::AccumFragment launch(typename Core::LeftStorage left_storage,@@ -333,6 +368,8 @@ struct IntegerSchedule {uint(params->K) / uint(LeftOperand::GROUP_SIZE),uint(params->K) / RIGHT_GROUP_SIZE,params->scale_group_stride,+ (uint(params->N) + uzu::matmul::QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT - 1) &+ ~(uzu::matmul::QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT - 1),};const uint first_right_group = tile.k_offset / RIGHT_GROUP_SIZE;@@ -341,14 +378,21 @@ struct IntegerSchedule {typename Core::AccumFragment accumulator;accumulator.clear();+ const bool hoist_row_metadata = right_storage.hoist_row_metadata;+ float row_correction_sums[LeftMetadata<Core, ALIGNED_M>::LEFT_VALUE_COUNT] = {};+const int right_group_count = int(params->aligned_inner_iterations);constexpr int CHUNKS_PER_K_GROUP = chunks_per_k_group<Core>();- constexpr bool PREFETCH_INT4_CHUNKS = prefetches_int4_chunks<Core>();+ constexpr bool PREFETCH_PACKED_CHUNKS = prefetches_packed_chunks<Core>();++ if (hoist_row_metadata) {+ right_metadata.load(right_storage, metadata_context, first_right_group);+ }METAL_PRAGMA_NO_UNROLLfor (int right_group_index = 0; right_group_index < right_group_count; ++right_group_index) {typename decltype(right_cursor)::PackedChunk packed_right_chunks[CHUNKS_PER_K_GROUP];- if constexpr (PREFETCH_INT4_CHUNKS) {+ if constexpr (PREFETCH_PACKED_CHUNKS) {for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {packed_right_chunks[chunk] = right_cursor.fetch(uint(chunk));});@@ -358,9 +402,11 @@ struct IntegerSchedule {right_cursor.begin_k_group(right_group_offset);auto load_metadata = [&]() {left_metadata.load(left_storage, metadata_context, tile.k_offset + right_group_offset);- right_metadata.load(right_storage, metadata_context, first_right_group + uint(right_group_index));+ if (!hoist_row_metadata) {+ right_metadata.load(right_storage, metadata_context, first_right_group + uint(right_group_index));+ }};- if constexpr (!PREFETCH_INT4_CHUNKS) {+ if constexpr (!PREFETCH_PACKED_CHUNKS) {load_metadata();}GroupProducts<Core> group_product;@@ -379,12 +425,24 @@ struct IntegerSchedule {} else {group_product = multiply_k_group<Core>(left_cursor, right_cursor, packed_right_chunks);}- if constexpr (PREFETCH_INT4_CHUNKS) {+ if constexpr (PREFETCH_PACKED_CHUNKS) {load_metadata();}- accumulate_group<Core, ALIGNED_M, ALIGNED_N>(accumulator, group_product, left_metadata, right_metadata);+ accumulate_group<Core, ALIGNED_M, ALIGNED_N>(+ accumulator,+ group_product,+ left_metadata,+ right_metadata,+ row_correction_sums,+ hoist_row_metadata+ );}+ if constexpr (HAS_BIAS) {+ if (hoist_row_metadata) {+ apply_row_metadata<Core, ALIGNED_N>(accumulator, right_metadata, row_correction_sums);+ }+ }return accumulator;}};diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hindex 696ad443e..ae4860bbe 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -174,6 +174,8 @@ struct SimdgroupMmaCore {if (needs_rht) {threadgroup_barrier(mem_flags::mem_device);apply_output_random_hadamard_transform(+ d,+ params->leading_dimension_d,d,rht_factors_block,bias_block,@@ -202,9 +204,10 @@ struct SimdgroupMmaCore {) {const uint partition = thread_context.threadgroup_position.z;const uint tile_row = thread_context.threadgroup_position.y;- const uint2 tile = tile_id(uint2(thread_context.threadgroup_position.x, tile_row), params);+ constexpr bool MAY_USE_MORTON = !Right::QUANTIZED;+ const uint2 tile = tile_id<MAY_USE_MORTON>(uint2(thread_context.threadgroup_position.x, tile_row), params);const auto geometry = ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::compute(tile, params);- if (geometry.out_of_bounds) {+ if (MAY_USE_MORTON && geometry.out_of_bounds) {return;}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/small_batch_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/small_batch_mma_core.hindex eeb6f272b..0aa84f073 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/small_batch_mma_core.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/small_batch_mma_core.h@@ -164,6 +164,8 @@ struct SmallBatchMmaCore {threadgroup_barrier(mem_flags::mem_device);apply_output_random_hadamard_transform(+ tile_output,+ params->leading_dimension_d,tile_output,rht_factors + tile_col_start,tile_bias,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metalindex fca6a18f4..3ece65d3e 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -73,8 +73,8 @@ VARIANTS(GemmBPrologueKind::ScaleBiasDequant,GemmBPrologueKind::ScaleZeroPointDequant,GemmBPrologueKind::ScaleSymmetricDequant)-VARIANTS(BITS, 0, 4, 8)-VARIANTS(GROUP_SIZE, 0, 32, 64)+VARIANTS(BITS, 0, 2, 4, 8)+VARIANTS(GROUP_SIZE, 0, 32, 64, 128)VARIANTS(A_PROLOGUE,GemmAPrologueKind::FullPrecision,@@ -94,7 +94,8 @@ CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleZeroPointDequant || BITS == 4)CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant || BITS == 8)-CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant || BITS == 4)+CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant || BITS == 2 || BITS == 4)+CONSTRAINT((BITS == 2) == (GROUP_SIZE == 128) && (BITS != 2 || A_IS_INT8))CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision ||(TRANSPOSE_B && GEMM_TILING != GemmTiling::Tile64x32x32_Simdgroups2x2))@@ -151,6 +152,7 @@ KERNEL(Gemm)(const GemmAlignment alignment SPECIALIZE,const bool signed_codes SPECIALIZE,const GemmBLayout gemm_b_layout SPECIALIZE,+ const bool hoist_row_metadata SPECIALIZE,threadgroup AT a_shared[GEMM_TGA_ELEMENTS],threadgroup BT b_shared[GEMM_TGB_ELEMENTS],const uint group_x GROUPS(group_count_x),@@ -176,8 +178,15 @@ KERNEL(Gemm)("kernel bindings and operand correction policy must agree");const auto left_storage = operands::pack_left<LeftOperand, AT>(a, a_int8, a_scales, a_group_sums, a_packed_half);- const auto right_storage =- operands::pack_right<RightOperand, BT>(b, scales, biases, zero_points, signed_codes, gemm_b_layout);+ const auto right_storage = operands::pack_right<RightOperand, BT>(+ b,+ scales,+ biases,+ zero_points,+ signed_codes,+ hoist_row_metadata,+ gemm_b_layout+ );if constexpr (USE_MXU) {using Core = MxuMmaCore<DT, GEMM_TILING, TRANSPOSE_B, LeftOperand, RightOperand>;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metalindex 547f1d416..0402a38e2 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal@@ -1,49 +1,67 @@#include "../../common/dsl.h"#include "../../common/defines.h"-#include "../common/mxu_fragment/ops.h"+#include "common/mxu_tiled_layout.h"using namespace metal;-#define PREPARE_B_LAYOUT_THREADS 1024u+using uzu::matmul::MXU_TILED_PREPARE_THREADS;++template <uint BITS>+static METAL_FUNC uint codes_in_activation_order(const device uint* source, const uint column) {+ constexpr uint BYTE_BITS = 8;+ constexpr uint CODES_PER_BYTE = BYTE_BITS / BITS;+ const uint source_byte = column / CODES_PER_BYTE;+ uint packed = 0;+ METAL_PRAGMA_UNROLL+ for (uint source_word = 0; source_word < CODES_PER_BYTE; ++source_word) {+ const uint codes = source[source_word] >> (BYTE_BITS * source_byte);+ METAL_PRAGMA_UNROLL+ for (uint code = 0; code < CODES_PER_BYTE; ++code) {+ packed |= ((codes >> (BITS * code)) & ((1u << BITS) - 1)) << (BYTE_BITS * code + BITS * source_word);+ }+ }+ return packed;+}template <uint BITS, uint WORDS_PER_THREAD>-VARIANTS(BITS, 4, 8)+VARIANTS(BITS, 2, 4, 8)VARIANTS(WORDS_PER_THREAD, 32, 72, 160)KERNEL(GemmPrepareBLayout)(device uint* weights,const constant uint& row_words,const constant uint& row_block_count,const uint row_block GROUPS(row_block_count),- const uint thread_index THREADS(PREPARE_B_LAYOUT_THREADS)+ const uint thread_index THREADS(MXU_TILED_PREPARE_THREADS)) {- using Ops = uzu::matmul::MxuFragmentOps<>;- constexpr uint CHUNK_BYTES = BITS == 4 ? 16 : 32;- constexpr uint COL_FRAGMENTS = BITS == 4 ? 1 : 2;- constexpr uint LANE_WORDS = 2 * Ops::THREAD_ELEMENT_ROWS * COL_FRAGMENTS;- const uint block_words = 2 * Ops::FRAGMENT_ROWS * row_words;+ using Layout = uzu::gemm::MxuTiledLayout<BITS>;+ using Ops = typename Layout::Ops;+ constexpr uint ROW_SLOT_STRIDE = BITS == 2 ? 1 : Layout::COL_FRAGMENTS;+ constexpr uint COLUMN_STRIDE = BITS == 2 ? Ops::THREAD_ELEMENT_ROWS : 1;+ const uint block_words = Layout::BLOCK_ROWS * row_words;device uint* block = weights + size_t(row_block) * block_words;uint words[WORDS_PER_THREAD];METAL_PRAGMA_UNROLLfor (uint slot = 0; slot < WORDS_PER_THREAD; ++slot) {- const uint index = slot * PREPARE_B_LAYOUT_THREADS + thread_index;+ const uint index = slot * MXU_TILED_PREPARE_THREADS + thread_index;if (index < block_words) {- const uint lane = (index / LANE_WORDS) % METAL_SIMD_SIZE;- const uint word = index % LANE_WORDS;- const uint fragment_row = word / (Ops::THREAD_ELEMENT_ROWS * COL_FRAGMENTS);- const uint row_slot = word / COL_FRAGMENTS % Ops::THREAD_ELEMENT_ROWS;- const uint fragment_column = word % COL_FRAGMENTS;- const short2 position = Ops::get_position(ushort(lane));+ const uint chunk = index / (METAL_SIMD_SIZE * Layout::LANE_WORDS);+ const uint word = index % Layout::LANE_WORDS;+ const short2 position = Ops::get_position(ushort(index / Layout::LANE_WORDS % METAL_SIMD_SIZE));+ const uint fragment_row = word / (Ops::THREAD_ELEMENT_ROWS * Layout::COL_FRAGMENTS);+ const uint row_slot = word / ROW_SLOT_STRIDE % Ops::THREAD_ELEMENT_ROWS;+ const uint fragment_column = word / COLUMN_STRIDE % Layout::COL_FRAGMENTS;const uint row = uint(position.y) + fragment_row * Ops::FRAGMENT_ROWS + row_slot * Ops::THREAD_ELEMENT_ROW_STRIDE;- const uint byte = index / (METAL_SIMD_SIZE * LANE_WORDS) * CHUNK_BYTES + uint(position.x) +- fragment_column * Ops::FRAGMENT_COLS;- words[slot] = block[row * row_words + byte / 4];+ const device uint* source = block + row * row_words ++ (chunk * Layout::COL_FRAGMENTS + fragment_column) * Ops::FRAGMENT_COLS / sizeof(uint);+ words[slot] = BITS == 2 ? codes_in_activation_order<BITS>(source, uint(position.x))+ : source[uint(position.x) / sizeof(uint)];}}threadgroup_barrier(mem_flags::mem_device);METAL_PRAGMA_UNROLLfor (uint slot = 0; slot < WORDS_PER_THREAD; ++slot) {- const uint index = slot * PREPARE_B_LAYOUT_THREADS + thread_index;+ const uint index = slot * MXU_TILED_PREPARE_THREADS + thread_index;if (index < block_words) {block[index] = words[slot];}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 7736a5f23..94c1f89e4 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -7,7 +7,10 @@ use crate::{common::{Backend, BufferMut, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,Context,- gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},+ gpu_types::{+ gemm::{GemmBPrologueKind, GemmDTransform},+ matmul::{MXU_TILED_BLOCK_ROWS, MXU_TILED_PREPARE_THREADS, MXU_TILED_W2_BLOCK_ROWS},+ },kernel::matmul::{Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulOutput, MatmulShape, QuantParamsLayout,QuantParamsStrides, QuantizedB,@@ -95,17 +98,27 @@ impl GemmKernel {specialization.alignment,specialization.signed_codes,specialization.gemm_b_layout,+ specialization.hoist_row_metadata,)?;Ok(entry.insert(kernel))},}}+ fn mxu_tiled_block_rows(bits: u32) -> u32 {+ if bits == 2 {+ MXU_TILED_W2_BLOCK_ROWS+ } else {+ MXU_TILED_BLOCK_ROWS+ }+ }+pub fn mxu_tiled_words_per_thread(k: u32,bits: u32,) -> Option<u32> {- let words_per_thread = (k * bits / 32).div_ceil(32);+ let words_per_thread =+ (Self::mxu_tiled_block_rows(bits) * (k * bits / u32::BITS)).div_ceil(MXU_TILED_PREPARE_THREADS);[32, 72, 160].into_iter().find(|&variant| variant >= words_per_thread)}@@ -116,13 +129,13 @@ impl GemmKernel {bits: u32,context: &MetalContext,) -> Result<(), MetalError> {- let row_words = k * bits / 32;+ let row_words = k * bits / u32::BITS;let variant = Self::mxu_tiled_words_per_thread(k, bits).expect("int8 eligibility admits only tileable weights");let mut command_buffer = context.create_command_buffer(Some("prepare GEMM B layout"), None)?;GemmPrepareBLayoutMetalKernel::new(context, bits, variant)?.encode(weights,row_words,- rows / 32,+ rows / Self::mxu_tiled_block_rows(bits),&mut command_buffer,);block_on(command_buffer.end_encoding().submit().wait_until_completed())?;@@ -259,6 +272,13 @@ impl GemmKernel {});}+ if shape.b_bits == Some(2) && shape.a_is_quantized() && gemm_b_layout != GemmBLayout::MxuTiled {+ return Err(MatmulMetalError::IncompatibleA {+ path: "Gemm",+ reason: "2-bit int8 GEMM requires the MxuTiled weight layout",+ });+ }+let is_quant = quantized.is_some();if !is_quant && !matches!(&a, MatmulA::FullPrecision { .. }) {return Err(MatmulMetalError::IncompatibleA {@@ -266,6 +286,7 @@ impl GemmKernel {reason: "int8 activations require quantized weights",});}+ let row_scaled_weights = quantized.as_ref().is_some_and(|quantized| quantized.params.is_per_row());let (scales, biases, zero_points, scale_strides, zero_point_strides) = match quantized.as_ref() {None => (None, None, None, Default::default(), Default::default()),Some(quantized) => (@@ -336,6 +357,7 @@ impl GemmKernel {ab_scale,scale_strides,zero_point_strides,+ row_scaled_weights,shape,gemm_b_layout,plan,@@ -392,6 +414,7 @@ impl GemmKernel {alignment,a_prologue,a_group_size,+ row_scaled_weights,)?;let kernel = self.get_or_create(command_buffer.context(), specialization, self.output_data_type)?;kernel.encode(@@ -429,6 +452,7 @@ impl GemmKernel {ab_scale: f32,scale_strides: QuantParamsStrides,zero_point_strides: QuantParamsStrides,+ row_scaled_weights: bool,shape: MatmulShape,gemm_b_layout: GemmBLayout,plan: GemmPlan,@@ -487,6 +511,7 @@ impl GemmKernel {alignment,a_prologue,a_group_size,+ row_scaled_weights,)?;let elem = (m as usize) * (n as usize);diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rsindex b0bd39bbe..6ba1af16d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs@@ -23,8 +23,11 @@ const MXU_N_BUCKET_MAXES: [u32; 2] = [63, 127];const SIMDGROUP_QUANT_NARROW_TILE_M_MAX: u32 = 16;const SIMDGROUP_QUANT_WIDE_BLOCK_M: u32 = 64;+const W2_TILE16_MAX_M: u32 = 64;+const SPLIT_K_TARGET_TILES_FP: u32 = 512;pub(super) const SPLIT_K_TARGET_TILES_A8: u32 = 256;+const SPLIT_K_TARGET_TILES_A8_W2: u32 = 128;pub(super) const SPLIT_K_TARGET_TILES_TRELLIS_SIMDGROUP: u32 = 384;pub(super) const SPLIT_K_TARGET_TILES_TRELLIS_TILE16: u32 = 768;pub(super) const SPLIT_K_TARGET_TILES_A8_TILE16X32: u32 = 4 * SPLIT_K_TARGET_TILES_A8;@@ -58,10 +61,13 @@ pub(super) fn mxu_mn_tile(pub(super) fn mxu_int8_gemm_tile(m: u32,n: u32,+ weight_bits: Option<u32>,) -> GemmTiling {match mxu_mn_tile(true, m, n) {GemmTiling::Tile128x128x256_Simdgroups4x4 => MXU_DEFAULT_TILE,GemmTiling::Tile16x32x256_Simdgroups1x1 => GemmTiling::Tile16x32x256_Simdgroups1x1x4,+ tiling @ GemmTiling::Tile64x32x256_Simdgroups4x1 => tiling,+ _ if weight_bits == Some(2) && m <= W2_TILE16_MAX_M => GemmTiling::Tile16x32x256_Simdgroups1x1x4,tiling => tiling,}}@@ -118,6 +124,7 @@ pub(super) fn split_k_target_tiles((true, GemmTiling::Tile32x64x256_Simdgroups2x2, Some(4)) => SPLIT_K_TARGET_TILES_A8_TILE32_W4,(true, GemmTiling::Tile32x64x256_Simdgroups2x2, _) => SPLIT_K_TARGET_TILES_A8_TILE32_W8,(true, GemmTiling::Tile16x32x256_Simdgroups1x1, _) => SPLIT_K_TARGET_TILES_A8_TILE16X32,+ (true, GemmTiling::Tile16x32x256_Simdgroups1x1x4, Some(2)) => SPLIT_K_TARGET_TILES_A8_W2,(true, GemmTiling::Tile16x32x256_Simdgroups1x1x4, Some(4)) => SPLIT_K_TARGET_TILES_A8,(true, GemmTiling::Tile16x32x256_Simdgroups1x1x4, _) => SPLIT_K_TARGET_TILES_A8_TILE16X32_K_SPLIT,(true, _, _) => SPLIT_K_TARGET_TILES_A8,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rsindex 157747cad..34370b0a4 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs@@ -271,7 +271,7 @@ fn select_tiling(match engine {GemmEngine::Simdgroup if shape.is_quant() => policy::simdgroup_quant_tile(shape.m),GemmEngine::Simdgroup => GemmTiling::Tile64x32x32_Simdgroups2x2,- GemmEngine::Mxu if shape.a_is_quantized() => match policy::mxu_int8_gemm_tile(shape.m, shape.n) {+ GemmEngine::Mxu if shape.a_is_quantized() => match policy::mxu_int8_gemm_tile(shape.m, shape.n, shape.b_bits) {tilingif tiling.simdgroups_k() > 1&& !shape.k.is_multiple_of(tiling.simdgroups_k() * ACTIVATION_SCALE_GROUP_SIZE) =>diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rsindex 6201cb429..5a69d25c8 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -24,6 +24,7 @@ pub(super) struct GemmSpecialization {pub(super) b_group_size: Option<u32>,pub(super) signed_codes: bool,pub(super) gemm_b_layout: GemmBLayout,+ pub(super) hoist_row_metadata: bool,pub(super) a_group_size: Option<u32>,}@@ -37,6 +38,7 @@ impl GemmSpecialization {alignment: GemmAlignment,a_prologue: GemmAPrologueKind,a_group_size: Option<u32>,+ row_scaled_weights: bool,) -> Result<Self, GemmSpecializationError> {let specialization = Self {weights_data_type,@@ -51,6 +53,9 @@ impl GemmSpecialization {b_group_size: shape.b_group_size,signed_codes: shape.signed_codes,gemm_b_layout,+ hoist_row_metadata: row_scaled_weights+ && a_prologue == GemmAPrologueKind::Int8Symmetric+ && shape.b_prologue == GemmBPrologueKind::ScaleBiasDequant,a_group_size,};specialization.validate()?;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.hindex dc05c6fcf..47350e421 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h@@ -23,9 +23,11 @@ struct QuantChunk {float origin,bool signed_codes) {- const uint word = chunk * (BITS == 4 ? 1u : 2u);- const uint word0 = words[word / 4u][word % 4u];- const uint word1 = BITS == 4 ? 0u : words[(word + 1u) / 4u][(word + 1u) % 4u];+ constexpr uint CHUNK_BITS = VALUES * BITS;+ const uint first_bit = chunk * CHUNK_BITS;+ const uint word = first_bit / WORD_BITS;+ const uint word0 = words[word / WORD_LANES][word % WORD_LANES] >> (first_bit % WORD_BITS);+ const uint word1 = CHUNK_BITS > WORD_BITS ? words[(word + 1u) / WORD_LANES][(word + 1u) % WORD_LANES] : 0u;decode_words(word0, word1, values, origin, signed_codes);}@@ -64,7 +66,7 @@ private:bool signed_codes) {constexpr uint CODES_PER_WORD = WORD_BITS / BITS;- if constexpr (BITS == 4) {+ if constexpr (BITS == 2 || BITS == 4) {word0 ^= signed_codes ? W4_SIGN_MASK : 0u;METAL_PRAGMA_UNROLLfor (uint i = 0; i < VALUES; i++) {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.hindex c6f6a962c..a33e91ead 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h@@ -106,9 +106,11 @@ public:) thread {const uint k = position.group * GROUP_SIZE + group_offset + position.slice * SLICE_VALUES;const uint offset_bytes = k * BITS / QuantChunk<BITS>::BITS_PER_BYTE;- if (params.gemm_b_layout == GemmBLayout::MxuTiled) {- load_mxu_tiled_weights(weights_base, weight_row_indices, row_stride, offset_bytes, params.gathered);- return;+ if constexpr (BITS == 4 || BITS == 8) {+ if (params.gemm_b_layout == GemmBLayout::MxuTiled) {+ load_mxu_tiled_weights(weights_base, weight_row_indices, row_stride, offset_bytes, params.gathered);+ return;+ }}Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {constexpr uint R = decltype(output_index)::value;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metalindex 041cf9a00..4293081e5 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -18,7 +18,8 @@ using namespace uzu::gemm;(OUTPUT_ROW_TILE == 8 / K_SPLIT || OUTPUT_ROW_TILE == 32 / K_SPLIT) && INPUT_ROW_TILE == 1)#define GEMV_QUANTIZED_ROW_TILE \(K_SPLIT == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE == 1 && \- ((BITS == 4 && GROUP_SIZE == 16 * GROUP_LANES) || (BITS == 8 && GROUP_SIZE == 8 * GROUP_LANES)))+ ((BITS == 2 && GROUP_SIZE == 32 * GROUP_LANES) || (BITS == 4 && GROUP_SIZE == 16 * GROUP_LANES) || \+ (BITS == 8 && GROUP_SIZE == 8 * GROUP_LANES)))#define GEMV_BATCHED_FMA_32_TILE \(K_SPLIT == 1 && REDUCTION_LANES == 8 && NUM_SIMDGROUPS == 4 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE >= 2 && \INPUT_ROW_TILE <= 4)@@ -50,8 +51,8 @@ VARIANTS(GemmBPrologueKind::ScaleBiasDequant,GemmBPrologueKind::ScaleZeroPointDequant,GemmBPrologueKind::ScaleSymmetricDequant)-VARIANTS(GROUP_SIZE, 0, 32, 64)-VARIANTS(BITS, 0, 4, 8)+VARIANTS(GROUP_SIZE, 0, 32, 64, 128)+VARIANTS(BITS, 0, 2, 4, 8)VARIANTS(K_SPLIT, 1, 2, 4, 8)VARIANTS(INPUT_ALIGNED, false, true)VARIANTS(INPUT_ROW_TILE, 1, 2, 3, 4)@@ -65,7 +66,9 @@ CONSTRAINT(((B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant ||B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) &&BITS == 4 && GROUP_SIZE != 0) ||- (B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant && BITS == 8 && GROUP_SIZE != 0))+ (B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant && BITS == 8 && GROUP_SIZE != 0) ||+ (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant && BITS == 2 && GEMV_QUANTIZED_ROW_TILE))+CONSTRAINT((BITS == 2) == (GROUP_SIZE == 128))CONSTRAINT(INPUT_ROW_TILE == 1 || GEMV_BATCHED_INPUT)CONSTRAINT(GEMV_FULL_PRECISION_TILE ||diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 7c2f56b33..ef08d5026 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -26,6 +26,7 @@ use crate::{};const GEMV_MAX_BATCH: u32 = 8;+const QUANT_K_BLOCK_BYTES: u32 = 256;const MXU_TILED_KERNEL_MIN_K: u32 = 2048;#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -103,15 +104,16 @@ impl GemvSpecialization {return None;}let bits = shape.b_bits.unwrap_or(0);+ if bits == 2 && gemm_b_layout == GemmBLayout::MxuTiled {+ return None;+ }if !is_quant && (shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH) {return None;}let block_size = if !is_quant {FP_K_BLOCK- } else if bits == 4 {- 512} else {- 256+ QUANT_K_BLOCK_BYTES * 8 / bits};let input_aligned = shape.k.is_multiple_of(block_size);// Gathered quantized rows cannot share one input tile, and batched tiles need whole K blocks.diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rsindex 035b8a042..3d113d869 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs@@ -8,11 +8,7 @@ const fn lane_tile(bits: u32,group: u32,) -> GemvTile {- let pack_factor = match bits {- 4 => 8,- 8 => 4,- _ => 1,- };+ let pack_factor = 32 / bits;GemvTile::quantized(num_simdgroups, results_per_simdgroup, 1, 32, group / (2 * pack_factor))}@@ -29,7 +25,7 @@ pub fn gathered_tile(m: u32,n: u32,) -> Option<GemvTile> {- if !matches!(bits, 4 | 8) || !matches!(group, 32 | 64) || !(1..=8).contains(&m) {+ if !matches!((bits, group), (4 | 8, 32 | 64) | (2, 128)) || !(1..=8).contains(&m) {return None;}let tile = lane_default(bits, group);@@ -44,7 +40,7 @@ pub fn select(bf16_io: bool,) -> Option<GemvTile> {// The GEMV pipelines exist for BF16 I/O only. Rows 5..=8 stay on GEMV for tiny N only.- if !bf16_io || (m > 4 && n > 64) {+ if !bf16_io || bits == 2 || (m > 4 && n > 64) {return None;}gathered_tile(bits, group, m, n)diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rsindex 408e9a328..ee33dbfa4 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs@@ -38,7 +38,7 @@ pub struct MatmulMetalKernel {}pub fn supports_integer_right_operand(shape: &MatmulShape) -> bool {- matches!((shape.b_bits, shape.signed_codes), (Some(4), _) | (Some(8), true))+ matches!((shape.b_bits, shape.signed_codes), (Some(2), false) | (Some(4), _) | (Some(8), true))}impl MatmulMetalKernel {diff --git a/crates/uzu-engine/src/backends/metal/kernel/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/mod.rsindex 82de4ac0e..9e0cf02c1 100644--- a/crates/uzu-engine/src/backends/metal/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/mod.rs@@ -1,5 +1,8 @@use crate::backends::{- common::{Kernels, gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},+ common::{+ Kernels,+ gpu_types::{hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE, matmul::MXU_TILED_PREPARE_THREADS},+ },metal::{Metal,buffer::MetalBufferExt,diff --git a/crates/uzu-engine/src/backends/slang/kernel/generated/matmul.slang b/crates/uzu-engine/src/backends/slang/kernel/generated/matmul.slangindex 4084cb707..014155f40 100644--- a/crates/uzu-engine/src/backends/slang/kernel/generated/matmul.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/generated/matmul.slang@@ -1 +1,7 @@-static const uint QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT = 4;\ No newline at end of file+static const uint QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT = 4;++static const uint MXU_TILED_BLOCK_ROWS = 32;++static const uint MXU_TILED_W2_BLOCK_ROWS = 16;++static const uint MXU_TILED_PREPARE_THREADS = 1024;\ No newline at end of filediff --git a/crates/uzu-engine/src/backends/slang/kernel/generated/quantization.slang b/crates/uzu-engine/src/backends/slang/kernel/generated/quantization.slangindex 165712637..8512af9ae 100644--- a/crates/uzu-engine/src/backends/slang/kernel/generated/quantization.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/generated/quantization.slang@@ -2,4 +2,5 @@ enum QuantizationMode : uint32_t {U4 = 0,I8 = 1,U8 = 2,+ U2 = 3,};\ No newline at end of filediff --git a/crates/uzu-engine/src/data_type.rs b/crates/uzu-engine/src/data_type.rsindex f58175327..b4efe91a5 100644--- a/crates/uzu-engine/src/data_type.rs+++ b/crates/uzu-engine/src/data_type.rs@@ -10,6 +10,7 @@ pub enum DataType {F16,F32,F64,+ U2,I4,U4,I8,@@ -25,6 +26,7 @@ pub enum DataType {impl DataType {pub const fn size_in_bits(&self) -> usize {match self {+ DataType::U2 => 2,DataType::I4 | DataType::U4 => 4,DataType::I8 | DataType::U8 => 8,DataType::I16 | DataType::U16 => 16,diff --git a/crates/uzu-engine/src/encodable_block/weight_matrix.rs b/crates/uzu-engine/src/encodable_block/weight_matrix.rsindex 43378b46c..3b22ef13c 100644--- a/crates/uzu-engine/src/encodable_block/weight_matrix.rs+++ b/crates/uzu-engine/src/encodable_block/weight_matrix.rs@@ -5,7 +5,10 @@ use crate::{backends::common::{Backend,gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},- kernel::matmul::{MatmulB, QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection, WeightsLayout},+ kernel::{+ activation_transform::ACTIVATION_SCALE_GROUP_SIZE,+ matmul::{MatmulB, QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection, WeightsLayout},+ },},config::weight_matrix::{AnyWeightMatrixSpec, Layout},data_type::DataType,@@ -56,6 +59,7 @@ pub fn parse_spec<B: Backend>(spec: &AnyWeightMatrixSpec) -> Result<ParsedWeightNone => None,Some((bits, group_size, method)) => {let mode = match bits {+ 2 => QuantizationMode::U2,4 => QuantizationMode::U4,8 => QuantizationMode::U8,_ => {@@ -114,7 +118,7 @@ impl<B: Backend> WeightMatrix<B> {}let (rows, columns) = physical_shape(&layout, output_dim, input_dim);- let Some(info) = quantization_info else {+ let Some(mut info) = quantization_info else {let values = tree.read_buffer("weights", &[rows, columns], data_type).await?;return Ok(Self {values,@@ -136,6 +140,9 @@ impl<B: Backend> WeightMatrix<B> {)));}let groups = columns.div_ceil(group_size);+ if info.mode == QuantizationMode::U2 {+ info.group_size = ACTIVATION_SCALE_GROUP_SIZE;+ }let values = tree.read_buffer("weights", &[rows, columns / packing_divisor], storage_data_type).await?;let params = QuantParams::new(params_layout, rows, groups);diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rsindex bb707a29f..9a9d38948 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs@@ -463,6 +463,9 @@ fn mxu_quant_parity_bf16(#[case::w4_m128_bias(128u32, 512u32, 256u32, 64u32, 4u32, QuantizationMethod::ScaleBias)]#[case::w4_m16_k17408(16u32, 17408u32, 64u32, 64u32, 4u32, QuantizationMethod::ScaleZeroPoint)]#[case::w8_m16_k17408(16u32, 17408u32, 64u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]+#[case::w2_m1(1u32, 4096u32, 1024u32, 128u32, 2u32, QuantizationMethod::ScaleBias)]+#[case::w2_m16(16u32, 4096u32, 1024u32, 128u32, 2u32, QuantizationMethod::ScaleBias)]+#[case::w2_m65(65u32, 4096u32, 160u32, 128u32, 2u32, QuantizationMethod::ScaleBias)]fn a8w_mxu_parity_bf16(#[case] m: u32,#[case] k: u32,@@ -473,14 +476,19 @@ fn a8w_mxu_parity_bf16(#[values(GemmBLayout::RowMajor, GemmBLayout::MxuTiled)] gemm_b_layout: GemmBLayout,) {let context = block_on(MetalContext::new()).expect("Metal context");- if !context.supports_mxu {+ if !context.supports_mxu || (bits == 2 && gemm_b_layout == GemmBLayout::RowMajor) {return;}- let (input, reference_input) = QuantInput::<bf16>::new(m, k, n, weight_gs, bits, method, 0)- .with_prepared_a_and_reference(- ACTIVATION_SCALE_GROUP_SIZE,- (method != QuantizationMethod::ScaleSymmetric).then_some(weight_gs),- );+ let input = QuantInput::<bf16>::new(m, k, n, weight_gs, bits, method, 0);+ let input = if bits == 2 {+ input.with_one_group_per_row()+ } else {+ input+ };+ let (input, reference_input) = input.with_prepared_a_and_reference(+ ACTIVATION_SCALE_GROUP_SIZE,+ (method != QuantizationMethod::ScaleSymmetric).then_some(weight_gs),+ );let actual = run_quant_metal(&context, &input.with_group_output(), gemm_b_layout, Some(GemmEngine::Mxu));let reference = run_quant_cpu::<bf16>(&reference_input);assert_parity::<bf16>(@@ -690,6 +698,7 @@ fn gemv_reads_mxu_tiled_layout_bf16(#[test_attr(uzu_test)]#[case::w4_zp_gs32(4u32, 32u32, ScaleZeroPoint)]#[case::w8_sym_gs64(8u32, 64u32, QuantizationMethod::ScaleSymmetric)]+#[case::w2_bias_gs128(2u32, 128u32, QuantizationMethod::ScaleBias)]fn gathered_gemv_reads_mxu_tiled_layout_bf16(#[case] bits: u32,#[case] group_size: u32,@@ -701,10 +710,10 @@ fn gathered_gemv_reads_mxu_tiled_layout_bf16(}let (k, vocab, gathered_rows) = (256u32, 256u32, 10u32);let input = QuantInput::<bf16>::new(1, k, vocab, group_size, bits, method, 0).with_group_output();- let input = if bits == 8 {- input.with_signed_weight_codes()- } else {- input+ let input = match bits {+ 2 => input.with_one_group_per_row(),+ 8 => input.with_signed_weight_codes(),+ _ => input,};let ids: Vec<u32> = (0..gathered_rows).map(|row| (row * 37 + 11) % vocab).collect();let dense = run_quant_cpu::<bf16>(&input);@@ -719,7 +728,9 @@ fn gathered_gemv_reads_mxu_tiled_layout_bf16(bf16::data_type(),).expect("Metal matmul kernel");- tile_weights(&matmul, &mut buffers, &input, &context);+ if bits != 2 {+ tile_weights(&matmul, &mut buffers, &input, &context);+ }let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");let mut arguments = quant_arguments(&mut buffers, &input);arguments.gather_indices = Some(&ids_buffer);diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rsindex 88984a91e..cf6d30a2d 100644--- a/crates/uzu-engine/unit/common/matmul/quant.rs+++ b/crates/uzu-engine/unit/common/matmul/quant.rs@@ -51,10 +51,12 @@ pub struct QuantInput<T: DataTypeRepr + Float> {pub params_layout: QuantParamsLayout,pub signed_codes: bool,pub prepared_a: Option<PreparedInt8A>,+ pub one_group_per_row: bool,}fn mode_for_bits(bits: u32) -> QuantizationMode {match bits {+ 2 => QuantizationMode::U2,4 => QuantizationMode::U4,8 => QuantizationMode::U8,_ => unreachable!("unsupported bits: {bits}"),@@ -182,6 +184,7 @@ impl<T: DataTypeRepr + Float> QuantInput<T> {params_layout: QuantParamsLayout::OutputGroup,signed_codes: false,prepared_a: None,+ one_group_per_row: false,}}@@ -190,6 +193,23 @@ impl<T: DataTypeRepr + Float> QuantInput<T> {self}+ pub fn with_one_group_per_row(mut self) -> Self {+ self.one_group_per_row = true;+ self.scales.truncate(self.n as usize);+ if let Some(biases) = &mut self.biases {+ biases.truncate(self.n as usize);+ }+ self+ }++ fn group_count(&self) -> u32 {+ if self.one_group_per_row {+ 1+ } else {+ self.k.div_ceil(self.group_size)+ }+ }+pub fn with_group_output(mut self) -> Self {self.params_layout = QuantParamsLayout::GroupOutput;self@@ -227,7 +247,7 @@ impl<T: DataTypeRepr + Float> QuantInput<T> {sum_group_size: Option<u32>,code_layout: Int8CodeLayout,) -> Self {- self.signed_codes = self.mode != QuantizationMode::U4;+ self.signed_codes = self.mode == QuantizationMode::U8;let rows = self.m;let columns = self.k;assert!(columns.is_multiple_of(activation_scale_group_size));@@ -313,7 +333,7 @@ impl<B: Backend, T: DataTypeRepr + Float> QuantBuffers<B, T> {context: &B::Context,input: &QuantInput<T>,) -> Self {- let groups = input.k.div_ceil(input.group_size);+ let groups = input.group_count();let params_layout = input.params_layout;let params = QuantParams::new(params_layout, input.n, groups);let metadata_elements = params.scale_shape().into_iter().product::<u32>() as usize;@@ -386,7 +406,7 @@ fn quant_b_variant<TB: BufferRef, T: DataTypeRepr + Float>(weights_layout: WeightsLayout<TB::Backend>,input: &QuantInput<T>,) -> MatmulB<TB> {- let params = QuantParams::new(params_layout, input.n, input.k.div_ceil(input.group_size));+ let params = QuantParams::new(params_layout, input.n, input.group_count());let correction = match input.quant_method {QuantizationMethod::ScaleBias => QuantizedCorrection::Biases(biases.expect("bias buffer")),QuantizationMethod::ScaleZeroPoint => QuantizedCorrection::ZeroPoints(zero_points.expect("zp buffer")),