Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Support 2 bit symmetric row scale (Dash, MXU) (#1054)

CCYeh69c64cbe2aa9
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/matmul.rs b/crates/uzu-engine/src/backends/common/gpu_types/matmul.rs
index ce4714f04..9d338c70b 100644
--- a/crates/uzu-engine/src/backends/common/gpu_types/matmul.rs
+++ b/crates/uzu-engine/src/backends/common/gpu_types/matmul.rs
@@ -1 +1,4 @@
pub const QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT: u32 = 4;
+pub const MXU_TILED_BLOCK_ROWS: u32 = 32;
+pub const MXU_TILED_W2_BLOCK_ROWS: u32 = 16;
+pub const MXU_TILED_PREPARE_THREADS: u32 = 1024;
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/quantization.rs b/crates/uzu-engine/src/backends/common/gpu_types/quantization.rs
index be0cce87c..d51b53b7c 100644
--- a/crates/uzu-engine/src/backends/common/gpu_types/quantization.rs
+++ b/crates/uzu-engine/src/backends/common/gpu_types/quantization.rs
@@ -13,6 +13,8 @@ pub enum QuantizationMode {
I8,
#[serde(rename = "uint8")]
U8,
+ #[serde(rename = "uint2")]
+ U2,
}
impl QuantizationMode {
@@ -24,7 +26,7 @@ impl QuantizationMode {
pub fn storage_type(&self) -> DataType {
match self {
- QuantizationMode::U4 => DataType::U8,
+ QuantizationMode::U2 | QuantizationMode::U4 => DataType::U8,
QuantizationMode::I8 => DataType::I8,
QuantizationMode::U8 => DataType::U8,
}
@@ -32,6 +34,7 @@ impl QuantizationMode {
pub fn weight_codes_sign_flip_mask(&self) -> Option<u8> {
match self {
+ QuantizationMode::U2 => None,
QuantizationMode::U4 => Some(0x88),
QuantizationMode::I8 => None,
QuantizationMode::U8 => Some(0x80),
@@ -42,6 +45,7 @@ impl QuantizationMode {
impl From<QuantizationMode> for DataType {
fn from(val: QuantizationMode) -> Self {
match val {
+ QuantizationMode::U2 => DataType::U2,
QuantizationMode::U4 => DataType::U4,
QuantizationMode::I8 => DataType::I8,
QuantizationMode::U8 => DataType::U8,
diff --git a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
index d5ff4cc24..462d291e1 100644
--- a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
@@ -33,12 +33,13 @@ impl ActivationQuantization {
emit_group_sums: bool,
code_layout: Int8CodeLayout,
) -> Option<Self> {
- (scale_group_size == ACTIVATION_SCALE_GROUP_SIZE && (!emit_group_sums || matches!(weight_group_size, 32 | 64)))
- .then_some(Self {
- scale_group_size,
- sum_group_size: emit_group_sums.then_some(weight_group_size),
- code_layout,
- })
+ (scale_group_size == ACTIVATION_SCALE_GROUP_SIZE
+ && (!emit_group_sums || matches!(weight_group_size, 32 | 64 | 128)))
+ .then_some(Self {
+ scale_group_size,
+ sum_group_size: emit_group_sums.then_some(weight_group_size),
+ code_layout,
+ })
}
pub const fn scale_group_size(self) -> u32 {
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs
index dc69c1203..14462b617 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs
@@ -18,8 +18,8 @@ impl Int8CodeLayout {
pub const fn for_right_bits(bits: u32) -> Option<Self> {
match bits {
+ 2 | 8 => Some(Self::Sequential),
4 => Some(Self::GroupedByNibble),
- 8 => Some(Self::Sequential),
_ => None,
}
}
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
index 1d2030d4a..b611a716d 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
@@ -59,6 +59,10 @@ impl QuantParams {
}
}
+ pub const fn is_per_row(self) -> bool {
+ self.group_count == 1
+ }
+
pub const fn layout(self) -> QuantParamsLayout {
self.layout
}
@@ -101,15 +105,16 @@ impl QuantParams {
packing_divisor: u32,
) -> QuantParamsStrides {
let row_values = self.padded_values_per_row(packing_divisor);
- match self.layout {
- QuantParamsLayout::OutputGroup => QuantParamsStrides {
- output_stride: row_values,
- group_stride: 1,
- },
- QuantParamsLayout::GroupOutput => QuantParamsStrides {
- output_stride: 1,
- group_stride: row_values,
- },
+ let (output_stride, mut group_stride) = match self.layout {
+ QuantParamsLayout::OutputGroup => (row_values, 1),
+ QuantParamsLayout::GroupOutput => (1, row_values),
+ };
+ if self.is_per_row() {
+ group_stride = 0;
+ }
+ QuantParamsStrides {
+ output_stride,
+ group_stride,
}
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs b/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs
index 21b36d47c..df79d2fa7 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/embedding/input_embedding_lookup.rs
@@ -79,6 +79,7 @@ pub fn input_embedding_lookup<T: DataTypeRepr + Float>(
let scale = unsafe { (*scales.add(index)).to_f32().unwrap() };
let offset = token * weights_stride;
let code = match quantization_mode {
+ QuantizationMode::U2 => unreachable!("2-bit weights are not embedding tables"),
QuantizationMode::U4 => read_u4(values, 2 * offset + column) as f32,
QuantizationMode::I8 => (unsafe { *(values as *const i8).add(offset + column) }) as f32,
QuantizationMode::U8 => (unsafe { *values.add(offset + column) }) as f32,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
index 7716791dc..72cb91c4f 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
@@ -129,8 +129,9 @@ impl MatmulKernel for MatmulCpuKernel {
let compatible = matches!(a_group_size, 32 | 64 | 128)
&& k.is_multiple_of(a_group_size)
&& matches!(b.group_size(), Some(32 | 64 | 128))
- && b.quantized()
- .is_some_and(|quantized| matches!(quantized.mode, QuantizationMode::U4 | QuantizationMode::U8));
+ && b.quantized().is_some_and(|quantized| {
+ matches!(quantized.mode, QuantizationMode::U2 | QuantizationMode::U4 | QuantizationMode::U8)
+ });
if !compatible {
return Err(MatmulCpuError::IncompatibleA {
path: "CpuMatmul",
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.h b/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.h
index 7b0cd7bbe..33993c836 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/matmul.h
@@ -7,6 +7,12 @@ using namespace metal;
namespace uzu::matmul {
static constant constexpr uint32_t QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT = 4;
+static constant constexpr uint32_t MXU_TILED_BLOCK_ROWS = 32;
+
+static constant constexpr uint32_t MXU_TILED_W2_BLOCK_ROWS = 16;
+
+static constant constexpr uint32_t MXU_TILED_PREPARE_THREADS = 1024;
+
typedef struct {
uint32_t M;
uint32_t N;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/quantization.h b/crates/uzu-engine/src/backends/metal/kernel/generated/quantization.h
index 01c1ffba7..c832a2a93 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/generated/quantization.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/quantization.h
@@ -9,5 +9,6 @@ enum class QuantizationMode : uint32_t {
U4 = 0,
I8 = 1,
U8 = 2,
+ U2 = 3,
};
} // namespace uzu::quantization
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h
index 19ba79a27..5f2c9ddf5 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h
@@ -23,9 +23,12 @@ static METAL_FUNC uint2 morton_tile_id(uint2 threadgroup_position) {
return uint2(morton_expand_bits(threadgroup_position.x), morton_expand_bits(threadgroup_position.x >> 1));
}
+template <bool MAY_USE_MORTON = true>
static METAL_FUNC uint2 tile_id(uint2 threadgroup_position, const constant uzu::matmul::GemmParams* params) {
- if (params->use_morton) {
- return morton_tile_id(threadgroup_position);
+ if constexpr (MAY_USE_MORTON) {
+ if (params->use_morton) {
+ return morton_tile_id(threadgroup_position);
+ }
}
return threadgroup_position;
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
index 23efd27b3..440c479d4 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
@@ -5,8 +5,10 @@
using namespace metal;
-template <typename T, typename BiasT, bool SINGLE_SIMDGROUP = false>
+template <typename T, typename BiasT, bool SINGLE_SIMDGROUP = false, typename Source>
static METAL_FUNC void apply_output_random_hadamard_transform(
+ Source source,
+ uint source_stride,
device T* output_block,
const device int32_t* rht_factors_block,
const device BiasT* bias_block,
@@ -25,12 +27,14 @@ static METAL_FUNC void apply_output_random_hadamard_transform(
const ushort row_local = ushort(cell / stripes_per_row);
const ushort stripe = ushort(cell % stripes_per_row);
const ushort col_local = stripe * ushort(HADAMARD_TRANSFORM_BLOCK_SIZE) + simd_lane;
- const size_t output_index = size_t(row_local) * size_t(leading_dimension_d) + size_t(col_local);
- T transformed =
- simdgroup_output_random_hadamard_transform(simd_lane, output_block[output_index], rht_factors_block[col_local]);
+ T transformed = simdgroup_output_random_hadamard_transform(
+ simd_lane,
+ source[size_t(row_local) * size_t(source_stride) + size_t(col_local)],
+ rht_factors_block[col_local]
+ );
if (apply_bias) {
transformed = T(float(transformed) + float(bias_block[col_local]));
}
- output_block[output_index] = transformed;
+ output_block[size_t(row_local) * size_t(leading_dimension_d) + size_t(col_local)] = transformed;
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index 2776903b2..11d3eca06 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -52,7 +52,7 @@ struct MxuMmaCore {
static_assert(SIMDGROUPS_PER_K == 1 || SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN == 1, "K slices share one tile");
UZU_CONST ushort SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;
UZU_CONST ushort SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;
- UZU_CONST ushort SIMDGROUP_BLOCK_K = static_cast<ushort>(MXU_SIMDGROUP_BLOCK_K);
+ UZU_CONST ushort SIMDGROUP_BLOCK_K = Right::MMA_BLOCK_K;
UZU_CONST bool TRANSPOSE_RIGHT = TRANSPOSE_B;
UZU_CONST ushort THREADGROUP_BLOCK_K_FP = gemm_tiling_block_k(GEMM_TILING);
static_assert(
@@ -86,9 +86,10 @@ struct MxuMmaCore {
const uint partition = thread_context.threadgroup_position.z;
const uint tile_y = thread_context.threadgroup_position.y;
- const uint2 tile = tile_id(uint2(thread_context.threadgroup_position.x, tile_y), params);
+ constexpr bool MAY_USE_MORTON = !Right::QUANTIZED;
+ const uint2 tile = tile_id<MAY_USE_MORTON>(uint2(thread_context.threadgroup_position.x, tile_y), params);
const auto geometry = ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::compute(tile, params);
- if (geometry.out_of_bounds) {
+ if (MAY_USE_MORTON && geometry.out_of_bounds) {
return;
}
@@ -126,69 +127,108 @@ struct MxuMmaCore {
.abs_row_base = uint(geometry.block_row_start) + tile_row_offset,
};
+ const bool simdgroup_rows_complete =
+ (alignment.contains(GemmAlignment::M) || (simdgroup_limit_m == SIMDGROUP_BLOCK_M)) &&
+ ((Right::QUANTIZED && SIMDGROUPS_PER_ROW == 1) || simdgroup_limit_m > 0);
+ const bool simdgroup_columns_complete =
+ (alignment.contains(GemmAlignment::N) || (simdgroup_limit_n == SIMDGROUP_BLOCK_N)) &&
+ ((Right::QUANTIZED && SIMDGROUPS_PER_COLUMN == 1) || simdgroup_limit_n > 0);
+
const bool apply_scale = output_transform.contains(GemmDTransform::SCALE);
const bool apply_bias = output_transform.contains(GemmDTransform::BIAS);
const bool apply_rht = output_transform.contains(GemmDTransform::RHT);
const device RightElementType* bias_simdgroup = output_bias + size_t(block_col) + size_t(tile_col_offset);
- dispatch_bool(
- (alignment.contains(GemmAlignment::M) || (simdgroup_limit_m == SIMDGROUP_BLOCK_M)) && simdgroup_limit_m > 0,
- [&](auto aligned_m) {
- dispatch_bool(
- (alignment.contains(GemmAlignment::N) || (simdgroup_limit_n == SIMDGROUP_BLOCK_N)) &&
- simdgroup_limit_n > 0,
- [&](auto aligned_n) {
- AccumFragment accumulator_tile =
- Schedule::template launch<MxuMmaCore, aligned_m.value, aligned_n.value>(
- left,
- right,
- b_shared,
- params,
- tile_context,
- alignment,
- thread_context
- );
-
- if (!accumulator_tile.template reduce_k_slices<SIMDGROUPS_PER_K, 1>(
- reinterpret_cast<threadgroup float*>(b_shared),
- thread_context
- )) {
- return;
- }
-
- if (apply_scale) {
- const AccumulatorType scale = AccumulatorType(params->ab_scale);
- accumulator_tile.map([&](auto value) { return value * scale; });
- }
-
- if (apply_bias && !apply_rht) {
- accumulator_tile.map_coords(thread_context.simd_lane_id, [&](short, short col, auto value) {
- if constexpr (aligned_n.value) {
- return value + AccumulatorType(bias_simdgroup[col]);
- } else {
- if (col < simdgroup_limit_n) {
- return value + AccumulatorType(bias_simdgroup[col]);
- }
- return value;
- }
- });
- }
-
- if constexpr (aligned_m.value && aligned_n.value) {
- accumulator_tile.store(thread_context.simd_lane_id, d_simdgroup, int(params->leading_dimension_d));
- } else {
- accumulator_tile.template store_safe<aligned_m.value, aligned_n.value>(
- thread_context.simd_lane_id,
- d_simdgroup,
- int(params->leading_dimension_d),
- short2(simdgroup_limit_n, simdgroup_limit_m)
- );
- }
+ threadgroup OutputElementType* tile_stage = reinterpret_cast<threadgroup OutputElementType*>(b_shared);
+ if constexpr (SIMDGROUPS_PER_K > 1) {
+ static_assert(Left::QUANTIZED, "K slices park their partials in b_shared, which int8 activations leave free");
+ static_assert(THREADGROUP_BLOCK_N == HADAMARD_TRANSFORM_BLOCK_SIZE, "the staged RHT runs one stripe per row");
+ static_assert(sizeof(OutputElementType) <= sizeof(AccumulatorType), "the tile must fit the partials");
+ }
+
+ dispatch_bool(simdgroup_rows_complete, [&](auto aligned_m) {
+ dispatch_bool(simdgroup_columns_complete, [&](auto aligned_n) {
+ AccumFragment accumulator_tile = Schedule::template launch<MxuMmaCore, aligned_m.value, aligned_n.value>(
+ left,
+ right,
+ b_shared,
+ params,
+ tile_context,
+ alignment,
+ thread_context
+ );
+
+ if (!accumulator_tile.template reduce_k_slices<SIMDGROUPS_PER_K, 1>(
+ reinterpret_cast<threadgroup float*>(b_shared),
+ thread_context
+ )) {
+ return;
+ }
+
+ if (apply_scale) {
+ const AccumulatorType scale = AccumulatorType(params->ab_scale);
+ accumulator_tile.map([&](auto value) { return value * scale; });
+ }
+
+ if constexpr (SIMDGROUPS_PER_K > 1) {
+ if (apply_rht) {
+ simdgroup_barrier(mem_flags::mem_threadgroup);
+ accumulator_tile.template store_safe<aligned_m.value, aligned_n.value>(
+ thread_context.simd_lane_id,
+ tile_stage,
+ int(THREADGROUP_BLOCK_N),
+ short2(simdgroup_limit_n, simdgroup_limit_m)
+ );
+ return;
+ }
+ }
+
+ if (apply_bias && !apply_rht) {
+ accumulator_tile.map_coords(thread_context.simd_lane_id, [&](short, short col, auto value) {
+ if constexpr (aligned_n.value) {
+ return value + AccumulatorType(bias_simdgroup[col]);
+ } else {
+ if (col < simdgroup_limit_n) {
+ return value + AccumulatorType(bias_simdgroup[col]);
}
+ return value;
+ }
+ });
+ }
+
+ if constexpr (aligned_m.value && aligned_n.value) {
+ accumulator_tile.store(thread_context.simd_lane_id, d_simdgroup, int(params->leading_dimension_d));
+ } else {
+ accumulator_tile.template store_safe<aligned_m.value, aligned_n.value>(
+ thread_context.simd_lane_id,
+ d_simdgroup,
+ int(params->leading_dimension_d),
+ short2(simdgroup_limit_n, simdgroup_limit_m)
);
}
- );
+ });
+ });
+
+ if constexpr (SIMDGROUPS_PER_K > 1) {
+ if (apply_rht) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ apply_output_random_hadamard_transform<OutputElementType, RightElementType>(
+ tile_stage,
+ uint(THREADGROUP_BLOCK_N),
+ d + block_row * params->leading_dimension_d + block_col,
+ rht_factors + block_col,
+ output_bias + block_col,
+ apply_bias,
+ ushort(min(int(THREADGROUP_BLOCK_M), int(params->M) - int(block_row))),
+ ushort(THREADGROUP_BLOCK_N),
+ params->leading_dimension_d,
+ SIMDGROUPS_PER_K,
+ thread_context
+ );
+ return;
+ }
+ }
if (k_slice != 0) {
return;
@@ -203,6 +243,8 @@ struct MxuMmaCore {
}
simdgroup_barrier(mem_flags::mem_device);
apply_output_random_hadamard_transform<OutputElementType, RightElementType, true>(
+ d_simdgroup,
+ params->leading_dimension_d,
d_simdgroup,
rht_factors + block_col + tile_col_offset,
bias_simdgroup,
@@ -225,7 +267,9 @@ struct MxuMmaCore {
device OutputElementType* d_block = d + block_row * params->leading_dimension_d + block_col;
const ushort tile_block_rows = ushort(min(int(THREADGROUP_BLOCK_M), int(params->M) - int(block_row)));
const ushort tile_block_cols = ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(block_col)));
- apply_output_random_hadamard_transform(
+ apply_output_random_hadamard_transform<OutputElementType, RightElementType>(
+ d_block,
+ params->leading_dimension_d,
d_block,
rht_factors + block_col,
output_bias + block_col,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_tiled_layout.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_tiled_layout.h
new file mode 100644
index 000000000..82c84900d
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_tiled_layout.h
@@ -0,0 +1,23 @@
+#pragma once
+
+#include "../../../common/defines.h"
+#include "../../../generated/matmul.h"
+#include "../../common/mxu_fragment/ops.h"
+
+namespace uzu {
+namespace gemm {
+
+template <uint BITS>
+struct MxuTiledLayout {
+ using Ops = uzu::matmul::MxuFragmentOps<>;
+ UZU_CONST uint BLOCK_ROWS = BITS == 2 ? uzu::matmul::MXU_TILED_W2_BLOCK_ROWS : uzu::matmul::MXU_TILED_BLOCK_ROWS;
+ UZU_CONST uint LANE_VECTOR_CHUNKS = uzu::matmul::MXU_TILED_BLOCK_ROWS / BLOCK_ROWS;
+ UZU_CONST uint ROW_FRAGMENTS = BLOCK_ROWS / Ops::FRAGMENT_ROWS;
+ UZU_CONST uint COL_FRAGMENTS = BITS == 4 ? 1 : 2;
+ UZU_CONST uint LANE_WORDS = ROW_FRAGMENTS * Ops::THREAD_ELEMENT_ROWS * COL_FRAGMENTS;
+ UZU_CONST uint LANE_BYTES = LANE_WORDS * sizeof(uint);
+ UZU_CONST uint BLOCK_CHUNK_BYTES = METAL_SIMD_SIZE * LANE_BYTES;
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h
index 2c4106a84..24d36f84d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h
@@ -47,11 +47,16 @@ struct RightOperand {
UZU_CONST GemmBPrologueKind SCHEME = PROLOGUE;
UZU_CONST bool NEEDS_CORRECTION = PROLOGUE == GemmBPrologueKind::ScaleBiasDequant;
- UZU_CONST ushort CODE_ORIGIN = (!QUANTIZED || (BITS_ == 4 && PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant))
- ? 0
- : ushort(1u << (BITS - 1));
+ UZU_CONST ushort CODE_ORIGIN =
+ (!QUANTIZED || ((BITS_ == 2 || BITS_ == 4) && PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant))
+ ? 0
+ : ushort(1u << (BITS - 1));
+ UZU_CONST ushort MMA_BLOCK_K = BITS_ == 2 ? 2 * MXU_SIMDGROUP_BLOCK_K : MXU_SIMDGROUP_BLOCK_K;
- static_assert(!QUANTIZED || BITS_ == 4 || BITS_ == 8, "quantized integer weights must use 4 or 8 bits");
+ static_assert(
+ !QUANTIZED || BITS_ == 2 || BITS_ == 4 || BITS_ == 8,
+ "quantized integer weights must use 2, 4 or 8 bits"
+ );
static_assert(!QUANTIZED || PROLOGUE != GemmBPrologueKind::FullPrecision, "quantized weights need a scheme");
using CodeElement = int8_t;
@@ -61,7 +66,7 @@ struct RightOperand {
template <ushort BLOCK_K>
static constexpr ushort outer_block_k() {
if constexpr (QUANTIZED) {
- static_assert(GROUP_SIZE_ % MXU_SIMDGROUP_BLOCK_K == 0, "weight groups must contain complete MMA chunks");
+ static_assert(GROUP_SIZE_ % MMA_BLOCK_K == 0, "weight groups must contain complete MMA chunks");
static_assert(BLOCK_K % GROUP_SIZE_ == 0, "tile block K must contain complete weight groups");
return GROUP_SIZE_;
} else {
@@ -92,6 +97,7 @@ struct RightStorage {
const device typename Right::ScaleElement* biases;
const device uint8_t* zero_points;
bool signed_codes;
+ bool hoist_row_metadata;
GemmBLayout gemm_b_layout;
METAL_FUNC uint sign_mask() const thread {
@@ -170,10 +176,11 @@ METAL_FUNC RightStorage<Right> pack_right(
const device Element* biases,
const device uint8_t* zero_points,
const bool signed_codes,
+ const bool hoist_row_metadata,
const GemmBLayout gemm_b_layout
) {
if constexpr (!Right::QUANTIZED) {
- return {dense, nullptr, nullptr, nullptr, nullptr, false, GemmBLayout::RowMajor};
+ return {dense, nullptr, nullptr, nullptr, nullptr, false, false, GemmBLayout::RowMajor};
} else {
return {
nullptr,
@@ -182,6 +189,7 @@ METAL_FUNC RightStorage<Right> pack_right(
Right::SCHEME == GemmBPrologueKind::ScaleBiasDequant ? biases : nullptr,
Right::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant ? zero_points : nullptr,
signed_codes,
+ hoist_row_metadata,
gemm_b_layout
};
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
index c9c163c11..937ee0302 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
@@ -7,6 +7,7 @@
#include "../../../common/fragment.h"
#include "../../../common/quant_pack.h"
#include "../../../common/quant_unpack.h"
+#include "../mxu_tiled_layout.h"
#include "../schedules/tile_context.h"
using namespace metal;
@@ -30,6 +31,11 @@ static METAL_FUNC void for_each_fragment_row(Visitor visitor) {
}
}
}
+
+template <bool ALIGNED>
+static METAL_FUNC short clamp_to_live_row(const short row_offset, const short row_limit) {
+ return ALIGNED ? row_offset : min(row_offset, short(row_limit - 1));
+}
} // namespace
template <typename Fragment, bool ALIGNED, bool CODES_GROUPED_BY_NIBBLE>
@@ -42,11 +48,10 @@ load_int8_tile(const device int8_t* src, const int row_stride, const short simdg
const device int8_t* base = src + int(position.y) * row_stride + int(get_pack_factor<W4_BITS>()) * int(position.x);
const short row_limit = simdgroup_limit - position.y;
for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short row_offset) {
- vec<uint, Fragment::COL_FRAGMENTS> packed_chunk(0u);
- if (ALIGNED || row_offset < row_limit) {
- packed_chunk =
- *reinterpret_cast<const device vec<uint, Fragment::COL_FRAGMENTS>*>(base + int(row_offset) * row_stride);
- }
+ const vec<uint, Fragment::COL_FRAGMENTS> packed_chunk =
+ *reinterpret_cast<const device vec<uint, Fragment::COL_FRAGMENTS>*>(
+ base + int(clamp_to_live_row<ALIGNED>(row_offset, row_limit)) * row_stride
+ );
METAL_PRAGMA_UNROLL
for (ushort fragment_column = 0; fragment_column < Fragment::COL_FRAGMENTS; ++fragment_column) {
reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, fragment_column))[row_slot] =
@@ -65,12 +70,10 @@ load_int8_tile(const device int8_t* src, const int row_stride, const short simdg
for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short row_offset) {
METAL_PRAGMA_UNROLL
for (ushort fragment_column = 0; fragment_column < Fragment::COL_FRAGMENTS; ++fragment_column) {
- char4 codes(0);
- if (ALIGNED || row_offset < row_limit) {
- codes = *reinterpret_cast<const device packed_char4*>(
- base + int(row_offset) * row_stride + int(fragment_column) * Ops::FRAGMENT_COLS
- );
- }
+ const char4 codes = *reinterpret_cast<const device packed_char4*>(
+ base + int(clamp_to_live_row<ALIGNED>(row_offset, row_limit)) * row_stride +
+ int(fragment_column) * Ops::FRAGMENT_COLS
+ );
reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, fragment_column))[row_slot] =
as_type<uint>(codes);
}
@@ -93,11 +96,12 @@ struct Int8Cursor {
GemmBLayout gemm_b_layout;
uint chunk_stride;
- METAL_FUNC Fragment load(const uint) const thread {
+ METAL_FUNC Fragment load(const uint chunk) const thread {
+ const device int8_t* chunk_address = address + chunk * chunk_stride;
if (gemm_b_layout == GemmBLayout::MxuTiled) {
Fragment tile;
- const uint4 low = *reinterpret_cast<const device uint4*>(address);
- const uint4 high = *reinterpret_cast<const device uint4*>(address + sizeof(uint4));
+ const uint4 low = *reinterpret_cast<const device uint4*>(chunk_address);
+ const uint4 high = *reinterpret_cast<const device uint4*>(chunk_address + sizeof(uint4));
const uint words[8] = {low.x, low.y, low.z, low.w, high.x, high.y, high.z, high.w};
for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short) {
METAL_PRAGMA_UNROLL
@@ -109,25 +113,27 @@ struct Int8Cursor {
return tile;
}
return load_int8_tile<Fragment, ALIGNED, CODES_GROUPED_BY_NIBBLE>(
- address,
+ chunk_address,
row_stride,
simdgroup_limit,
simd_lane_id
);
}
- METAL_FUNC void advance() thread { address += chunk_stride; }
+ METAL_FUNC void advance(const uint chunks) thread { address += chunks * chunk_stride; }
METAL_FUNC void begin_k_group(const uint) thread {}
};
template <typename Fragment, typename Operand, bool ALIGNED>
-struct W4Cursor {
+struct PackedCursor {
using Ops = typename Fragment::FragmentOpsType;
+ UZU_CONST ushort BITS = Operand::BITS;
+ UZU_CONST uint CODE_MASK = ((1u << BITS) - 1) * 0x01010101u;
UZU_CONST short BLOCK_K = short(Fragment::COL_FRAGMENTS * Ops::FRAGMENT_ROWS);
UZU_CONST bool HAS_ZERO_POINTS = Operand::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant;
- UZU_CONST uint CHUNK_BYTES = uint(BLOCK_K) * uint(get_bytes_per_pack<W4_BITS>()) / uint(get_pack_factor<W4_BITS>());
+ UZU_CONST uint CHUNK_BYTES = uint(BLOCK_K) * uint(get_bytes_per_pack<BITS>()) / uint(get_pack_factor<BITS>());
struct PackedChunk {
UZU_CONST ushort WORD_COUNT = Fragment::ROW_FRAGMENTS * Ops::THREAD_ELEMENT_ROWS;
@@ -150,6 +156,19 @@ struct W4Cursor {
METAL_FUNC PackedChunk fetch(const uint chunk_index) const thread {
if (gemm_b_layout == GemmBLayout::MxuTiled) {
+ if constexpr (BITS == 2) {
+ const device uint8_t* lane_vector = current + (chunk_index & ~1u) * chunk_stride;
+ const uint block_stride = MxuTiledLayout<BITS>::BLOCK_ROWS * uint(row_stride_bytes);
+ PackedChunk packed_chunk;
+ METAL_PRAGMA_UNROLL
+ for (ushort fragment_row = 0; fragment_row < Fragment::ROW_FRAGMENTS; ++fragment_row) {
+ const uint4 vector = *reinterpret_cast<const device uint4*>(lane_vector + fragment_row * block_stride);
+ const uint2 words = chunk_index % 2 ? vector.zw : vector.xy;
+ packed_chunk.words[PackedChunk::word_index(fragment_row, 0)] = words.x;
+ packed_chunk.words[PackedChunk::word_index(fragment_row, 1)] = words.y;
+ }
+ return packed_chunk;
+ }
const uint4 words = *reinterpret_cast<const device uint4*>(current + chunk_index * chunk_stride);
return {{words.x, words.y, words.z, words.w}};
}
@@ -174,6 +193,14 @@ struct W4Cursor {
for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short row_offset) {
const uint word =
packed_chunk.words[PackedChunk::word_index(fragment_row, row_slot)] ^ (signed_codes ? W4_SIGN_MASK : 0u);
+ if constexpr (BITS == 2) {
+ METAL_PRAGMA_UNROLL
+ for (ushort fragment_column = 0; fragment_column < Fragment::COL_FRAGMENTS; ++fragment_column) {
+ reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, fragment_column))[row_slot] =
+ (word >> (BITS * fragment_column)) & CODE_MASK;
+ }
+ return;
+ }
uint low = word & W4_NIBBLE_MASK;
uint high = (word >> W4_BITS) & W4_NIBBLE_MASK;
if constexpr (HAS_ZERO_POINTS) {
@@ -197,9 +224,9 @@ struct W4Cursor {
return tile;
}
- METAL_FUNC Fragment load(const uint) const thread { return decode(fetch(0)); }
+ METAL_FUNC Fragment load(const uint chunk) const thread { return decode(fetch(chunk)); }
- METAL_FUNC void advance() thread { current += chunk_stride; }
+ METAL_FUNC void advance(const uint chunks) thread { current += chunks * chunk_stride; }
METAL_FUNC void begin_k_group(const uint k_offset) thread {
if constexpr (HAS_ZERO_POINTS) {
@@ -243,21 +270,18 @@ static METAL_FUNC auto make_right_cursor(
const device uint8_t* current =
source.codes + size_t(tile.block_col + tile.tile_col_offset) * size_t(row_stride_bytes) +
size_t(tile.k_offset) * size_t(get_bytes_per_pack<Operand::BITS>()) / size_t(get_pack_factor<Operand::BITS>());
- static_assert(Core::SIMDGROUP_BLOCK_N == 2 * Ops::FRAGMENT_ROWS, "the MxuTiled layout holds 32-row blocks");
- const uint mxu_tiled_lane_bytes = Operand::BITS == 4 ? 16 : 32;
- const uint mxu_tiled_chunk_bytes = METAL_SIMD_SIZE * mxu_tiled_lane_bytes;
+ using Layout = MxuTiledLayout<Operand::BITS>;
+ static_assert(Core::SIMDGROUP_BLOCK_N % Layout::BLOCK_ROWS == 0, "a simdgroup tile covers whole MxuTiled blocks");
if (source.gemm_b_layout == GemmBLayout::MxuTiled) {
- const uint chunk_count = uint(params->K) / uint(Core::SIMDGROUP_BLOCK_K);
- const uint row_block = uint(tile.block_col + tile.tile_col_offset) / uint(Core::SIMDGROUP_BLOCK_N);
- current =
- source.codes +
- (size_t(row_block) * chunk_count + tile.k_offset / uint(Core::SIMDGROUP_BLOCK_K)) * mxu_tiled_chunk_bytes +
- thread_context.simd_lane_id * mxu_tiled_lane_bytes;
+ constexpr uint LANE_VECTOR_K = Layout::LANE_VECTOR_CHUNKS * Operand::MMA_BLOCK_K;
+ current = source.codes + size_t(tile.block_col + tile.tile_col_offset) * size_t(row_stride_bytes) +
+ size_t(tile.k_offset / LANE_VECTOR_K) * Layout::BLOCK_CHUNK_BYTES +
+ thread_context.simd_lane_id * Layout::LANE_BYTES;
}
- if constexpr (Operand::BITS == 4) {
- static_assert(Core::TILES_K == get_pack_factor<Operand::BITS>(), "W4 requires two K fragments");
- using Cursor = W4Cursor<Fragment, Operand, ALIGNED>;
+ if constexpr (Operand::BITS == 2 || Operand::BITS == 4) {
+ static_assert(Core::TILES_K == get_pack_factor<Operand::BITS>(), "packed codes need one K fragment per code");
+ using Cursor = PackedCursor<Fragment, Operand, ALIGNED>;
const device uint8_t* zero_points = nullptr;
const uint zero_point_group_stride_bytes = params->zero_point_group_stride / get_pack_factor<W4_BITS>();
if constexpr (Cursor::HAS_ZERO_POINTS) {
@@ -273,10 +297,11 @@ static METAL_FUNC auto make_right_cursor(
zero_points,
zero_point_group_stride_bytes,
source.gemm_b_layout,
- source.gemm_b_layout == GemmBLayout::MxuTiled ? mxu_tiled_chunk_bytes : Cursor::CHUNK_BYTES
+ source.gemm_b_layout == GemmBLayout::MxuTiled ? Layout::BLOCK_CHUNK_BYTES / Layout::LANE_VECTOR_CHUNKS
+ : Cursor::CHUNK_BYTES
};
} else {
- static_assert(Operand::BITS == 8, "integer tile cursors support 4-bit and 8-bit codes");
+ static_assert(Operand::BITS == 8, "integer tile cursors support 2-bit, 4-bit and 8-bit codes");
const device int8_t* origin_int8 = reinterpret_cast<const device int8_t*>(current);
return Int8Cursor<Fragment, ALIGNED, false>{
origin_int8,
@@ -284,7 +309,7 @@ static METAL_FUNC auto make_right_cursor(
tile.simdgroup_limit_n,
ushort(thread_context.simd_lane_id),
source.gemm_b_layout,
- source.gemm_b_layout == GemmBLayout::MxuTiled ? mxu_tiled_chunk_bytes
+ source.gemm_b_layout == GemmBLayout::MxuTiled ? Layout::BLOCK_CHUNK_BYTES
: uint(Int8Cursor<Fragment, ALIGNED, false>::BLOCK_K)
};
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
index acd2b8e62..0446b8cf0 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
@@ -55,6 +55,7 @@ struct MetadataContext {
uint left_group_count;
uint right_group_count;
uint right_scale_group_stride;
+ uint padded_output_count;
};
template <typename LeftOperand, typename RightOperand>
@@ -143,7 +144,7 @@ struct IntegerSchedule {
uint right_scale_column_start = metadata_context.right_column_base + uint(right_column_offset);
if constexpr (!ALIGNED_N) {
right_scale_column_start =
- clamp_group_output_column(right_scale_column_start, metadata_context.right_scale_group_stride);
+ clamp_group_output_column(right_scale_column_start, metadata_context.padded_output_count);
}
scales[tile_n] = *reinterpret_cast<const device ScaleVector*>(
right.scales + right_group_index * metadata_context.right_scale_group_stride + right_scale_column_start
@@ -175,9 +176,9 @@ struct IntegerSchedule {
}
template <typename Core>
- static constexpr bool prefetches_int4_chunks() {
+ static constexpr bool prefetches_packed_chunks() {
return (Core::TILES_M == 1 || (Core::TILES_M == 2 && Core::TILES_N == 2 && CENTER_RIGHT_ZERO_POINTS)) &&
- chunks_per_k_group<Core>() > 1 && RightOperand::BITS == 4;
+ chunks_per_k_group<Core>() > 1 && (RightOperand::BITS == 2 || RightOperand::BITS == 4);
}
template <typename Core, bool ALIGNED_M, bool ALIGNED_N>
@@ -238,26 +239,24 @@ struct IntegerSchedule {
) {
GroupProducts<Core> group_product;
group_product.clear();
- if constexpr (prefetches_int4_chunks<Core>()) {
+ if constexpr (prefetches_packed_chunks<Core>()) {
for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {
auto left_tile = left_cursor.load(uint(chunk));
auto right_tile = right_cursor.decode(packed_right_chunks[chunk]);
uzu::matmul::fragment_mma(group_product, left_tile, right_tile);
- left_cursor.advance();
- right_cursor.advance();
});
} else {
decltype(right_cursor.load(0u)) right_tiles[CHUNKS_PER_K_GROUP];
for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {
right_tiles[chunk] = right_cursor.load(uint(chunk));
- right_cursor.advance();
});
for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {
auto left_tile = left_cursor.load(uint(chunk));
uzu::matmul::fragment_mma(group_product, left_tile, right_tiles[chunk]);
- left_cursor.advance();
});
}
+ left_cursor.advance(CHUNKS_PER_K_GROUP);
+ right_cursor.advance(CHUNKS_PER_K_GROUP);
return group_product;
}
@@ -266,10 +265,19 @@ struct IntegerSchedule {
thread typename Core::AccumFragment& accumulator,
thread GroupProducts<Core>& group_product,
const thread LeftMetadata<Core, ALIGNED_M>& left_metadata,
- const thread RightMetadata<Core, ALIGNED_N>& right_metadata
+ const thread RightMetadata<Core, ALIGNED_N>& right_metadata,
+ thread float (&row_correction_sums)[LeftMetadata<Core, ALIGNED_M>::LEFT_VALUE_COUNT],
+ const bool hoist_row_metadata
) {
using Ops = typename Core::FragmentOps;
+ if constexpr (HAS_BIAS) {
+ if (hoist_row_metadata) {
+ for_each_static_index<int(LeftMetadata<Core, ALIGNED_M>::LEFT_VALUE_COUNT)>([&](const ushort index) {
+ row_correction_sums[index] += left_metadata.group_corrections[index];
+ });
+ }
+ }
for_each_fragment(
accumulator,
group_product,
@@ -278,6 +286,11 @@ struct IntegerSchedule {
[&](const ushort element) {
const ushort right_column = element % Ops::THREAD_ELEMENT_COLS;
const ushort left_row = tile_m * Ops::THREAD_ELEMENT_ROWS + element / Ops::THREAD_ELEMENT_COLS;
+ if (hoist_row_metadata) {
+ accumulated[element] =
+ fma(left_metadata.scales[left_row], float(group_product_value[element]), accumulated[element]);
+ return;
+ }
const float right_scale = float(right_metadata.scales[tile_n][right_column]);
accumulated[element] =
fma(left_metadata.scales[left_row] * right_scale,
@@ -295,6 +308,28 @@ struct IntegerSchedule {
);
}
+ template <typename Core, bool ALIGNED_N>
+ static METAL_FUNC UZU_ALWAYS_INLINE void apply_row_metadata(
+ thread typename Core::AccumFragment& accumulator,
+ const thread RightMetadata<Core, ALIGNED_N>& right_metadata,
+ const thread float (&row_correction_sums)[Core::TILES_M * Core::FragmentOps::THREAD_ELEMENT_ROWS]
+ ) {
+ using Ops = typename Core::FragmentOps;
+ for_each_static_index<int(Core::TILES_N)>([&](const ushort tile_n) {
+ for_each_static_index<int(Core::TILES_M)>([&](const ushort tile_m) {
+ thread auto& accumulated = accumulator.fragment_at(tile_m, tile_n);
+ for_each_static_index<int(Ops::THREAD_ELEMENT_ROWS) * int(Ops::THREAD_ELEMENT_COLS)>([&](const ushort element) {
+ const ushort right_column = element % Ops::THREAD_ELEMENT_COLS;
+ const ushort left_row = tile_m * Ops::THREAD_ELEMENT_ROWS + element / Ops::THREAD_ELEMENT_COLS;
+ accumulated[element] =
+ fma(float(right_metadata.bias_offsets[tile_n][right_column]),
+ row_correction_sums[left_row],
+ float(right_metadata.scales[tile_n][right_column]) * accumulated[element]);
+ });
+ });
+ });
+ }
+
template <typename Core, bool ALIGNED_M, bool ALIGNED_N>
static METAL_FUNC typename Core::AccumFragment launch(
typename Core::LeftStorage left_storage,
@@ -333,6 +368,8 @@ struct IntegerSchedule {
uint(params->K) / uint(LeftOperand::GROUP_SIZE),
uint(params->K) / RIGHT_GROUP_SIZE,
params->scale_group_stride,
+ (uint(params->N) + uzu::matmul::QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT - 1) &
+ ~(uzu::matmul::QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT - 1),
};
const uint first_right_group = tile.k_offset / RIGHT_GROUP_SIZE;
@@ -341,14 +378,21 @@ struct IntegerSchedule {
typename Core::AccumFragment accumulator;
accumulator.clear();
+ const bool hoist_row_metadata = right_storage.hoist_row_metadata;
+ float row_correction_sums[LeftMetadata<Core, ALIGNED_M>::LEFT_VALUE_COUNT] = {};
+
const int right_group_count = int(params->aligned_inner_iterations);
constexpr int CHUNKS_PER_K_GROUP = chunks_per_k_group<Core>();
- constexpr bool PREFETCH_INT4_CHUNKS = prefetches_int4_chunks<Core>();
+ constexpr bool PREFETCH_PACKED_CHUNKS = prefetches_packed_chunks<Core>();
+
+ if (hoist_row_metadata) {
+ right_metadata.load(right_storage, metadata_context, first_right_group);
+ }
METAL_PRAGMA_NO_UNROLL
for (int right_group_index = 0; right_group_index < right_group_count; ++right_group_index) {
typename decltype(right_cursor)::PackedChunk packed_right_chunks[CHUNKS_PER_K_GROUP];
- if constexpr (PREFETCH_INT4_CHUNKS) {
+ if constexpr (PREFETCH_PACKED_CHUNKS) {
for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {
packed_right_chunks[chunk] = right_cursor.fetch(uint(chunk));
});
@@ -358,9 +402,11 @@ struct IntegerSchedule {
right_cursor.begin_k_group(right_group_offset);
auto load_metadata = [&]() {
left_metadata.load(left_storage, metadata_context, tile.k_offset + right_group_offset);
- right_metadata.load(right_storage, metadata_context, first_right_group + uint(right_group_index));
+ if (!hoist_row_metadata) {
+ right_metadata.load(right_storage, metadata_context, first_right_group + uint(right_group_index));
+ }
};
- if constexpr (!PREFETCH_INT4_CHUNKS) {
+ if constexpr (!PREFETCH_PACKED_CHUNKS) {
load_metadata();
}
GroupProducts<Core> group_product;
@@ -379,12 +425,24 @@ struct IntegerSchedule {
} else {
group_product = multiply_k_group<Core>(left_cursor, right_cursor, packed_right_chunks);
}
- if constexpr (PREFETCH_INT4_CHUNKS) {
+ if constexpr (PREFETCH_PACKED_CHUNKS) {
load_metadata();
}
- accumulate_group<Core, ALIGNED_M, ALIGNED_N>(accumulator, group_product, left_metadata, right_metadata);
+ accumulate_group<Core, ALIGNED_M, ALIGNED_N>(
+ accumulator,
+ group_product,
+ left_metadata,
+ right_metadata,
+ row_correction_sums,
+ hoist_row_metadata
+ );
}
+ if constexpr (HAS_BIAS) {
+ if (hoist_row_metadata) {
+ apply_row_metadata<Core, ALIGNED_N>(accumulator, right_metadata, row_correction_sums);
+ }
+ }
return accumulator;
}
};
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
index 696ad443e..ae4860bbe 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
@@ -174,6 +174,8 @@ struct SimdgroupMmaCore {
if (needs_rht) {
threadgroup_barrier(mem_flags::mem_device);
apply_output_random_hadamard_transform(
+ d,
+ params->leading_dimension_d,
d,
rht_factors_block,
bias_block,
@@ -202,9 +204,10 @@ struct SimdgroupMmaCore {
) {
const uint partition = thread_context.threadgroup_position.z;
const uint tile_row = thread_context.threadgroup_position.y;
- const uint2 tile = tile_id(uint2(thread_context.threadgroup_position.x, tile_row), params);
+ constexpr bool MAY_USE_MORTON = !Right::QUANTIZED;
+ const uint2 tile = tile_id<MAY_USE_MORTON>(uint2(thread_context.threadgroup_position.x, tile_row), params);
const auto geometry = ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::compute(tile, params);
- if (geometry.out_of_bounds) {
+ if (MAY_USE_MORTON && geometry.out_of_bounds) {
return;
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/small_batch_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/small_batch_mma_core.h
index eeb6f272b..0aa84f073 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/small_batch_mma_core.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/small_batch_mma_core.h
@@ -164,6 +164,8 @@ struct SmallBatchMmaCore {
threadgroup_barrier(mem_flags::mem_device);
apply_output_random_hadamard_transform(
+ tile_output,
+ params->leading_dimension_d,
tile_output,
rht_factors + tile_col_start,
tile_bias,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
index fca6a18f4..3ece65d3e 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -73,8 +73,8 @@ VARIANTS(
GemmBPrologueKind::ScaleBiasDequant,
GemmBPrologueKind::ScaleZeroPointDequant,
GemmBPrologueKind::ScaleSymmetricDequant)
-VARIANTS(BITS, 0, 4, 8)
-VARIANTS(GROUP_SIZE, 0, 32, 64)
+VARIANTS(BITS, 0, 2, 4, 8)
+VARIANTS(GROUP_SIZE, 0, 32, 64, 128)
VARIANTS(
A_PROLOGUE,
GemmAPrologueKind::FullPrecision,
@@ -94,7 +94,8 @@ CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))
CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))
CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleZeroPointDequant || BITS == 4)
CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant || BITS == 8)
-CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant || BITS == 4)
+CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant || BITS == 2 || BITS == 4)
+CONSTRAINT((BITS == 2) == (GROUP_SIZE == 128) && (BITS != 2 || A_IS_INT8))
CONSTRAINT(
B_PROLOGUE == GemmBPrologueKind::FullPrecision ||
(TRANSPOSE_B && GEMM_TILING != GemmTiling::Tile64x32x32_Simdgroups2x2))
@@ -151,6 +152,7 @@ KERNEL(Gemm)(
const GemmAlignment alignment SPECIALIZE,
const bool signed_codes SPECIALIZE,
const GemmBLayout gemm_b_layout SPECIALIZE,
+ const bool hoist_row_metadata SPECIALIZE,
threadgroup AT a_shared[GEMM_TGA_ELEMENTS],
threadgroup BT b_shared[GEMM_TGB_ELEMENTS],
const uint group_x GROUPS(group_count_x),
@@ -176,8 +178,15 @@ KERNEL(Gemm)(
"kernel bindings and operand correction policy must agree"
);
const auto left_storage = operands::pack_left<LeftOperand, AT>(a, a_int8, a_scales, a_group_sums, a_packed_half);
- const auto right_storage =
- operands::pack_right<RightOperand, BT>(b, scales, biases, zero_points, signed_codes, gemm_b_layout);
+ const auto right_storage = operands::pack_right<RightOperand, BT>(
+ b,
+ scales,
+ biases,
+ zero_points,
+ signed_codes,
+ hoist_row_metadata,
+ gemm_b_layout
+ );
if constexpr (USE_MXU) {
using Core = MxuMmaCore<DT, GEMM_TILING, TRANSPOSE_B, LeftOperand, RightOperand>;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal
index 547f1d416..0402a38e2 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_prepare_b_layout.metal
@@ -1,49 +1,67 @@
#include "../../common/dsl.h"
#include "../../common/defines.h"
-#include "../common/mxu_fragment/ops.h"
+#include "common/mxu_tiled_layout.h"
using namespace metal;
-#define PREPARE_B_LAYOUT_THREADS 1024u
+using uzu::matmul::MXU_TILED_PREPARE_THREADS;
+
+template <uint BITS>
+static METAL_FUNC uint codes_in_activation_order(const device uint* source, const uint column) {
+ constexpr uint BYTE_BITS = 8;
+ constexpr uint CODES_PER_BYTE = BYTE_BITS / BITS;
+ const uint source_byte = column / CODES_PER_BYTE;
+ uint packed = 0;
+ METAL_PRAGMA_UNROLL
+ for (uint source_word = 0; source_word < CODES_PER_BYTE; ++source_word) {
+ const uint codes = source[source_word] >> (BYTE_BITS * source_byte);
+ METAL_PRAGMA_UNROLL
+ for (uint code = 0; code < CODES_PER_BYTE; ++code) {
+ packed |= ((codes >> (BITS * code)) & ((1u << BITS) - 1)) << (BYTE_BITS * code + BITS * source_word);
+ }
+ }
+ return packed;
+}
template <uint BITS, uint WORDS_PER_THREAD>
-VARIANTS(BITS, 4, 8)
+VARIANTS(BITS, 2, 4, 8)
VARIANTS(WORDS_PER_THREAD, 32, 72, 160)
KERNEL(GemmPrepareBLayout)(
device uint* weights,
const constant uint& row_words,
const constant uint& row_block_count,
const uint row_block GROUPS(row_block_count),
- const uint thread_index THREADS(PREPARE_B_LAYOUT_THREADS)
+ const uint thread_index THREADS(MXU_TILED_PREPARE_THREADS)
) {
- using Ops = uzu::matmul::MxuFragmentOps<>;
- constexpr uint CHUNK_BYTES = BITS == 4 ? 16 : 32;
- constexpr uint COL_FRAGMENTS = BITS == 4 ? 1 : 2;
- constexpr uint LANE_WORDS = 2 * Ops::THREAD_ELEMENT_ROWS * COL_FRAGMENTS;
- const uint block_words = 2 * Ops::FRAGMENT_ROWS * row_words;
+ using Layout = uzu::gemm::MxuTiledLayout<BITS>;
+ using Ops = typename Layout::Ops;
+ constexpr uint ROW_SLOT_STRIDE = BITS == 2 ? 1 : Layout::COL_FRAGMENTS;
+ constexpr uint COLUMN_STRIDE = BITS == 2 ? Ops::THREAD_ELEMENT_ROWS : 1;
+ const uint block_words = Layout::BLOCK_ROWS * row_words;
device uint* block = weights + size_t(row_block) * block_words;
uint words[WORDS_PER_THREAD];
METAL_PRAGMA_UNROLL
for (uint slot = 0; slot < WORDS_PER_THREAD; ++slot) {
- const uint index = slot * PREPARE_B_LAYOUT_THREADS + thread_index;
+ const uint index = slot * MXU_TILED_PREPARE_THREADS + thread_index;
if (index < block_words) {
- const uint lane = (index / LANE_WORDS) % METAL_SIMD_SIZE;
- const uint word = index % LANE_WORDS;
- const uint fragment_row = word / (Ops::THREAD_ELEMENT_ROWS * COL_FRAGMENTS);
- const uint row_slot = word / COL_FRAGMENTS % Ops::THREAD_ELEMENT_ROWS;
- const uint fragment_column = word % COL_FRAGMENTS;
- const short2 position = Ops::get_position(ushort(lane));
+ const uint chunk = index / (METAL_SIMD_SIZE * Layout::LANE_WORDS);
+ const uint word = index % Layout::LANE_WORDS;
+ const short2 position = Ops::get_position(ushort(index / Layout::LANE_WORDS % METAL_SIMD_SIZE));
+ const uint fragment_row = word / (Ops::THREAD_ELEMENT_ROWS * Layout::COL_FRAGMENTS);
+ const uint row_slot = word / ROW_SLOT_STRIDE % Ops::THREAD_ELEMENT_ROWS;
+ const uint fragment_column = word / COLUMN_STRIDE % Layout::COL_FRAGMENTS;
const uint row = uint(position.y) + fragment_row * Ops::FRAGMENT_ROWS + row_slot * Ops::THREAD_ELEMENT_ROW_STRIDE;
- const uint byte = index / (METAL_SIMD_SIZE * LANE_WORDS) * CHUNK_BYTES + uint(position.x) +
- fragment_column * Ops::FRAGMENT_COLS;
- words[slot] = block[row * row_words + byte / 4];
+ const device uint* source = block + row * row_words +
+ (chunk * Layout::COL_FRAGMENTS + fragment_column) * Ops::FRAGMENT_COLS / sizeof(uint);
+ words[slot] = BITS == 2 ? codes_in_activation_order<BITS>(source, uint(position.x))
+ : source[uint(position.x) / sizeof(uint)];
}
}
threadgroup_barrier(mem_flags::mem_device);
METAL_PRAGMA_UNROLL
for (uint slot = 0; slot < WORDS_PER_THREAD; ++slot) {
- const uint index = slot * PREPARE_B_LAYOUT_THREADS + thread_index;
+ const uint index = slot * MXU_TILED_PREPARE_THREADS + thread_index;
if (index < block_words) {
block[index] = words[slot];
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 7736a5f23..94c1f89e4 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -7,7 +7,10 @@ use crate::{
common::{
Backend, BufferMut, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending,
Context,
- gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
+ gpu_types::{
+ gemm::{GemmBPrologueKind, GemmDTransform},
+ matmul::{MXU_TILED_BLOCK_ROWS, MXU_TILED_PREPARE_THREADS, MXU_TILED_W2_BLOCK_ROWS},
+ },
kernel::matmul::{
Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulOutput, MatmulShape, QuantParamsLayout,
QuantParamsStrides, QuantizedB,
@@ -95,17 +98,27 @@ impl GemmKernel {
specialization.alignment,
specialization.signed_codes,
specialization.gemm_b_layout,
+ specialization.hoist_row_metadata,
)?;
Ok(entry.insert(kernel))
},
}
}
+ fn mxu_tiled_block_rows(bits: u32) -> u32 {
+ if bits == 2 {
+ MXU_TILED_W2_BLOCK_ROWS
+ } else {
+ MXU_TILED_BLOCK_ROWS
+ }
+ }
+
pub fn mxu_tiled_words_per_thread(
k: u32,
bits: u32,
) -> Option<u32> {
- let words_per_thread = (k * bits / 32).div_ceil(32);
+ let words_per_thread =
+ (Self::mxu_tiled_block_rows(bits) * (k * bits / u32::BITS)).div_ceil(MXU_TILED_PREPARE_THREADS);
[32, 72, 160].into_iter().find(|&variant| variant >= words_per_thread)
}
@@ -116,13 +129,13 @@ impl GemmKernel {
bits: u32,
context: &MetalContext,
) -> Result<(), MetalError> {
- let row_words = k * bits / 32;
+ let row_words = k * bits / u32::BITS;
let variant = Self::mxu_tiled_words_per_thread(k, bits).expect("int8 eligibility admits only tileable weights");
let mut command_buffer = context.create_command_buffer(Some("prepare GEMM B layout"), None)?;
GemmPrepareBLayoutMetalKernel::new(context, bits, variant)?.encode(
weights,
row_words,
- rows / 32,
+ rows / Self::mxu_tiled_block_rows(bits),
&mut command_buffer,
);
block_on(command_buffer.end_encoding().submit().wait_until_completed())?;
@@ -259,6 +272,13 @@ impl GemmKernel {
});
}
+ if shape.b_bits == Some(2) && shape.a_is_quantized() && gemm_b_layout != GemmBLayout::MxuTiled {
+ return Err(MatmulMetalError::IncompatibleA {
+ path: "Gemm",
+ reason: "2-bit int8 GEMM requires the MxuTiled weight layout",
+ });
+ }
+
let is_quant = quantized.is_some();
if !is_quant && !matches!(&a, MatmulA::FullPrecision { .. }) {
return Err(MatmulMetalError::IncompatibleA {
@@ -266,6 +286,7 @@ impl GemmKernel {
reason: "int8 activations require quantized weights",
});
}
+ let row_scaled_weights = quantized.as_ref().is_some_and(|quantized| quantized.params.is_per_row());
let (scales, biases, zero_points, scale_strides, zero_point_strides) = match quantized.as_ref() {
None => (None, None, None, Default::default(), Default::default()),
Some(quantized) => (
@@ -336,6 +357,7 @@ impl GemmKernel {
ab_scale,
scale_strides,
zero_point_strides,
+ row_scaled_weights,
shape,
gemm_b_layout,
plan,
@@ -392,6 +414,7 @@ impl GemmKernel {
alignment,
a_prologue,
a_group_size,
+ row_scaled_weights,
)?;
let kernel = self.get_or_create(command_buffer.context(), specialization, self.output_data_type)?;
kernel.encode(
@@ -429,6 +452,7 @@ impl GemmKernel {
ab_scale: f32,
scale_strides: QuantParamsStrides,
zero_point_strides: QuantParamsStrides,
+ row_scaled_weights: bool,
shape: MatmulShape,
gemm_b_layout: GemmBLayout,
plan: GemmPlan,
@@ -487,6 +511,7 @@ impl GemmKernel {
alignment,
a_prologue,
a_group_size,
+ row_scaled_weights,
)?;
let elem = (m as usize) * (n as usize);
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
index b0bd39bbe..6ba1af16d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
@@ -23,8 +23,11 @@ const MXU_N_BUCKET_MAXES: [u32; 2] = [63, 127];
const SIMDGROUP_QUANT_NARROW_TILE_M_MAX: u32 = 16;
const SIMDGROUP_QUANT_WIDE_BLOCK_M: u32 = 64;
+const W2_TILE16_MAX_M: u32 = 64;
+
const SPLIT_K_TARGET_TILES_FP: u32 = 512;
pub(super) const SPLIT_K_TARGET_TILES_A8: u32 = 256;
+const SPLIT_K_TARGET_TILES_A8_W2: u32 = 128;
pub(super) const SPLIT_K_TARGET_TILES_TRELLIS_SIMDGROUP: u32 = 384;
pub(super) const SPLIT_K_TARGET_TILES_TRELLIS_TILE16: u32 = 768;
pub(super) const SPLIT_K_TARGET_TILES_A8_TILE16X32: u32 = 4 * SPLIT_K_TARGET_TILES_A8;
@@ -58,10 +61,13 @@ pub(super) fn mxu_mn_tile(
pub(super) fn mxu_int8_gemm_tile(
m: u32,
n: u32,
+ weight_bits: Option<u32>,
) -> GemmTiling {
match mxu_mn_tile(true, m, n) {
GemmTiling::Tile128x128x256_Simdgroups4x4 => MXU_DEFAULT_TILE,
GemmTiling::Tile16x32x256_Simdgroups1x1 => GemmTiling::Tile16x32x256_Simdgroups1x1x4,
+ tiling @ GemmTiling::Tile64x32x256_Simdgroups4x1 => tiling,
+ _ if weight_bits == Some(2) && m <= W2_TILE16_MAX_M => GemmTiling::Tile16x32x256_Simdgroups1x1x4,
tiling => tiling,
}
}
@@ -118,6 +124,7 @@ pub(super) fn split_k_target_tiles(
(true, GemmTiling::Tile32x64x256_Simdgroups2x2, Some(4)) => SPLIT_K_TARGET_TILES_A8_TILE32_W4,
(true, GemmTiling::Tile32x64x256_Simdgroups2x2, _) => SPLIT_K_TARGET_TILES_A8_TILE32_W8,
(true, GemmTiling::Tile16x32x256_Simdgroups1x1, _) => SPLIT_K_TARGET_TILES_A8_TILE16X32,
+ (true, GemmTiling::Tile16x32x256_Simdgroups1x1x4, Some(2)) => SPLIT_K_TARGET_TILES_A8_W2,
(true, GemmTiling::Tile16x32x256_Simdgroups1x1x4, Some(4)) => SPLIT_K_TARGET_TILES_A8,
(true, GemmTiling::Tile16x32x256_Simdgroups1x1x4, _) => SPLIT_K_TARGET_TILES_A8_TILE16X32_K_SPLIT,
(true, _, _) => SPLIT_K_TARGET_TILES_A8,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
index 157747cad..34370b0a4 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
@@ -271,7 +271,7 @@ fn select_tiling(
match engine {
GemmEngine::Simdgroup if shape.is_quant() => policy::simdgroup_quant_tile(shape.m),
GemmEngine::Simdgroup => GemmTiling::Tile64x32x32_Simdgroups2x2,
- GemmEngine::Mxu if shape.a_is_quantized() => match policy::mxu_int8_gemm_tile(shape.m, shape.n) {
+ GemmEngine::Mxu if shape.a_is_quantized() => match policy::mxu_int8_gemm_tile(shape.m, shape.n, shape.b_bits) {
tiling
if tiling.simdgroups_k() > 1
&& !shape.k.is_multiple_of(tiling.simdgroups_k() * ACTIVATION_SCALE_GROUP_SIZE) =>
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
index 6201cb429..5a69d25c8 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -24,6 +24,7 @@ pub(super) struct GemmSpecialization {
pub(super) b_group_size: Option<u32>,
pub(super) signed_codes: bool,
pub(super) gemm_b_layout: GemmBLayout,
+ pub(super) hoist_row_metadata: bool,
pub(super) a_group_size: Option<u32>,
}
@@ -37,6 +38,7 @@ impl GemmSpecialization {
alignment: GemmAlignment,
a_prologue: GemmAPrologueKind,
a_group_size: Option<u32>,
+ row_scaled_weights: bool,
) -> Result<Self, GemmSpecializationError> {
let specialization = Self {
weights_data_type,
@@ -51,6 +53,9 @@ impl GemmSpecialization {
b_group_size: shape.b_group_size,
signed_codes: shape.signed_codes,
gemm_b_layout,
+ hoist_row_metadata: row_scaled_weights
+ && a_prologue == GemmAPrologueKind::Int8Symmetric
+ && shape.b_prologue == GemmBPrologueKind::ScaleBiasDequant,
a_group_size,
};
specialization.validate()?;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h
index dc05c6fcf..47350e421 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_chunk.h
@@ -23,9 +23,11 @@ struct QuantChunk {
float origin,
bool signed_codes
) {
- const uint word = chunk * (BITS == 4 ? 1u : 2u);
- const uint word0 = words[word / 4u][word % 4u];
- const uint word1 = BITS == 4 ? 0u : words[(word + 1u) / 4u][(word + 1u) % 4u];
+ constexpr uint CHUNK_BITS = VALUES * BITS;
+ const uint first_bit = chunk * CHUNK_BITS;
+ const uint word = first_bit / WORD_BITS;
+ const uint word0 = words[word / WORD_LANES][word % WORD_LANES] >> (first_bit % WORD_BITS);
+ const uint word1 = CHUNK_BITS > WORD_BITS ? words[(word + 1u) / WORD_LANES][(word + 1u) % WORD_LANES] : 0u;
decode_words(word0, word1, values, origin, signed_codes);
}
@@ -64,7 +66,7 @@ private:
bool signed_codes
) {
constexpr uint CODES_PER_WORD = WORD_BITS / BITS;
- if constexpr (BITS == 4) {
+ if constexpr (BITS == 2 || BITS == 4) {
word0 ^= signed_codes ? W4_SIGN_MASK : 0u;
METAL_PRAGMA_UNROLL
for (uint i = 0; i < VALUES; i++) {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h
index c6f6a962c..a33e91ead 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/common/quant_slice.h
@@ -106,9 +106,11 @@ public:
) thread {
const uint k = position.group * GROUP_SIZE + group_offset + position.slice * SLICE_VALUES;
const uint offset_bytes = k * BITS / QuantChunk<BITS>::BITS_PER_BYTE;
- if (params.gemm_b_layout == GemmBLayout::MxuTiled) {
- load_mxu_tiled_weights(weights_base, weight_row_indices, row_stride, offset_bytes, params.gathered);
- return;
+ if constexpr (BITS == 4 || BITS == 8) {
+ if (params.gemm_b_layout == GemmBLayout::MxuTiled) {
+ load_mxu_tiled_weights(weights_base, weight_row_indices, row_stride, offset_bytes, params.gathered);
+ return;
+ }
}
Tile::for_each_output_row([&](auto output_index) UZU_ALWAYS_INLINE {
constexpr uint R = decltype(output_index)::value;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
index 041cf9a00..4293081e5 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -18,7 +18,8 @@ using namespace uzu::gemm;
(OUTPUT_ROW_TILE == 8 / K_SPLIT || OUTPUT_ROW_TILE == 32 / K_SPLIT) && INPUT_ROW_TILE == 1)
#define GEMV_QUANTIZED_ROW_TILE \
(K_SPLIT == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE == 1 && \
- ((BITS == 4 && GROUP_SIZE == 16 * GROUP_LANES) || (BITS == 8 && GROUP_SIZE == 8 * GROUP_LANES)))
+ ((BITS == 2 && GROUP_SIZE == 32 * GROUP_LANES) || (BITS == 4 && GROUP_SIZE == 16 * GROUP_LANES) || \
+ (BITS == 8 && GROUP_SIZE == 8 * GROUP_LANES)))
#define GEMV_BATCHED_FMA_32_TILE \
(K_SPLIT == 1 && REDUCTION_LANES == 8 && NUM_SIMDGROUPS == 4 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE >= 2 && \
INPUT_ROW_TILE <= 4)
@@ -50,8 +51,8 @@ VARIANTS(
GemmBPrologueKind::ScaleBiasDequant,
GemmBPrologueKind::ScaleZeroPointDequant,
GemmBPrologueKind::ScaleSymmetricDequant)
-VARIANTS(GROUP_SIZE, 0, 32, 64)
-VARIANTS(BITS, 0, 4, 8)
+VARIANTS(GROUP_SIZE, 0, 32, 64, 128)
+VARIANTS(BITS, 0, 2, 4, 8)
VARIANTS(K_SPLIT, 1, 2, 4, 8)
VARIANTS(INPUT_ALIGNED, false, true)
VARIANTS(INPUT_ROW_TILE, 1, 2, 3, 4)
@@ -65,7 +66,9 @@ CONSTRAINT(
((B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant ||
B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) &&
BITS == 4 && GROUP_SIZE != 0) ||
- (B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant && BITS == 8 && GROUP_SIZE != 0))
+ (B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant && BITS == 8 && GROUP_SIZE != 0) ||
+ (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant && BITS == 2 && GEMV_QUANTIZED_ROW_TILE))
+CONSTRAINT((BITS == 2) == (GROUP_SIZE == 128))
CONSTRAINT(INPUT_ROW_TILE == 1 || GEMV_BATCHED_INPUT)
CONSTRAINT(
GEMV_FULL_PRECISION_TILE ||
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 7c2f56b33..ef08d5026 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -26,6 +26,7 @@ use crate::{
};
const GEMV_MAX_BATCH: u32 = 8;
+const QUANT_K_BLOCK_BYTES: u32 = 256;
const MXU_TILED_KERNEL_MIN_K: u32 = 2048;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -103,15 +104,16 @@ impl GemvSpecialization {
return None;
}
let bits = shape.b_bits.unwrap_or(0);
+ if bits == 2 && gemm_b_layout == GemmBLayout::MxuTiled {
+ return None;
+ }
if !is_quant && (shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH) {
return None;
}
let block_size = if !is_quant {
FP_K_BLOCK
- } else if bits == 4 {
- 512
} else {
- 256
+ QUANT_K_BLOCK_BYTES * 8 / bits
};
let input_aligned = shape.k.is_multiple_of(block_size);
// Gathered quantized rows cannot share one input tile, and batched tiles need whole K blocks.
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
index 035b8a042..3d113d869 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
@@ -8,11 +8,7 @@ const fn lane_tile(
bits: u32,
group: u32,
) -> GemvTile {
- let pack_factor = match bits {
- 4 => 8,
- 8 => 4,
- _ => 1,
- };
+ let pack_factor = 32 / bits;
GemvTile::quantized(num_simdgroups, results_per_simdgroup, 1, 32, group / (2 * pack_factor))
}
@@ -29,7 +25,7 @@ pub fn gathered_tile(
m: u32,
n: u32,
) -> Option<GemvTile> {
- if !matches!(bits, 4 | 8) || !matches!(group, 32 | 64) || !(1..=8).contains(&m) {
+ if !matches!((bits, group), (4 | 8, 32 | 64) | (2, 128)) || !(1..=8).contains(&m) {
return None;
}
let tile = lane_default(bits, group);
@@ -44,7 +40,7 @@ pub fn select(
bf16_io: bool,
) -> Option<GemvTile> {
// The GEMV pipelines exist for BF16 I/O only. Rows 5..=8 stay on GEMV for tiny N only.
- if !bf16_io || (m > 4 && n > 64) {
+ if !bf16_io || bits == 2 || (m > 4 && n > 64) {
return None;
}
gathered_tile(bits, group, m, n)
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
index 408e9a328..ee33dbfa4 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
@@ -38,7 +38,7 @@ pub struct MatmulMetalKernel {
}
pub fn supports_integer_right_operand(shape: &MatmulShape) -> bool {
- matches!((shape.b_bits, shape.signed_codes), (Some(4), _) | (Some(8), true))
+ matches!((shape.b_bits, shape.signed_codes), (Some(2), false) | (Some(4), _) | (Some(8), true))
}
impl MatmulMetalKernel {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/mod.rs
index 82de4ac0e..9e0cf02c1 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/mod.rs
@@ -1,5 +1,8 @@
use crate::backends::{
- common::{Kernels, gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},
+ common::{
+ Kernels,
+ gpu_types::{hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE, matmul::MXU_TILED_PREPARE_THREADS},
+ },
metal::{
Metal,
buffer::MetalBufferExt,
diff --git a/crates/uzu-engine/src/backends/slang/kernel/generated/matmul.slang b/crates/uzu-engine/src/backends/slang/kernel/generated/matmul.slang
index 4084cb707..014155f40 100644
--- a/crates/uzu-engine/src/backends/slang/kernel/generated/matmul.slang
+++ b/crates/uzu-engine/src/backends/slang/kernel/generated/matmul.slang
@@ -1 +1,7 @@
-static const uint QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT = 4;
\ No newline at end of file
+static const uint QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT = 4;
+
+static const uint MXU_TILED_BLOCK_ROWS = 32;
+
+static const uint MXU_TILED_W2_BLOCK_ROWS = 16;
+
+static const uint MXU_TILED_PREPARE_THREADS = 1024;
\ No newline at end of file
diff --git a/crates/uzu-engine/src/backends/slang/kernel/generated/quantization.slang b/crates/uzu-engine/src/backends/slang/kernel/generated/quantization.slang
index 165712637..8512af9ae 100644
--- a/crates/uzu-engine/src/backends/slang/kernel/generated/quantization.slang
+++ b/crates/uzu-engine/src/backends/slang/kernel/generated/quantization.slang
@@ -2,4 +2,5 @@ enum QuantizationMode : uint32_t {
U4 = 0,
I8 = 1,
U8 = 2,
+ U2 = 3,
};
\ No newline at end of file
diff --git a/crates/uzu-engine/src/data_type.rs b/crates/uzu-engine/src/data_type.rs
index f58175327..b4efe91a5 100644
--- a/crates/uzu-engine/src/data_type.rs
+++ b/crates/uzu-engine/src/data_type.rs
@@ -10,6 +10,7 @@ pub enum DataType {
F16,
F32,
F64,
+ U2,
I4,
U4,
I8,
@@ -25,6 +26,7 @@ pub enum DataType {
impl DataType {
pub const fn size_in_bits(&self) -> usize {
match self {
+ DataType::U2 => 2,
DataType::I4 | DataType::U4 => 4,
DataType::I8 | DataType::U8 => 8,
DataType::I16 | DataType::U16 => 16,
diff --git a/crates/uzu-engine/src/encodable_block/weight_matrix.rs b/crates/uzu-engine/src/encodable_block/weight_matrix.rs
index 43378b46c..3b22ef13c 100644
--- a/crates/uzu-engine/src/encodable_block/weight_matrix.rs
+++ b/crates/uzu-engine/src/encodable_block/weight_matrix.rs
@@ -5,7 +5,10 @@ use crate::{
backends::common::{
Backend,
gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},
- kernel::matmul::{MatmulB, QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection, WeightsLayout},
+ kernel::{
+ activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
+ matmul::{MatmulB, QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection, WeightsLayout},
+ },
},
config::weight_matrix::{AnyWeightMatrixSpec, Layout},
data_type::DataType,
@@ -56,6 +59,7 @@ pub fn parse_spec<B: Backend>(spec: &AnyWeightMatrixSpec) -> Result<ParsedWeight
None => None,
Some((bits, group_size, method)) => {
let mode = match bits {
+ 2 => QuantizationMode::U2,
4 => QuantizationMode::U4,
8 => QuantizationMode::U8,
_ => {
@@ -114,7 +118,7 @@ impl<B: Backend> WeightMatrix<B> {
}
let (rows, columns) = physical_shape(&layout, output_dim, input_dim);
- let Some(info) = quantization_info else {
+ let Some(mut info) = quantization_info else {
let values = tree.read_buffer("weights", &[rows, columns], data_type).await?;
return Ok(Self {
values,
@@ -136,6 +140,9 @@ impl<B: Backend> WeightMatrix<B> {
)));
}
let groups = columns.div_ceil(group_size);
+ if info.mode == QuantizationMode::U2 {
+ info.group_size = ACTIVATION_SCALE_GROUP_SIZE;
+ }
let values = tree.read_buffer("weights", &[rows, columns / packing_divisor], storage_data_type).await?;
let params = QuantParams::new(params_layout, rows, groups);
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
index bb707a29f..9a9d38948 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
@@ -463,6 +463,9 @@ fn mxu_quant_parity_bf16(
#[case::w4_m128_bias(128u32, 512u32, 256u32, 64u32, 4u32, QuantizationMethod::ScaleBias)]
#[case::w4_m16_k17408(16u32, 17408u32, 64u32, 64u32, 4u32, QuantizationMethod::ScaleZeroPoint)]
#[case::w8_m16_k17408(16u32, 17408u32, 64u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]
+#[case::w2_m1(1u32, 4096u32, 1024u32, 128u32, 2u32, QuantizationMethod::ScaleBias)]
+#[case::w2_m16(16u32, 4096u32, 1024u32, 128u32, 2u32, QuantizationMethod::ScaleBias)]
+#[case::w2_m65(65u32, 4096u32, 160u32, 128u32, 2u32, QuantizationMethod::ScaleBias)]
fn a8w_mxu_parity_bf16(
#[case] m: u32,
#[case] k: u32,
@@ -473,14 +476,19 @@ fn a8w_mxu_parity_bf16(
#[values(GemmBLayout::RowMajor, GemmBLayout::MxuTiled)] gemm_b_layout: GemmBLayout,
) {
let context = block_on(MetalContext::new()).expect("Metal context");
- if !context.supports_mxu {
+ if !context.supports_mxu || (bits == 2 && gemm_b_layout == GemmBLayout::RowMajor) {
return;
}
- let (input, reference_input) = QuantInput::<bf16>::new(m, k, n, weight_gs, bits, method, 0)
- .with_prepared_a_and_reference(
- ACTIVATION_SCALE_GROUP_SIZE,
- (method != QuantizationMethod::ScaleSymmetric).then_some(weight_gs),
- );
+ let input = QuantInput::<bf16>::new(m, k, n, weight_gs, bits, method, 0);
+ let input = if bits == 2 {
+ input.with_one_group_per_row()
+ } else {
+ input
+ };
+ let (input, reference_input) = input.with_prepared_a_and_reference(
+ ACTIVATION_SCALE_GROUP_SIZE,
+ (method != QuantizationMethod::ScaleSymmetric).then_some(weight_gs),
+ );
let actual = run_quant_metal(&context, &input.with_group_output(), gemm_b_layout, Some(GemmEngine::Mxu));
let reference = run_quant_cpu::<bf16>(&reference_input);
assert_parity::<bf16>(
@@ -690,6 +698,7 @@ fn gemv_reads_mxu_tiled_layout_bf16(
#[test_attr(uzu_test)]
#[case::w4_zp_gs32(4u32, 32u32, ScaleZeroPoint)]
#[case::w8_sym_gs64(8u32, 64u32, QuantizationMethod::ScaleSymmetric)]
+#[case::w2_bias_gs128(2u32, 128u32, QuantizationMethod::ScaleBias)]
fn gathered_gemv_reads_mxu_tiled_layout_bf16(
#[case] bits: u32,
#[case] group_size: u32,
@@ -701,10 +710,10 @@ fn gathered_gemv_reads_mxu_tiled_layout_bf16(
}
let (k, vocab, gathered_rows) = (256u32, 256u32, 10u32);
let input = QuantInput::<bf16>::new(1, k, vocab, group_size, bits, method, 0).with_group_output();
- let input = if bits == 8 {
- input.with_signed_weight_codes()
- } else {
- input
+ let input = match bits {
+ 2 => input.with_one_group_per_row(),
+ 8 => input.with_signed_weight_codes(),
+ _ => input,
};
let ids: Vec<u32> = (0..gathered_rows).map(|row| (row * 37 + 11) % vocab).collect();
let dense = run_quant_cpu::<bf16>(&input);
@@ -719,7 +728,9 @@ fn gathered_gemv_reads_mxu_tiled_layout_bf16(
bf16::data_type(),
)
.expect("Metal matmul kernel");
- tile_weights(&matmul, &mut buffers, &input, &context);
+ if bits != 2 {
+ tile_weights(&matmul, &mut buffers, &input, &context);
+ }
let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");
let mut arguments = quant_arguments(&mut buffers, &input);
arguments.gather_indices = Some(&ids_buffer);
diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rs
index 88984a91e..cf6d30a2d 100644
--- a/crates/uzu-engine/unit/common/matmul/quant.rs
+++ b/crates/uzu-engine/unit/common/matmul/quant.rs
@@ -51,10 +51,12 @@ pub struct QuantInput<T: DataTypeRepr + Float> {
pub params_layout: QuantParamsLayout,
pub signed_codes: bool,
pub prepared_a: Option<PreparedInt8A>,
+ pub one_group_per_row: bool,
}
fn mode_for_bits(bits: u32) -> QuantizationMode {
match bits {
+ 2 => QuantizationMode::U2,
4 => QuantizationMode::U4,
8 => QuantizationMode::U8,
_ => unreachable!("unsupported bits: {bits}"),
@@ -182,6 +184,7 @@ impl<T: DataTypeRepr + Float> QuantInput<T> {
params_layout: QuantParamsLayout::OutputGroup,
signed_codes: false,
prepared_a: None,
+ one_group_per_row: false,
}
}
@@ -190,6 +193,23 @@ impl<T: DataTypeRepr + Float> QuantInput<T> {
self
}
+ pub fn with_one_group_per_row(mut self) -> Self {
+ self.one_group_per_row = true;
+ self.scales.truncate(self.n as usize);
+ if let Some(biases) = &mut self.biases {
+ biases.truncate(self.n as usize);
+ }
+ self
+ }
+
+ fn group_count(&self) -> u32 {
+ if self.one_group_per_row {
+ 1
+ } else {
+ self.k.div_ceil(self.group_size)
+ }
+ }
+
pub fn with_group_output(mut self) -> Self {
self.params_layout = QuantParamsLayout::GroupOutput;
self
@@ -227,7 +247,7 @@ impl<T: DataTypeRepr + Float> QuantInput<T> {
sum_group_size: Option<u32>,
code_layout: Int8CodeLayout,
) -> Self {
- self.signed_codes = self.mode != QuantizationMode::U4;
+ self.signed_codes = self.mode == QuantizationMode::U8;
let rows = self.m;
let columns = self.k;
assert!(columns.is_multiple_of(activation_scale_group_size));
@@ -313,7 +333,7 @@ impl<B: Backend, T: DataTypeRepr + Float> QuantBuffers<B, T> {
context: &B::Context,
input: &QuantInput<T>,
) -> Self {
- let groups = input.k.div_ceil(input.group_size);
+ let groups = input.group_count();
let params_layout = input.params_layout;
let params = QuantParams::new(params_layout, input.n, groups);
let metadata_elements = params.scale_shape().into_iter().product::<u32>() as usize;
@@ -386,7 +406,7 @@ fn quant_b_variant<TB: BufferRef, T: DataTypeRepr + Float>(
weights_layout: WeightsLayout<TB::Backend>,
input: &QuantInput<T>,
) -> MatmulB<TB> {
- let params = QuantParams::new(params_layout, input.n, input.k.div_ceil(input.group_size));
+ let params = QuantParams::new(params_layout, input.n, input.group_count());
let correction = match input.quant_method {
QuantizationMethod::ScaleBias => QuantizedCorrection::Biases(biases.expect("bias buffer")),
QuantizationMethod::ScaleZeroPoint => QuantizedCorrection::ZeroPoints(zero_points.expect("zp buffer")),