Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Pre-M5 GEMM: aligned A copy, half-precision dequant (#1068)

CCYehe7e7523cce14
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
index 15492da5f..9480e6012 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
@@ -84,34 +84,47 @@ METAL_FUNC float magic_code_float(uint code) {
METAL_FUNC float magic_dequant_bias(float scale) { return -12582912.0f * scale; }
-// fma(code, scale, bias) in f32, rounded once to U, stored in pairs.
-template <typename U, int N, int bits>
+template <typename StageT, typename ComputeT, int N, int bits>
inline void dequantize(
const device uint8_t* w,
float scale,
float bias,
- threadgroup U* w_local,
+ threadgroup StageT* w_local,
const bool signed_codes
) {
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
static_assert(bits == 8 || N % 2 == 0, "int4 values are unpacked in pairs");
- threadgroup vec<U, 2>* pairs = reinterpret_cast<threadgroup vec<U, 2>*>(w_local);
- if constexpr (bits == 4) {
+ constexpr bool IS_16_BIT_COMPUTE = sizeof(ComputeT) == 2;
+ constexpr bool READ_CODE_WORDS = bits == 4 && N % 8 == 0 && IS_16_BIT_COMPUTE;
+ threadgroup vec<StageT, 2>* pairs = reinterpret_cast<threadgroup vec<StageT, 2>*>(w_local);
+ const ComputeT compute_scale = ComputeT(scale);
+ const ComputeT compute_bias = ComputeT(bias);
+ auto dequantize_code = [&](uint code) { return fma(ComputeT(code), compute_scale, compute_bias); };
+ if constexpr (READ_CODE_WORDS) {
+ const device uint* words = reinterpret_cast<const device uint*>(w);
+ for (int i = 0; i < N / 8; i++) {
+ const uint word = signed_codes ? words[i] ^ W4_SIGN_MASK : words[i];
+ for (int j = 0; j < 4; j++) {
+ pairs[i * 4 + j] = vec<StageT, 2>(
+ vec<ComputeT, 2>(dequantize_code((word >> (8 * j)) & 0x0fu), dequantize_code((word >> (8 * j + 4)) & 0x0fu))
+ );
+ }
+ }
+ } else if constexpr (bits == 4) {
const uint sign_flip = signed_codes ? 0x88u : 0u;
for (int i = 0; i < (N / 2); i++) {
const uint code = uint(w[i]) ^ sign_flip;
- pairs[i] = vec<U, 2>(vec<float, 2>(fma(float(code & 0x0fu), scale, bias), fma(float(code >> 4), scale, bias)));
+ pairs[i] = vec<StageT, 2>(vec<ComputeT, 2>(dequantize_code(code & 0x0fu), dequantize_code(code >> 4)));
}
} else {
const uint sign_flip = signed_codes ? 0x80u : 0u;
for (int i = 0; i < (N / 2); i++) {
- pairs[i] = vec<U, 2>(vec<float, 2>(
- fma(float(uint(w[2 * i]) ^ sign_flip), scale, bias),
- fma(float(uint(w[2 * i + 1]) ^ sign_flip), scale, bias)
- ));
+ pairs[i] = vec<StageT, 2>(
+ vec<ComputeT, 2>(dequantize_code(uint(w[2 * i]) ^ sign_flip), dequantize_code(uint(w[2 * i + 1]) ^ sign_flip))
+ );
}
if constexpr (N % 2 == 1) {
- w_local[N - 1] = U(fma(float(uint(w[N - 1]) ^ sign_flip), scale, bias));
+ w_local[N - 1] = StageT(fma(float(uint(w[N - 1]) ^ sign_flip), scale, bias));
}
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index 0e300983c..2776903b2 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -35,6 +35,7 @@ struct MxuMmaCore {
using LeftElementType = typename Left::ElementType;
using RightElementType = typename Right::ElementType;
using RightStageType = RightElementType;
+ using RightComputeType = float;
using LeftStorage = operands::LeftStorage<Left>;
using RightStorage = operands::RightStorage<Right>;
using FragmentOps = uzu::matmul::MxuFragmentOps<>;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
index ff5ebaa18..c15d00022 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
@@ -20,7 +20,8 @@ template <
short THREADGROUP_SIZE,
short GROUP_SIZE,
short BITS,
- typename StageT = T>
+ typename StageT,
+ typename ComputeT>
struct QuantizedBlockLoaderScaleBias {
static_assert(THREADGROUP_TILE_COLS <= GROUP_SIZE, "Group size should be larger than columns");
static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");
@@ -83,7 +84,13 @@ struct QuantizedBlockLoaderScaleBias {
}
}
- dequantize<StageT, READS_PER_THREAD * PACK_FACTOR, BITS>(src, float(*scales), float(*biases), dst, signed_codes);
+ dequantize<StageT, ComputeT, READS_PER_THREAD * PACK_FACTOR, BITS>(
+ src,
+ float(*scales),
+ float(*biases),
+ dst,
+ signed_codes
+ );
}
void load_safe(short2 src_tile_dim) const {
@@ -107,7 +114,7 @@ struct QuantizedBlockLoaderScaleBias {
for (int i = 0; i < READS_PER_THREAD; i++) {
const int pack_index = tile_col_index + i;
if (pack_index < valid_packs) {
- dequantize<StageT, PACK_FACTOR, BITS>(
+ dequantize<StageT, ComputeT, PACK_FACTOR, BITS>(
src + i * BYTES_PER_PACK,
scale,
bias,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
index 87f7fe985..3ab8d9b59 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
@@ -20,8 +20,9 @@ template <
short THREADGROUP_SIZE,
short GROUP_SIZE,
short BITS,
- bool SCALE_SYMMETRIC = false,
- typename StageT = T>
+ bool SCALE_SYMMETRIC,
+ typename StageT,
+ typename ComputeT>
struct QuantizedBlockLoaderScaleZeroPoint {
static_assert(THREADGROUP_TILE_COLS <= GROUP_SIZE, "Group size should be larger than columns");
static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");
@@ -138,7 +139,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {
float scale;
float bias;
current_scale_bias(scale, bias);
- dequantize<StageT, READS_PER_THREAD * PACK_FACTOR, BITS>(src, scale, bias, dst, signed_codes);
+ dequantize<StageT, ComputeT, READS_PER_THREAD * PACK_FACTOR, BITS>(src, scale, bias, dst, signed_codes);
}
void load_safe(short2 src_tile_dim) const {
@@ -163,7 +164,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {
for (int i = 0; i < READS_PER_THREAD; i++) {
const int pack_index = tile_col_index + i;
if (pack_index < valid_packs) {
- dequantize<StageT, PACK_FACTOR, BITS>(
+ dequantize<StageT, ComputeT, PACK_FACTOR, BITS>(
src + i * BYTES_PER_PACK,
scale,
bias,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h
index 2fdcd57c4..5d6d97d22 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h
@@ -46,7 +46,8 @@ static METAL_FUNC auto make_staged_loader(
Core::THREADGROUP_THREADS,
RightOperand::GROUP_SIZE,
RightOperand::BITS,
- typename Core::RightStageType>;
+ typename Core::RightStageType,
+ typename Core::RightComputeType>;
return Loader(
values,
scales,
@@ -69,7 +70,8 @@ static METAL_FUNC auto make_staged_loader(
RightOperand::GROUP_SIZE,
RightOperand::BITS,
false,
- typename Core::RightStageType>;
+ typename Core::RightStageType,
+ typename Core::RightComputeType>;
return Loader(
values,
scales,
@@ -99,7 +101,8 @@ static METAL_FUNC auto make_staged_loader(
RightOperand::GROUP_SIZE,
RightOperand::BITS,
true,
- typename Core::RightStageType>;
+ typename Core::RightStageType,
+ typename Core::RightComputeType>;
return Loader(
values,
scales,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
index 889c931bc..022ee3501 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
@@ -34,8 +34,8 @@ struct SimdgroupMmaCore {
using Right = RightOperand;
using LeftElementType = typename Left::ElementType;
using RightElementType = typename Right::ElementType;
- // Quantized weights are dequantized in f32 and staged as half, the fast pre-M5 MMA operand.
using RightStageType = conditional_t<Right::QUANTIZED, half, RightElementType>;
+ using RightComputeType = half;
static_assert(sizeof(RightStageType) == sizeof(RightElementType), "staging memory is sized for the weight type");
using LeftStorage = operands::LeftStorage<Left>;
using RightStorage = operands::RightStorage<Right>;
@@ -52,13 +52,19 @@ struct SimdgroupMmaCore {
UZU_CONST ushort SHARED_STRIDE_B = (TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;
UZU_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
+ UZU_CONST ushort A_READS_PER_THREAD = THREADGROUP_BLOCK_M * THREADGROUP_BLOCK_K / THREADGROUP_THREADS;
+ UZU_CONST ushort A_VECTOR_COPY_ALIGNMENT =
+ !Right::QUANTIZED
+ ? 1
+ : (A_READS_PER_THREAD * sizeof(LeftElementType) < 16 ? A_READS_PER_THREAD : 16 / sizeof(LeftElementType));
using ALoader = uzu::matmul::ThreadgroupLoader<
LeftElementType,
THREADGROUP_BLOCK_M,
THREADGROUP_BLOCK_K,
SHARED_STRIDE_A,
true,
- THREADGROUP_THREADS>;
+ THREADGROUP_THREADS,
+ A_VECTOR_COPY_ALIGNMENT>;
using FullPrecisionRightLoader = uzu::matmul::ThreadgroupLoader<
RightElementType,
TRANSPOSE_B ? THREADGROUP_BLOCK_N : THREADGROUP_BLOCK_K,