A high-performance inference engine for AI models
Pre-M5 GEMM: aligned A copy, half-precision dequant (#1068)
CCYehe7e7523cce14
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.hindex 15492da5f..9480e6012 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h@@ -84,34 +84,47 @@ METAL_FUNC float magic_code_float(uint code) {METAL_FUNC float magic_dequant_bias(float scale) { return -12582912.0f * scale; }-// fma(code, scale, bias) in f32, rounded once to U, stored in pairs.-template <typename U, int N, int bits>+template <typename StageT, typename ComputeT, int N, int bits>inline void dequantize(const device uint8_t* w,float scale,float bias,- threadgroup U* w_local,+ threadgroup StageT* w_local,const bool signed_codes) {static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");static_assert(bits == 8 || N % 2 == 0, "int4 values are unpacked in pairs");- threadgroup vec<U, 2>* pairs = reinterpret_cast<threadgroup vec<U, 2>*>(w_local);- if constexpr (bits == 4) {+ constexpr bool IS_16_BIT_COMPUTE = sizeof(ComputeT) == 2;+ constexpr bool READ_CODE_WORDS = bits == 4 && N % 8 == 0 && IS_16_BIT_COMPUTE;+ threadgroup vec<StageT, 2>* pairs = reinterpret_cast<threadgroup vec<StageT, 2>*>(w_local);+ const ComputeT compute_scale = ComputeT(scale);+ const ComputeT compute_bias = ComputeT(bias);+ auto dequantize_code = [&](uint code) { return fma(ComputeT(code), compute_scale, compute_bias); };+ if constexpr (READ_CODE_WORDS) {+ const device uint* words = reinterpret_cast<const device uint*>(w);+ for (int i = 0; i < N / 8; i++) {+ const uint word = signed_codes ? words[i] ^ W4_SIGN_MASK : words[i];+ for (int j = 0; j < 4; j++) {+ pairs[i * 4 + j] = vec<StageT, 2>(+ vec<ComputeT, 2>(dequantize_code((word >> (8 * j)) & 0x0fu), dequantize_code((word >> (8 * j + 4)) & 0x0fu))+ );+ }+ }+ } else if constexpr (bits == 4) {const uint sign_flip = signed_codes ? 0x88u : 0u;for (int i = 0; i < (N / 2); i++) {const uint code = uint(w[i]) ^ sign_flip;- pairs[i] = vec<U, 2>(vec<float, 2>(fma(float(code & 0x0fu), scale, bias), fma(float(code >> 4), scale, bias)));+ pairs[i] = vec<StageT, 2>(vec<ComputeT, 2>(dequantize_code(code & 0x0fu), dequantize_code(code >> 4)));}} else {const uint sign_flip = signed_codes ? 0x80u : 0u;for (int i = 0; i < (N / 2); i++) {- pairs[i] = vec<U, 2>(vec<float, 2>(- fma(float(uint(w[2 * i]) ^ sign_flip), scale, bias),- fma(float(uint(w[2 * i + 1]) ^ sign_flip), scale, bias)- ));+ pairs[i] = vec<StageT, 2>(+ vec<ComputeT, 2>(dequantize_code(uint(w[2 * i]) ^ sign_flip), dequantize_code(uint(w[2 * i + 1]) ^ sign_flip))+ );}if constexpr (N % 2 == 1) {- w_local[N - 1] = U(fma(float(uint(w[N - 1]) ^ sign_flip), scale, bias));+ w_local[N - 1] = StageT(fma(float(uint(w[N - 1]) ^ sign_flip), scale, bias));}}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 0e300983c..2776903b2 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -35,6 +35,7 @@ struct MxuMmaCore {using LeftElementType = typename Left::ElementType;using RightElementType = typename Right::ElementType;using RightStageType = RightElementType;+ using RightComputeType = float;using LeftStorage = operands::LeftStorage<Left>;using RightStorage = operands::RightStorage<Right>;using FragmentOps = uzu::matmul::MxuFragmentOps<>;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.hindex ff5ebaa18..c15d00022 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h@@ -20,7 +20,8 @@ template <short THREADGROUP_SIZE,short GROUP_SIZE,short BITS,- typename StageT = T>+ typename StageT,+ typename ComputeT>struct QuantizedBlockLoaderScaleBias {static_assert(THREADGROUP_TILE_COLS <= GROUP_SIZE, "Group size should be larger than columns");static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");@@ -83,7 +84,13 @@ struct QuantizedBlockLoaderScaleBias {}}- dequantize<StageT, READS_PER_THREAD * PACK_FACTOR, BITS>(src, float(*scales), float(*biases), dst, signed_codes);+ dequantize<StageT, ComputeT, READS_PER_THREAD * PACK_FACTOR, BITS>(+ src,+ float(*scales),+ float(*biases),+ dst,+ signed_codes+ );}void load_safe(short2 src_tile_dim) const {@@ -107,7 +114,7 @@ struct QuantizedBlockLoaderScaleBias {for (int i = 0; i < READS_PER_THREAD; i++) {const int pack_index = tile_col_index + i;if (pack_index < valid_packs) {- dequantize<StageT, PACK_FACTOR, BITS>(+ dequantize<StageT, ComputeT, PACK_FACTOR, BITS>(src + i * BYTES_PER_PACK,scale,bias,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.hindex 87f7fe985..3ab8d9b59 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h@@ -20,8 +20,9 @@ template <short THREADGROUP_SIZE,short GROUP_SIZE,short BITS,- bool SCALE_SYMMETRIC = false,- typename StageT = T>+ bool SCALE_SYMMETRIC,+ typename StageT,+ typename ComputeT>struct QuantizedBlockLoaderScaleZeroPoint {static_assert(THREADGROUP_TILE_COLS <= GROUP_SIZE, "Group size should be larger than columns");static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");@@ -138,7 +139,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {float scale;float bias;current_scale_bias(scale, bias);- dequantize<StageT, READS_PER_THREAD * PACK_FACTOR, BITS>(src, scale, bias, dst, signed_codes);+ dequantize<StageT, ComputeT, READS_PER_THREAD * PACK_FACTOR, BITS>(src, scale, bias, dst, signed_codes);}void load_safe(short2 src_tile_dim) const {@@ -163,7 +164,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {for (int i = 0; i < READS_PER_THREAD; i++) {const int pack_index = tile_col_index + i;if (pack_index < valid_packs) {- dequantize<StageT, PACK_FACTOR, BITS>(+ dequantize<StageT, ComputeT, PACK_FACTOR, BITS>(src + i * BYTES_PER_PACK,scale,bias,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.hindex 2fdcd57c4..5d6d97d22 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h@@ -46,7 +46,8 @@ static METAL_FUNC auto make_staged_loader(Core::THREADGROUP_THREADS,RightOperand::GROUP_SIZE,RightOperand::BITS,- typename Core::RightStageType>;+ typename Core::RightStageType,+ typename Core::RightComputeType>;return Loader(values,scales,@@ -69,7 +70,8 @@ static METAL_FUNC auto make_staged_loader(RightOperand::GROUP_SIZE,RightOperand::BITS,false,- typename Core::RightStageType>;+ typename Core::RightStageType,+ typename Core::RightComputeType>;return Loader(values,scales,@@ -99,7 +101,8 @@ static METAL_FUNC auto make_staged_loader(RightOperand::GROUP_SIZE,RightOperand::BITS,true,- typename Core::RightStageType>;+ typename Core::RightStageType,+ typename Core::RightComputeType>;return Loader(values,scales,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hindex 889c931bc..022ee3501 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -34,8 +34,8 @@ struct SimdgroupMmaCore {using Right = RightOperand;using LeftElementType = typename Left::ElementType;using RightElementType = typename Right::ElementType;- // Quantized weights are dequantized in f32 and staged as half, the fast pre-M5 MMA operand.using RightStageType = conditional_t<Right::QUANTIZED, half, RightElementType>;+ using RightComputeType = half;static_assert(sizeof(RightStageType) == sizeof(RightElementType), "staging memory is sized for the weight type");using LeftStorage = operands::LeftStorage<Left>;using RightStorage = operands::RightStorage<Right>;@@ -52,13 +52,19 @@ struct SimdgroupMmaCore {UZU_CONST ushort SHARED_STRIDE_B = (TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;UZU_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;+ UZU_CONST ushort A_READS_PER_THREAD = THREADGROUP_BLOCK_M * THREADGROUP_BLOCK_K / THREADGROUP_THREADS;+ UZU_CONST ushort A_VECTOR_COPY_ALIGNMENT =+ !Right::QUANTIZED+ ? 1+ : (A_READS_PER_THREAD * sizeof(LeftElementType) < 16 ? A_READS_PER_THREAD : 16 / sizeof(LeftElementType));using ALoader = uzu::matmul::ThreadgroupLoader<LeftElementType,THREADGROUP_BLOCK_M,THREADGROUP_BLOCK_K,SHARED_STRIDE_A,true,- THREADGROUP_THREADS>;+ THREADGROUP_THREADS,+ A_VECTOR_COPY_ALIGNMENT>;using FullPrecisionRightLoader = uzu::matmul::ThreadgroupLoader<RightElementType,TRANSPOSE_B ? THREADGROUP_BLOCK_N : THREADGROUP_BLOCK_K,