LLM inference in C/C++
C++ ggml
ggml-cpu : add AVX2 dot product for Q2_0 (#30447)
homeofe0a731cced9ed
diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.hindex e4882984a2..4e59d0ae0a 100644--- a/ggml/src/ggml-cpu/arch-fallback.h+++ b/ggml/src/ggml-cpu/arch-fallback.h@@ -86,8 +86,6 @@#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K#elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)-// quants.c-#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0// repack.cpp#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.cindex ea54cfe44c..21cd6312a1 100644--- a/ggml/src/ggml-cpu/arch/x86/quants.c+++ b/ggml/src/ggml-cpu/arch/x86/quants.c@@ -698,6 +698,70 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi#endif}+#if defined(__AVX2__)+// four consecutive Q2_0 weights per byte: byte b becomes b & 3, (b >> 2) & 3, (b >> 4) & 3, (b >> 6) & 3 (OR, never add: the overlapping copies only touch masked-off bits)+static inline __m256i q2_0_unpack_32_avx2(const uint8_t * qs) {+ __m256i v = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *) qs));+ v = _mm256_or_si256(v, _mm256_slli_epi32(v, 12));+ v = _mm256_or_si256(v, _mm256_slli_epi32(v, 6));+ return _mm256_and_si256(v, _mm256_set1_epi32(0x03030303));+}++// 32 weights against one Q8_0 block: sum((v - 1) * y) = sum(v * y) - sum(y), exact for every int8 y+static inline __m256 q2_0_dot_32_avx2(const uint8_t * qs, const int8_t * qy) {+ const __m256i v = q2_0_unpack_32_avx2(qs);+ const __m256i y = _mm256_loadu_si256((const __m256i *) qy);+ const __m256i p = _mm256_sub_epi16(_mm256_maddubs_epi16(v, y), _mm256_maddubs_epi16(_mm256_set1_epi8(1), y));+ return _mm256_cvtepi32_ps(_mm256_madd_epi16(p, _mm256_set1_epi16(1)));+}++static inline __m256 q2_0_acc_block_avx2(const block_q2_0 * x, const block_q8_0 * y, __m256 acc) {+ const float dx = GGML_CPU_FP16_TO_FP32(x->d);+ const __m256 d0 = _mm256_set1_ps(dx * GGML_CPU_FP16_TO_FP32(y[0].d));+ const __m256 d1 = _mm256_set1_ps(dx * GGML_CPU_FP16_TO_FP32(y[1].d));+ const __m256 p0 = q2_0_dot_32_avx2(x->qs + 0, y[0].qs);+ const __m256 p1 = q2_0_dot_32_avx2(x->qs + 8, y[1].qs);+#if defined(__FMA__)+ acc = _mm256_fmadd_ps(d0, p0, acc);+ return _mm256_fmadd_ps(d1, p1, acc);+#else+ acc = _mm256_add_ps(acc, _mm256_mul_ps(d0, p0));+ return _mm256_add_ps(acc, _mm256_mul_ps(d1, p1));+#endif+}+#endif++void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {+#if defined(__AVX2__)+ assert(n % QK2_0 == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q2_0 * GGML_RESTRICT x = vx;+ const block_q8_0 * GGML_RESTRICT y = vy;+ const int nb = n / QK2_0;++ // two Q2_0 blocks per step, each into its own accumulator (two FMA chains)+ __m256 acc0 = _mm256_setzero_ps();+ __m256 acc1 = _mm256_setzero_ps();+ int i = 0;+ for (; i + 1 < nb; i += 2) {+ acc0 = q2_0_acc_block_avx2(x + i + 0, y + 2*i + 0, acc0);+ acc1 = q2_0_acc_block_avx2(x + i + 1, y + 2*i + 2, acc1);+ }+ if (i < nb) {+ acc0 = q2_0_acc_block_avx2(x + i, y + 2*i, acc0);+ }++ *s = hsum_float_8(_mm256_add_ps(acc0, acc1));+#else+ ggml_vec_dot_q2_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);+#endif+}+void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {const int qk = QK8_0;const int nb = n / qk;