LLM inference in C/C++
C++ ggml
ggml-cpu: ARM Repack kernels for Q1_0 (#23492)
pl7528034c1d1f166
diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.hindex 2b9a426577..4dbd1982ba 100644--- a/ggml/src/ggml-cpu/arch-fallback.h+++ b/ggml/src/ggml-cpu/arch-fallback.h@@ -39,6 +39,8 @@#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8+#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0+#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0@@ -55,6 +57,8 @@#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0+#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0+#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0@@ -87,6 +91,8 @@// repack.cpp#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4+#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0+#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K@@ -98,6 +104,8 @@#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0+#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0+#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K@@ -124,6 +132,8 @@#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8+#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0+#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0@@ -140,6 +150,8 @@#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0+#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0+#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0@@ -171,6 +183,8 @@#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8+#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0+#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0@@ -187,6 +201,8 @@#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0+#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0+#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0@@ -213,6 +229,8 @@#define ggml_quantize_mat_q8_K_4x1_generic ggml_quantize_mat_q8_K_4x1#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8+#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0+#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K@@ -228,6 +246,8 @@#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0+#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0+#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K@@ -262,6 +282,8 @@#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8+#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0+#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K@@ -277,6 +299,8 @@#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0+#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0+#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K@@ -314,6 +338,8 @@#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8+#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0+#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0@@ -330,6 +356,8 @@#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0+#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0+#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cppindex a753444309..ad0e5ccaf7 100644--- a/ggml/src/ggml-cpu/arch/arm/repack.cpp+++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp@@ -48,6 +48,24 @@ static inline void decode_q_Kx8_6bit_scales(const uint8_t * scales_in, int16x8_t}#endif+#if defined(__aarch64__) && defined(__ARM_NEON) && (defined(__ARM_FEATURE_DOTPROD) || defined(__ARM_FEATURE_MATMUL_INT8))+#define B1(c,s,n) 0x ## n ## c , 0x ## n ## s+#define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s)+#define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s)+#define B4(c,s,n) B3(c,s,n ## c), B3(c,s,n ## s)+#define B5(c,s,n) B4(c,s,n ## c), B4(c,s,n ## s)+#define B6(c,s,n) B5(c,s,n ## c), B5(c,s,n ## s)+#define B7(c,s,n) B6(c,s,n ## c), B6(c,s,n ## s)+#define B8(c,s ) B7(c,s, c), B7(c,s, s)++static const uint64_t table_q1_signs[256] = { B8(ff, 01) };++static inline int8x16_t ggml_q1_0_unpack_pair(uint8_t bits0, uint8_t bits1) {+ return vreinterpretq_s8_u8(vcombine_u8(vcreate_u8(table_q1_signs[bits0]),+ vcreate_u8(table_q1_signs[bits1])));+}+#endif+void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {assert(QK8_0 == 32);assert(k % QK8_0 == 0);@@ -1823,6 +1841,132 @@ void ggml_gemv_q8_0_4x8_q8_0(int n,ggml_gemv_q8_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);}+void ggml_gemv_q1_0_4x4_q8_0(int n,+ float * GGML_RESTRICT s,+ size_t bs,+ const void * GGML_RESTRICT vx,+ const void * GGML_RESTRICT vy,+ int nr,+ int nc) {+ const int qk = QK1_0;+ const int nb = n / qk;+ const int ncols_interleaved = 4;++ assert(n % qk == 0);+ assert(nc % ncols_interleaved == 0);++ UNUSED(nb);+ UNUSED(ncols_interleaved);++#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)+ for (int c = 0; c < nc; c += ncols_interleaved) {+ const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (c / ncols_interleaved) * nb;+ const block_q8_0 * a_ptr = (const block_q8_0 *) vy;+ float32x4_t acc = vdupq_n_f32(0);++ for (int l = 0; l < nb; l++) {+ const float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *) b_ptr[l].d));+ float32x4_t accb = vdupq_n_f32(0);++ for (int k = 0; k < 4; k++) {+ const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * 4 + k;+ const float ad = GGML_CPU_FP16_TO_FP32(a_blk->d);+ int32x4_t ret = vdupq_n_s32(0);++ for (int tile = 0; tile < 8; tile += 4) {+ const int8x16_t signs0 = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * (tile + 0) + 0],+ b_ptr[l].qs[k * 16 + 2 * (tile + 0) + 1]);+ const int8x16_t signs1 = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * (tile + 1) + 0],+ b_ptr[l].qs[k * 16 + 2 * (tile + 1) + 1]);+ const int8x16_t signs2 = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * (tile + 2) + 0],+ b_ptr[l].qs[k * 16 + 2 * (tile + 2) + 1]);+ const int8x16_t signs3 = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * (tile + 3) + 0],+ b_ptr[l].qs[k * 16 + 2 * (tile + 3) + 1]);+ const int8x16_t q_tiles = vld1q_s8(a_blk->qs + tile * 4);++ ret = vdotq_laneq_s32(ret, signs0, q_tiles, 0);+ ret = vdotq_laneq_s32(ret, signs1, q_tiles, 1);+ ret = vdotq_laneq_s32(ret, signs2, q_tiles, 2);+ ret = vdotq_laneq_s32(ret, signs3, q_tiles, 3);+ }++ accb = vfmaq_n_f32(accb, vcvtq_f32_s32(ret), ad);+ }+ acc = vfmaq_f32(acc, accb, b_d);+ }+ vst1q_f32(s, acc);+ s += ncols_interleaved;+ }+ return;+#endif+ ggml_gemv_q1_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);+}++void ggml_gemv_q1_0_4x8_q8_0(int n,+ float * GGML_RESTRICT s,+ size_t bs,+ const void * GGML_RESTRICT vx,+ const void * GGML_RESTRICT vy,+ int nr,+ int nc) {+ const int qk = QK1_0;+ const int nb = n / qk;+ const int ncols_interleaved = 4;++ assert(n % qk == 0);+ assert(nc % ncols_interleaved == 0);++ UNUSED(nb);+ UNUSED(ncols_interleaved);++#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)+ for (int c = 0; c < nc; c += ncols_interleaved) {+ const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (c / ncols_interleaved) * nb;+ const block_q8_0 * a_ptr = (const block_q8_0 *) vy;+ float32x4_t acc = vdupq_n_f32(0);++ for (int l = 0; l < nb; l++) {+ const float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *) b_ptr[l].d));+ float32x4_t accb = vdupq_n_f32(0);++ for (int k = 0; k < 4; ++k) {+ const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * 4 + k;+ const uint8_t * GGML_RESTRICT b_qs = (const uint8_t *) b_ptr[l].qs + k * 16;+ const float ad = GGML_CPU_FP16_TO_FP32(a_blk->d);++ int8x8x4_t a_chunks = vld1_s8_x4(a_blk->qs);+ int8x16_t a0 = vcombine_s8(a_chunks.val[0], a_chunks.val[0]);+ int8x16_t a1 = vcombine_s8(a_chunks.val[1], a_chunks.val[1]);+ int8x16_t a2 = vcombine_s8(a_chunks.val[2], a_chunks.val[2]);+ int8x16_t a3 = vcombine_s8(a_chunks.val[3], a_chunks.val[3]);++ int32x4_t ret0 = vdupq_n_s32(0);+ int32x4_t ret1 = vdupq_n_s32(0);++ ret0 = vdotq_s32(ret0, ggml_q1_0_unpack_pair(b_qs[0], b_qs[1]), a0);+ ret1 = vdotq_s32(ret1, ggml_q1_0_unpack_pair(b_qs[2], b_qs[3]), a0);+ ret0 = vdotq_s32(ret0, ggml_q1_0_unpack_pair(b_qs[4], b_qs[5]), a1);+ ret1 = vdotq_s32(ret1, ggml_q1_0_unpack_pair(b_qs[6], b_qs[7]), a1);+ ret0 = vdotq_s32(ret0, ggml_q1_0_unpack_pair(b_qs[8], b_qs[9]), a2);+ ret1 = vdotq_s32(ret1, ggml_q1_0_unpack_pair(b_qs[10], b_qs[11]), a2);+ ret0 = vdotq_s32(ret0, ggml_q1_0_unpack_pair(b_qs[12], b_qs[13]), a3);+ ret1 = vdotq_s32(ret1, ggml_q1_0_unpack_pair(b_qs[14], b_qs[15]), a3);++ accb = vfmaq_n_f32(accb, vcvtq_f32_s32(vpaddq_s32(ret0, ret1)), ad);+ }++ acc = vfmaq_f32(acc, accb, b_d);+ }++ vst1q_f32(s, acc);+ s += ncols_interleaved;+ }+ return;+#endif++ ggml_gemv_q1_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);+}+void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {const int qk = QK8_0;const int nb = n / qk;@@ -5154,3 +5298,168 @@ void ggml_gemm_q8_0_4x8_q8_0(int n,#endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)ggml_gemm_q8_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);}++void ggml_gemm_q1_0_4x4_q8_0(int n,+ float * GGML_RESTRICT s,+ size_t bs,+ const void * GGML_RESTRICT vx,+ const void * GGML_RESTRICT vy,+ int nr,+ int nc) {+ const int qk = QK1_0;+ const int nb = n / qk;+ const int ncols_interleaved = 4;++ assert(n % qk == 0);+ assert(nr % 4 == 0);+ assert(nc % ncols_interleaved == 0);++ UNUSED(nb);+ UNUSED(ncols_interleaved);++#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)+ for (int y = 0; y < nr / 4; y++) {+ const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb);+ for (int x = 0; x < nc / ncols_interleaved; x++) {+ const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb);++ float32x4_t sumf[4];+ for (int m = 0; m < 4; m++) {+ sumf[m] = vdupq_n_f32(0);+ }++ for (int l = 0; l < nb; l++) {+ float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *) b_ptr[l].d));+ float32x4_t blockf_0 = vdupq_n_f32(0);+ float32x4_t blockf_1 = vdupq_n_f32(0);+ float32x4_t blockf_2 = vdupq_n_f32(0);+ float32x4_t blockf_3 = vdupq_n_f32(0);++ for (int k = 0; k < 4; ++k) {+ const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k;+ float32x4_t a_d = vcvt_f32_f16(vld1_f16((const float16_t *) a_blk->d));++ int32x4_t sumi_0 = vdupq_n_s32(0);+ int32x4_t sumi_1 = vdupq_n_s32(0);+ int32x4_t sumi_2 = vdupq_n_s32(0);+ int32x4_t sumi_3 = vdupq_n_s32(0);++ for (int tile = 0; tile < 8; ++tile) {+ const int8x16_t signs = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * tile + 0],+ b_ptr[l].qs[k * 16 + 2 * tile + 1]);+ const int8x16_t a_tile = vld1q_s8(a_blk->qs + tile * 16);++ sumi_0 = vdotq_laneq_s32(sumi_0, signs, a_tile, 0);+ sumi_1 = vdotq_laneq_s32(sumi_1, signs, a_tile, 1);+ sumi_2 = vdotq_laneq_s32(sumi_2, signs, a_tile, 2);+ sumi_3 = vdotq_laneq_s32(sumi_3, signs, a_tile, 3);+ }++ blockf_0 = vfmaq_laneq_f32(blockf_0, vcvtq_f32_s32(sumi_0), a_d, 0);+ blockf_1 = vfmaq_laneq_f32(blockf_1, vcvtq_f32_s32(sumi_1), a_d, 1);+ blockf_2 = vfmaq_laneq_f32(blockf_2, vcvtq_f32_s32(sumi_2), a_d, 2);+ blockf_3 = vfmaq_laneq_f32(blockf_3, vcvtq_f32_s32(sumi_3), a_d, 3);+ }++ sumf[0] = vfmaq_f32(sumf[0], blockf_0, b_d);+ sumf[1] = vfmaq_f32(sumf[1], blockf_1, b_d);+ sumf[2] = vfmaq_f32(sumf[2], blockf_2, b_d);+ sumf[3] = vfmaq_f32(sumf[3], blockf_3, b_d);+ }++ for (int m = 0; m < 4; m++) {+ vst1q_f32(s + (y * 4 + m) * bs + x * 4, sumf[m]);+ }+ }+ }+ return;+#endif+ ggml_gemm_q1_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);+}++void ggml_gemm_q1_0_4x8_q8_0(int n,+ float * GGML_RESTRICT s,+ size_t bs,+ const void * GGML_RESTRICT vx,+ const void * GGML_RESTRICT vy,+ int nr,+ int nc) {+ const int qk = QK1_0;+ const int nb = n / qk;+ const int ncols_interleaved = 4;++ assert(n % qk == 0);+ assert(nr % 4 == 0);+ assert(nc % ncols_interleaved == 0);++ UNUSED(nb);+ UNUSED(ncols_interleaved);++#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)+ for (int y = 0; y < nr / 4; y++) {+ const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb);++ for (int x = 0; x < nc / ncols_interleaved; x++) {+ const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb);++ float32x4_t sumf[4];+ for (int m = 0; m < 4; ++m) {+ sumf[m] = vdupq_n_f32(0);+ }++ for (int l = 0; l < nb; l++) {+ const float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *) b_ptr[l].d));+ float32x4_t blockf[4];+ for (int m = 0; m < 4; ++m) {+ blockf[m] = vdupq_n_f32(0);+ }++ for (int k = 0; k < 4; ++k) {+ const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k;+ const uint8_t * GGML_RESTRICT b_qs = (const uint8_t *) b_ptr[l].qs + k * 16;++ int32x4_t acc[4];+ for (int i = 0; i < 4; ++i) {+ acc[i] = vdupq_n_s32(0);+ }++ for (int chunk = 0; chunk < 4; ++chunk) {+ const int8x16_t a01 = vld1q_s8(a_blk->qs + chunk * 32);+ const int8x16_t a23 = vld1q_s8(a_blk->qs + chunk * 32 + 16);+ const int8x16_t b01 = ggml_q1_0_unpack_pair(b_qs[chunk * 4 + 0], b_qs[chunk * 4 + 1]);+ const int8x16_t b23 = ggml_q1_0_unpack_pair(b_qs[chunk * 4 + 2], b_qs[chunk * 4 + 3]);++ acc[0] = vmmlaq_s32(acc[0], a01, b01);+ acc[1] = vmmlaq_s32(acc[1], a01, b23);+ acc[2] = vmmlaq_s32(acc[2], a23, b01);+ acc[3] = vmmlaq_s32(acc[3], a23, b23);+ }++ const int32x4_t row0 = vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1]));+ const int32x4_t row1 = vcombine_s32(vget_high_s32(acc[0]), vget_high_s32(acc[1]));+ const int32x4_t row2 = vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3]));+ const int32x4_t row3 = vcombine_s32(vget_high_s32(acc[2]), vget_high_s32(acc[3]));+ const float32x4_t a_d = vcvt_f32_f16(vld1_f16((const float16_t *) a_blk->d));++ blockf[0] = vfmaq_laneq_f32(blockf[0], vcvtq_f32_s32(row0), a_d, 0);+ blockf[1] = vfmaq_laneq_f32(blockf[1], vcvtq_f32_s32(row1), a_d, 1);+ blockf[2] = vfmaq_laneq_f32(blockf[2], vcvtq_f32_s32(row2), a_d, 2);+ blockf[3] = vfmaq_laneq_f32(blockf[3], vcvtq_f32_s32(row3), a_d, 3);+ }++ sumf[0] = vfmaq_f32(sumf[0], blockf[0], b_d);+ sumf[1] = vfmaq_f32(sumf[1], blockf[1], b_d);+ sumf[2] = vfmaq_f32(sumf[2], blockf[2], b_d);+ sumf[3] = vfmaq_f32(sumf[3], blockf[3], b_d);+ }++ for (int m = 0; m < 4; ++m) {+ vst1q_f32(s + (y * 4 + m) * bs + x * 4, sumf[m]);+ }+ }+ }+ return;+#endif++ ggml_gemm_q1_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);+}diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cppindex f5e419c1ec..d56db98023 100644--- a/ggml/src/ggml-cpu/repack.cpp+++ b/ggml/src/ggml-cpu/repack.cpp@@ -1365,6 +1365,133 @@ void ggml_gemv_q8_0_4x8_q8_0_generic(int n,}}+void ggml_gemv_q1_0_4x4_q8_0_generic(int n,+ float * GGML_RESTRICT s,+ size_t bs,+ const void * GGML_RESTRICT vx,+ const void * GGML_RESTRICT vy,+ int nr,+ int nc) {+ const int qk = QK1_0;+ const int nb = n / qk;+ const int ncols_interleaved = 4;++ assert(nr == 1);+ assert(n % qk == 0);+ assert(nc % ncols_interleaved == 0);++ UNUSED(bs);+ UNUSED(nr);++ float sumf[4];++ const block_q8_0 * a_ptr = (const block_q8_0 *) vy;+ for (int x = 0; x < nc / ncols_interleaved; x++) {+ const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb);++ for (int j = 0; j < ncols_interleaved; j++) {+ sumf[j] = 0.0;+ }++ for (int l = 0; l < nb; l++) {+ const float d0[4] = {+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]),+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]),+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]),+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]),+ };++ for (int k = 0; k < QK1_0 / QK8_0; ++k) {+ const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * (QK1_0 / QK8_0) + k;+ const float d1 = GGML_CPU_FP16_TO_FP32(a_blk->d);+ const float scale[4] = { d0[0] * d1, d0[1] * d1, d0[2] * d1, d0[3] * d1 };++ for (int tile = 0; tile < QK8_0 / 4; ++tile) {+ const uint8_t bits_lo = b_ptr[l].qs[k * 16 + 2 * tile + 0];+ const uint8_t bits_hi = b_ptr[l].qs[k * 16 + 2 * tile + 1];++ for (int p = 0; p < 4; ++p) {+ const float q = (float) a_blk->qs[tile * 4 + p];++ sumf[0] += ((bits_lo & (1u << p)) ? scale[0] : -scale[0]) * q;+ sumf[1] += ((bits_lo & (1u << (4 + p))) ? scale[1] : -scale[1]) * q;+ sumf[2] += ((bits_hi & (1u << p)) ? scale[2] : -scale[2]) * q;+ sumf[3] += ((bits_hi & (1u << (4 + p))) ? scale[3] : -scale[3]) * q;+ }+ }+ }+ }++ for (int j = 0; j < ncols_interleaved; j++) {+ s[x * ncols_interleaved + j] = sumf[j];+ }+ }+}++void ggml_gemv_q1_0_4x8_q8_0_generic(int n,+ float * GGML_RESTRICT s,+ size_t bs,+ const void * GGML_RESTRICT vx,+ const void * GGML_RESTRICT vy,+ int nr,+ int nc) {+ const int qk = QK1_0;+ const int nb = n / qk;+ const int ncols_interleaved = 4;+ const int blocklen = 8;++ assert(nr == 1);+ assert(n % qk == 0);+ assert(nc % ncols_interleaved == 0);++ UNUSED(bs);+ UNUSED(nr);++ float sumf[4];++ const block_q8_0 * a_ptr = (const block_q8_0 *) vy;+ for (int x = 0; x < nc / ncols_interleaved; x++) {+ const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb);++ for (int j = 0; j < ncols_interleaved; j++) {+ sumf[j] = 0.0f;+ }++ for (int l = 0; l < nb; l++) {+ const float d0[4] = {+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]),+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]),+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]),+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]),+ };++ for (int k = 0; k < qk / blocklen; ++k) {+ const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * (qk / QK8_0) + k / (QK8_0 / blocklen);+ const float d1 = GGML_CPU_FP16_TO_FP32(a_blk->d);+ const float scale[4] = { d0[0] * d1, d0[1] * d1, d0[2] * d1, d0[3] * d1 };+ const uint8_t bits0 = b_ptr[l].qs[k * ncols_interleaved + 0];+ const uint8_t bits1 = b_ptr[l].qs[k * ncols_interleaved + 1];+ const uint8_t bits2 = b_ptr[l].qs[k * ncols_interleaved + 2];+ const uint8_t bits3 = b_ptr[l].qs[k * ncols_interleaved + 3];+ const int q_offset = (k % (QK8_0 / blocklen)) * blocklen;++ for (int p = 0; p < blocklen; ++p) {+ const float q = (float) a_blk->qs[q_offset + p];++ sumf[0] += ((bits0 & (1u << p)) ? scale[0] : -scale[0]) * q;+ sumf[1] += ((bits1 & (1u << p)) ? scale[1] : -scale[1]) * q;+ sumf[2] += ((bits2 & (1u << p)) ? scale[2] : -scale[2]) * q;+ sumf[3] += ((bits3 & (1u << p)) ? scale[3] : -scale[3]) * q;+ }+ }+ }++ for (int j = 0; j < ncols_interleaved; j++) {+ s[x * ncols_interleaved + j] = sumf[j];+ }+ }+}+// Only enable these for RISC-V.#if defined __riscv_zvfhvoid ggml_gemv_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {@@ -2383,6 +2510,176 @@ void ggml_gemm_q8_0_4x8_q8_0_generic(int n,}}+void ggml_gemm_q1_0_4x4_q8_0_generic(int n,+ float * GGML_RESTRICT s,+ size_t bs,+ const void * GGML_RESTRICT vx,+ const void * GGML_RESTRICT vy,+ int nr,+ int nc) {+ const int qk = QK1_0;+ const int nb = n / qk;+ const int ncols_interleaved = 4;++ assert(n % qk == 0);+ assert(nr % 4 == 0);+ assert(nc % ncols_interleaved == 0);++ float sumf[4][4];++ for (int y = 0; y < nr / 4; y++) {+ const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb);+ for (int x = 0; x < nc / ncols_interleaved; x++) {+ const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb);++ for (int m = 0; m < 4; m++) {+ for (int j = 0; j < ncols_interleaved; j++) {+ sumf[m][j] = 0.0;+ }+ }++ for (int l = 0; l < nb; l++) {+ const float d0[4] = {+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]),+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]),+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]),+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]),+ };++ for (int k = 0; k < QK1_0 / QK8_0; ++k) {+ const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k;+ const float a_d[4] = {+ GGML_CPU_FP16_TO_FP32(a_blk->d[0]),+ GGML_CPU_FP16_TO_FP32(a_blk->d[1]),+ GGML_CPU_FP16_TO_FP32(a_blk->d[2]),+ GGML_CPU_FP16_TO_FP32(a_blk->d[3]),+ };++ for (int tile = 0; tile < QK8_0 / 4; ++tile) {+ const uint8_t bits_lo = b_ptr[l].qs[k * 16 + 2 * tile + 0];+ const uint8_t bits_hi = b_ptr[l].qs[k * 16 + 2 * tile + 1];+ const int tile_offset = tile * 16;++ for (int p = 0; p < 4; ++p) {+ const int8_t q_row[4] = {+ a_blk->qs[tile_offset + 0 * 4 + p],+ a_blk->qs[tile_offset + 1 * 4 + p],+ a_blk->qs[tile_offset + 2 * 4 + p],+ a_blk->qs[tile_offset + 3 * 4 + p],+ };+ const int sign[4] = {+ (bits_lo & (1u << p)) ? 1 : -1,+ (bits_lo & (1u << (4 + p))) ? 1 : -1,+ (bits_hi & (1u << p)) ? 1 : -1,+ (bits_hi & (1u << (4 + p))) ? 1 : -1,+ };++ for (int m = 0; m < 4; ++m) {+ const float row_scale = a_d[m];+ sumf[m][0] += sign[0] * q_row[m] * d0[0] * row_scale;+ sumf[m][1] += sign[1] * q_row[m] * d0[1] * row_scale;+ sumf[m][2] += sign[2] * q_row[m] * d0[2] * row_scale;+ sumf[m][3] += sign[3] * q_row[m] * d0[3] * row_scale;+ }+ }+ }+ }+ }++ for (int m = 0; m < 4; m++) {+ for (int j = 0; j < ncols_interleaved; j++) {+ s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];+ }+ }+ }+ }+}++void ggml_gemm_q1_0_4x8_q8_0_generic(int n,+ float * GGML_RESTRICT s,+ size_t bs,+ const void * GGML_RESTRICT vx,+ const void * GGML_RESTRICT vy,+ int nr,+ int nc) {+ const int qk = QK1_0;+ const int nb = n / qk;+ const int ncols_interleaved = 4;+ const int blocklen = 8;++ assert(n % qk == 0);+ assert(nr % 4 == 0);+ assert(nc % ncols_interleaved == 0);++ float sumf[4][4];++ for (int y = 0; y < nr / 4; y++) {+ const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb);+ for (int x = 0; x < nc / ncols_interleaved; x++) {+ const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb);++ for (int m = 0; m < 4; m++) {+ for (int j = 0; j < ncols_interleaved; j++) {+ sumf[m][j] = 0.0f;+ }+ }++ for (int l = 0; l < nb; l++) {+ const float d0[4] = {+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]),+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]),+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]),+ GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]),+ };++ for (int k = 0; k < qk / blocklen; ++k) {+ const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k / (QK8_0 / blocklen);+ const float a_d[4] = {+ GGML_CPU_FP16_TO_FP32(a_blk->d[0]),+ GGML_CPU_FP16_TO_FP32(a_blk->d[1]),+ GGML_CPU_FP16_TO_FP32(a_blk->d[2]),+ GGML_CPU_FP16_TO_FP32(a_blk->d[3]),+ };+ const uint8_t bits0 = b_ptr[l].qs[k * ncols_interleaved + 0];+ const uint8_t bits1 = b_ptr[l].qs[k * ncols_interleaved + 1];+ const uint8_t bits2 = b_ptr[l].qs[k * ncols_interleaved + 2];+ const uint8_t bits3 = b_ptr[l].qs[k * ncols_interleaved + 3];+ const int q_offset = (k % (QK8_0 / blocklen)) * 4 * blocklen;++ for (int p = 0; p < blocklen; ++p) {+ const int8_t q_row[4] = {+ a_blk->qs[q_offset + 0 * blocklen + p],+ a_blk->qs[q_offset + 1 * blocklen + p],+ a_blk->qs[q_offset + 2 * blocklen + p],+ a_blk->qs[q_offset + 3 * blocklen + p],+ };+ const int sign[4] = {+ (bits0 & (1u << p)) ? 1 : -1,+ (bits1 & (1u << p)) ? 1 : -1,+ (bits2 & (1u << p)) ? 1 : -1,+ (bits3 & (1u << p)) ? 1 : -1,+ };++ for (int m = 0; m < 4; ++m) {+ const float row_scale = a_d[m];+ sumf[m][0] += sign[0] * q_row[m] * d0[0] * row_scale;+ sumf[m][1] += sign[1] * q_row[m] * d0[1] * row_scale;+ sumf[m][2] += sign[2] * q_row[m] * d0[2] * row_scale;+ sumf[m][3] += sign[3] * q_row[m] * d0[3] * row_scale;+ }+ }+ }+ }++ for (int m = 0; m < 4; m++) {+ for (int j = 0; j < ncols_interleaved; j++) {+ s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];+ }+ }+ }+ }+}+// Only enable these for RISC-V.#if defined __riscv_zvfhvoid ggml_gemm_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {@@ -2739,6 +3036,50 @@ static block_q8_0x4 make_block_q8_0x4(block_q8_0 * in, unsigned int blck_size_inreturn out;}+static block_q1_0x4 make_block_q1_0x4(block_q1_0 * in, unsigned int blck_size_interleave) {+ block_q1_0x4 out;++ for (int i = 0; i < 4; i++) {+ out.d[i] = in[i].d;+ }++ GGML_ASSERT(blck_size_interleave == 4 || blck_size_interleave == 8);++ if (blck_size_interleave == 4) {+ for (int k = 0; k < QK1_0 / QK8_0; ++k) {+ for (int tile = 0; tile < QK8_0 / 4; ++tile) {+ uint8_t packed_lo = 0;+ uint8_t packed_hi = 0;++ const int weight_base = k * QK8_0 + tile * 4;+ for (int pos = 0; pos < 4; ++pos) {+ const int weight_idx = weight_base + pos;+ const int byte_idx = weight_idx / 8;+ const int bit_idx = weight_idx % 8;++ packed_lo |= ((in[0].qs[byte_idx] >> bit_idx) & 1u) << pos;+ packed_lo |= ((in[1].qs[byte_idx] >> bit_idx) & 1u) << (4 + pos);+ packed_hi |= ((in[2].qs[byte_idx] >> bit_idx) & 1u) << pos;+ packed_hi |= ((in[3].qs[byte_idx] >> bit_idx) & 1u) << (4 + pos);+ }++ out.qs[k * 16 + 2 * tile + 0] = packed_lo;+ out.qs[k * 16 + 2 * tile + 1] = packed_hi;+ }+ }+ return out;+ }++ for (int byte_idx = 0; byte_idx < QK1_0 / 8; ++byte_idx) {+ out.qs[byte_idx * 4 + 0] = in[0].qs[byte_idx];+ out.qs[byte_idx * 4 + 1] = in[1].qs[byte_idx];+ out.qs[byte_idx * 4 + 2] = in[2].qs[byte_idx];+ out.qs[byte_idx * 4 + 3] = in[3].qs[byte_idx];+ }++ return out;+}+static block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, int blck_size_interleave) {block_q4_0x4 out;@@ -3509,6 +3850,38 @@ static int repack_q8_0_to_q8_0_4_bl(struct ggml_tensor * t,return 0;}+static int repack_q1_0_to_q1_0_4_bl(struct ggml_tensor * t,+ int interleave_block,+ const void * GGML_RESTRICT data,+ size_t data_size) {+ GGML_ASSERT(t->type == GGML_TYPE_Q1_0);+ GGML_ASSERT(interleave_block == 4 || interleave_block == 8);+ constexpr int nrows_interleaved = 4;++ block_q1_0x4 * dst = (block_q1_0x4 *) t->data;+ const block_q1_0 * src = (const block_q1_0 *) data;+ block_q1_0 dst_tmp[4];+ int nrow = ggml_nrows(t);+ int nblocks = t->ne[0] / QK1_0;++ GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q1_0));++ if (t->ne[1] % nrows_interleaved != 0) {+ return -1;+ }++ for (int b = 0; b < nrow; b += nrows_interleaved) {+ for (int64_t x = 0; x < nblocks; x++) {+ for (int i = 0; i < nrows_interleaved; i++) {+ dst_tmp[i] = src[x + i * nblocks];+ }+ *dst++ = make_block_q1_0x4(dst_tmp, interleave_block);+ }+ src += nrows_interleaved * nblocks;+ }+ return 0;+}+static block_q8_0x16 make_block_q8_0x16(block_q8_0 * in, unsigned int blck_size_interleave) {block_q8_0x16 out;@@ -3865,6 +4238,14 @@ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS>int repack(struct ggml_tensor *, const void *, size_t);// TODO: generalise.+template <> int repack<block_q1_0, 4, 4>(struct ggml_tensor * t, const void * data, size_t data_size) {+ return repack_q1_0_to_q1_0_4_bl(t, 4, data, data_size);+}++template <> int repack<block_q1_0, 8, 4>(struct ggml_tensor * t, const void * data, size_t data_size) {+ return repack_q1_0_to_q1_0_4_bl(t, 8, data, data_size);+}+template <> int repack<block_q4_0, 4, 4>(struct ggml_tensor * t, const void * data, size_t data_size) {return repack_q4_0_to_q4_0_4_bl(t, 4, data, data_size);}@@ -3960,6 +4341,14 @@ template <> int repack<block_q2_K, 1, 16>(struct ggml_tensor * t, const void * dtemplate <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS, ggml_type PARAM_TYPE>void gemv(int, float *, size_t, const void *, const void *, int, int);+template <> void gemv<block_q1_0, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {+ ggml_gemv_q1_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc);+}++template <> void gemv<block_q1_0, 8, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {+ ggml_gemv_q1_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc);+}+template <> void gemv<block_q4_0, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {ggml_gemv_q4_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc);}@@ -4057,6 +4446,14 @@ template <> void gemv<block_q2_K, 1, 16, GGML_TYPE_Q8_K>(int n, float * s, size_template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS, ggml_type PARAM_TYPE>void gemm(int, float *, size_t, const void *, const void *, int, int);+template <> void gemm<block_q1_0, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {+ ggml_gemm_q1_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc);+}++template <> void gemm<block_q1_0, 8, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {+ ggml_gemm_q1_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc);+}+template <> void gemm<block_q4_0, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {ggml_gemm_q4_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc);}@@ -4526,6 +4923,10 @@ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS, ggml_type PAR} // namespace ggml::cpu::repackstatic const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(const struct ggml_tensor * cur) {+ // instance for Q1_0+ static const ggml::cpu::repack::tensor_traits<block_q1_0, 4, 4, GGML_TYPE_Q8_0> q1_0_4x4_q8_0;+ static const ggml::cpu::repack::tensor_traits<block_q1_0, 8, 4, GGML_TYPE_Q8_0> q1_0_4x8_q8_0;+// instance for Q4static const ggml::cpu::repack::tensor_traits<block_q4_0, 4, 4, GGML_TYPE_Q8_0> q4_0_4x4_q8_0;static const ggml::cpu::repack::tensor_traits<block_q4_0, 8, 4, GGML_TYPE_Q8_0> q4_0_4x8_q8_0;@@ -4723,6 +5124,17 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons}#endif}+ } else if (cur->type == GGML_TYPE_Q1_0) {+ if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {+ if (cur->ne[1] % 4 == 0) {+ return &q1_0_4x8_q8_0;+ }+ }+ if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {+ if (cur->ne[1] % 4 == 0) {+ return &q1_0_4x4_q8_0;+ }+ }}return nullptr;diff --git a/ggml/src/ggml-cpu/repack.h b/ggml/src/ggml-cpu/repack.hindex cb21edf623..fc6715c39c 100644--- a/ggml/src/ggml-cpu/repack.h+++ b/ggml/src/ggml-cpu/repack.h@@ -11,6 +11,9 @@ggml_backend_buffer_type_t ggml_backend_cpu_repack_buffer_type(void);template <int K> constexpr int QK_0() {+ if constexpr (K == 1) {+ return QK1_0;+ }if constexpr (K == 4) {return QK4_0;}@@ -26,6 +29,7 @@ template <int K, int N> struct block {};// control size+static_assert(sizeof(block<1, 4>) == 4 * sizeof(ggml_half) + QK1_0 / 2, "wrong block<1,4> size/padding");static_assert(sizeof(block<4, 4>) == 4 * sizeof(ggml_half) + QK8_0 * 2, "wrong block<4,4> size/padding");static_assert(sizeof(block<4, 8>) == 8 * sizeof(ggml_half) + QK8_0 * 4, "wrong block<4,8> size/padding");static_assert(sizeof(block<4, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 8, "wrong block<4,16> size/padding");@@ -33,6 +37,7 @@ static_assert(sizeof(block<8, 4>) == 4 * sizeof(ggml_half) + QK8_0 * 4, "wrong bstatic_assert(sizeof(block<8, 8>) == 8 * sizeof(ggml_half) + QK8_0 * 8, "wrong block<8,8> size/padding");static_assert(sizeof(block<8, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 16, "wrong block<8,16> size/padding");+using block_q1_0x4 = block<1, 4>;using block_q4_0x4 = block<4, 4>;using block_q4_0x8 = block<4, 8>;using block_q4_0x16 = block<4, 16>;@@ -141,6 +146,8 @@ void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRvoid ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);void ggml_quantize_mat_q8_K_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);void ggml_quantize_mat_q8_K_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);+void ggml_gemv_q1_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);+void ggml_gemv_q1_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemv_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);@@ -157,6 +164,8 @@ void ggml_gemv_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vvoid ggml_gemv_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemv_q8_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemv_q8_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);+void ggml_gemm_q1_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);+void ggml_gemm_q1_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemm_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);@@ -193,6 +202,8 @@ void ggml_quantize_mat_q8_0_4x4_generic(const float * GGML_RESTRICT x, void * GGvoid ggml_quantize_mat_q8_0_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);void ggml_quantize_mat_q8_K_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);void ggml_quantize_mat_q8_K_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);+void ggml_gemv_q1_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);+void ggml_gemv_q1_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemv_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemv_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemv_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);@@ -209,6 +220,8 @@ void ggml_gemv_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs,void ggml_gemv_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemv_q8_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemv_q8_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);+void ggml_gemm_q1_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);+void ggml_gemm_q1_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemm_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemm_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);void ggml_gemm_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);