LLM inference in C/C++
C++ ggml
ggml-cpu: refactor s309x repack helper functions (#30588)
Aaron Teoc804754529cf
diff --git a/ggml/src/ggml-cpu/arch/s390/quants.c b/ggml/src/ggml-cpu/arch/s390/quants.cindex 52344828e3..22b8b83109 100644--- a/ggml/src/ggml-cpu/arch/s390/quants.c+++ b/ggml/src/ggml-cpu/arch/s390/quants.c@@ -310,7 +310,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voiconst int8x16_t v_yl = vec_xl(0 , y[ib].qs);const int8x16_t v_yh = vec_xl(QK8_1/2, y[ib].qs);- const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);+ const int32x4_t v_xy_ = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh);const float32x4_t v_xy = vec_float(v_xy_);const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));@@ -379,8 +379,8 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voconst int8x16_t v_y1l = vec_xl(0, y1->qs);const int8x16_t v_y1h = vec_xl(QK8_0/2, y1->qs);- const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0l), v_x0h, v_y0h);- const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y1l), v_x1h, v_y1h);+ const int32x4_t v_xy0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0l, v_y0l), v_x0h, v_y0h);+ const int32x4_t v_xy1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1l, v_y1l), v_x1h, v_y1h);const float32x4_t v_xy0f = vec_float(v_xy0);const float32x4_t v_xy1f = vec_float(v_xy1);@@ -407,7 +407,7 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voconst int8x16_t v_yl = vec_xl(0, y0->qs);const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs);- const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);+ const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh);const float32x4_t v_xyf = vec_float(v_xy);const float32x4_t v_d = vec_splats(GGML_E8M0_TO_FP32_HALF(x0->e) * GGML_CPU_FP16_TO_FP32(y0->d));@@ -502,8 +502,8 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voiconst int8x16_t v_y1l = vec_xl(0, (const int8_t *)y1->qs);const int8x16_t v_y1h = vec_xl(QK8_0/2, (const int8_t *)y1->qs);- const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h);- const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h);+ const int32x4_t v_xy0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h);+ const int32x4_t v_xy1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h);const float32x4_t v_xy0f = vec_float(v_xy0);const float32x4_t v_xy1f = vec_float(v_xy1);@@ -548,7 +548,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voiconst int8x16_t v_yl = vec_xl(0, (const int8_t *)y0->qs);const int8x16_t v_yh = vec_xl(QK8_0/2, (const int8_t *)y0->qs);- const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh);+ const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh);const float32x4_t v_xyf = vec_float(v_xy);const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));@@ -653,8 +653,8 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voiconst int8x16_t v_y1l = vec_xl(0 , y1->qs);const int8x16_t v_y1h = vec_xl(QK8_1/2, y1->qs);- const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h);- const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h);+ const int32x4_t v_xy0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h);+ const int32x4_t v_xy1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h);const float32x4_t v_xy0f = vec_float(v_xy0);const float32x4_t v_xy1f = vec_float(v_xy1);@@ -701,7 +701,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voiconst int8x16_t v_yl = vec_xl(0 , y0->qs);const int8x16_t v_yh = vec_xl(QK8_1/2, y0->qs);- const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh);+ const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh);const float32x4_t v_xyf = vec_float(v_xy);const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));@@ -751,7 +751,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voiconst int8x16_t v_yl = vec_xl(0 , y[ib].qs);const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs);- const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);+ const int32x4_t v_xy_ = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh);const float32x4_t v_xy = vec_float(v_xy_);const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));@@ -855,10 +855,10 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voiq3bytes[2] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 2), v_3m), (int8x16_t)q3h[2]);q3bytes[3] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 2), v_3m), (int8x16_t)q3h[3]);- isum0 = ggml_vec_dot(v_z, q3bytes[0], q8bytes[0]);- isum1 = ggml_vec_dot(v_z, q3bytes[1], q8bytes[1]);- isum2 = ggml_vec_dot(v_z, q3bytes[2], q8bytes[2]);- isum3 = ggml_vec_dot(v_z, q3bytes[3], q8bytes[3]);+ isum0 = ggml_vec_dot_s32(v_z, q3bytes[0], q8bytes[0]);+ isum1 = ggml_vec_dot_s32(v_z, q3bytes[1], q8bytes[1]);+ isum2 = ggml_vec_dot_s32(v_z, q3bytes[2], q8bytes[2]);+ isum3 = ggml_vec_dot_s32(v_z, q3bytes[3], q8bytes[3]);isum += (isum0[0] + isum0[1] + isum0[2] + isum0[3]) * scale[0];isum += (isum1[0] + isum1[1] + isum1[2] + isum1[3]) * scale[1];@@ -877,10 +877,10 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voiq3bytes[2] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 6), v_3m), (int8x16_t)q3h[2]);q3bytes[3] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 6), v_3m), (int8x16_t)q3h[3]);- isum0 = ggml_vec_dot(v_z, q3bytes[0], q8bytes[4]);- isum1 = ggml_vec_dot(v_z, q3bytes[1], q8bytes[5]);- isum2 = ggml_vec_dot(v_z, q3bytes[2], q8bytes[6]);- isum3 = ggml_vec_dot(v_z, q3bytes[3], q8bytes[7]);+ isum0 = ggml_vec_dot_s32(v_z, q3bytes[0], q8bytes[4]);+ isum1 = ggml_vec_dot_s32(v_z, q3bytes[1], q8bytes[5]);+ isum2 = ggml_vec_dot_s32(v_z, q3bytes[2], q8bytes[6]);+ isum3 = ggml_vec_dot_s32(v_z, q3bytes[3], q8bytes[7]);isum += vec_hsum_i32x4(isum0) * scale[0];isum += vec_hsum_i32x4(isum1) * scale[1];@@ -981,7 +981,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voiv_xl[0] = (int8x16_t)vec_and(v_x[0], v_lm);v_xl[1] = (int8x16_t)vec_and(v_x[1], v_lm);- const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]);+ const int32x4_t p1 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]);sumi1 += vec_hsum_i32x4(p1) * scales[2*j+0];v_y[0] = vec_xl(0 , y0);@@ -991,7 +991,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voiv_xl[0] = (int8x16_t)vec_sr(v_x[0], 4);v_xl[1] = (int8x16_t)vec_sr(v_x[1], 4);- const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]);+ const int32x4_t p2 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]);sumi2 += vec_hsum_i32x4(p2) * scales[2*j+1];}@@ -1107,8 +1107,8 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voiq5b[2] = (int8x16_t)vec_or(vec_sr(v_xl[0], 4), q5h[2]);q5b[3] = (int8x16_t)vec_or(vec_sr(v_xl[1], 4), q5h[3]);- int32x4_t sumi0 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[0], v_y[0]), q5b[1], v_y[1]);- int32x4_t sumi1 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[2], v_y[2]), q5b[3], v_y[3]);+ int32x4_t sumi0 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, q5b[0], v_y[0]), q5b[1], v_y[1]);+ int32x4_t sumi1 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, q5b[2], v_y[2]), q5b[3], v_y[3]);sumi += vec_hsum_i32x4(sumi0) * *scales++;sumi += vec_hsum_i32x4(sumi1) * *scales++;@@ -1217,10 +1217,10 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voiq6b[2] = (int8x16_t)(vec_or(vec_and(v_xl[2], v_lm), q6h[2]));q6b[3] = (int8x16_t)(vec_or(vec_and(v_xl[3], v_lm), q6h[3]));- int32x4_t summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]);- int32x4_t summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]);- int32x4_t summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]);- int32x4_t summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]);+ int32x4_t summs0 = ggml_vec_dot_s32(v_z, q6b[0], v_y[0]);+ int32x4_t summs1 = ggml_vec_dot_s32(v_z, q6b[1], v_y[1]);+ int32x4_t summs2 = ggml_vec_dot_s32(v_z, q6b[2], v_y[2]);+ int32x4_t summs3 = ggml_vec_dot_s32(v_z, q6b[3], v_y[3]);isum += vec_hsum_i32x4(summs0) * scale[0] +vec_hsum_i32x4(summs1) * scale[1] +@@ -1251,10 +1251,10 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voiq6b[2] = (int8x16_t)(vec_or(vec_sr(v_xl[2], 4), q6h[2]));q6b[3] = (int8x16_t)(vec_or(vec_sr(v_xl[3], 4), q6h[3]));- summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]);- summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]);- summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]);- summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]);+ summs0 = ggml_vec_dot_s32(v_z, q6b[0], v_y[0]);+ summs1 = ggml_vec_dot_s32(v_z, q6b[1], v_y[1]);+ summs2 = ggml_vec_dot_s32(v_z, q6b[2], v_y[2]);+ summs3 = ggml_vec_dot_s32(v_z, q6b[3], v_y[3]);isum += vec_hsum_i32x4(summs0) * scale[0] +vec_hsum_i32x4(summs1) * scale[1] +@@ -1367,8 +1367,8 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi// q2u2 = vec_mul(q2u2, q2s2);// q2u3 = vec_mul(q2u3, q2s3);-// const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u0, q8b0), q2u1, q8b1);-// const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u2, q8b2), q2u3, q8b3);+// const int32x4_t p1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splat_s32(0), q2u0, q8b0), q2u1, q8b1);+// const int32x4_t p2 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splat_s32(0), q2u2, q8b2), q2u3, q8b3);// sumf1 += (p1[0] + p1[1] + p1[2] + p1[3]) * (0.5f + (aux32[1] >> 28));// sumf2 += (p2[0] + p2[1] + p2[2] + p2[3]) * (0.5f + (aux32[3] >> 28));@@ -1445,7 +1445,7 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vconst int8x16_t v_yl = vec_xl(0 , y0->qs);const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs);- const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);+ const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh);sumf += GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d) * vec_hsum_i32x4(v_xy);}@@ -1508,8 +1508,8 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vconst int8x16_t v_y3 = vec_xl(48, q8);q8 += 64;- int32x4_t vsumi0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0), v_x0h, v_y1);- int32x4_t vsumi1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y2), v_x1h, v_y3);+ int32x4_t vsumi0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0l, v_y0), v_x0h, v_y1);+ int32x4_t vsumi1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1l, v_y2), v_x1h, v_y3);int ls1 = ((x[ibl].scales_l[ib] & 0xF) | ((h << 4) & 0x30)) - 32;int ls2 = ((x[ibl].scales_l[ib] >> 4) | ((h << 2) & 0x30)) - 32;diff --git a/ggml/src/ggml-cpu/arch/s390/repack.cpp b/ggml/src/ggml-cpu/arch/s390/repack.cppindex abf3433add..c12fe215e8 100644--- a/ggml/src/ggml-cpu/arch/s390/repack.cpp+++ b/ggml/src/ggml-cpu/arch/s390/repack.cpp@@ -70,23 +70,6 @@ void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTR#endif}-#if defined(__VXE__) || defined(__VXE2__)-static inline int16x8_t vxe_dot_acc(const int8x16_t v_x, const int8x16_t v_y, const int16x8_t v_acc) {- return vec_meadd(v_x, v_y, vec_moadd(v_x, v_y, v_acc));-}--static inline int8x16_t vxe_splat_granule(const int8_t * qs) {- uint32_t g;- memcpy(&g, qs, sizeof(g));- return (int8x16_t)vec_splats(g);-}--static inline int32x4_t vxe_fold(const int16x8_t v_sumi) {- const int16x8_t v_ones = vec_splats((int16_t)1);- return vec_add(vec_mule(v_sumi, v_ones), vec_mulo(v_sumi, v_ones));-}-#endif-void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {const int qk = QK8_0;const int nb = n / qk;@@ -131,21 +114,21 @@ void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voint16x8_t v_sumi = vec_splats((int16_t)0);- v_sumi = vxe_dot_acc(v_x0l, vxe_splat_granule(y_lo + 0), v_sumi);- v_sumi = vxe_dot_acc(v_x1l, vxe_splat_granule(y_lo + 4), v_sumi);- v_sumi = vxe_dot_acc(v_x2l, vxe_splat_granule(y_lo + 8), v_sumi);- v_sumi = vxe_dot_acc(v_x3l, vxe_splat_granule(y_lo + 12), v_sumi);+ v_sumi = ggml_vec_dot_s16(v_sumi, v_x0l, ggml_vec_xl_s8x4_splat(y_lo + 0));+ v_sumi = ggml_vec_dot_s16(v_sumi, v_x1l, ggml_vec_xl_s8x4_splat(y_lo + 4));+ v_sumi = ggml_vec_dot_s16(v_sumi, v_x2l, ggml_vec_xl_s8x4_splat(y_lo + 8));+ v_sumi = ggml_vec_dot_s16(v_sumi, v_x3l, ggml_vec_xl_s8x4_splat(y_lo + 12));- v_sumi = vxe_dot_acc(v_x0h, vxe_splat_granule(y_hi + 0), v_sumi);- v_sumi = vxe_dot_acc(v_x1h, vxe_splat_granule(y_hi + 4), v_sumi);- v_sumi = vxe_dot_acc(v_x2h, vxe_splat_granule(y_hi + 8), v_sumi);- v_sumi = vxe_dot_acc(v_x3h, vxe_splat_granule(y_hi + 12), v_sumi);+ v_sumi = ggml_vec_dot_s16(v_sumi, v_x0h, ggml_vec_xl_s8x4_splat(y_hi + 0));+ v_sumi = ggml_vec_dot_s16(v_sumi, v_x1h, ggml_vec_xl_s8x4_splat(y_hi + 4));+ v_sumi = ggml_vec_dot_s16(v_sumi, v_x2h, ggml_vec_xl_s8x4_splat(y_hi + 8));+ v_sumi = ggml_vec_dot_s16(v_sumi, v_x3h, ggml_vec_xl_s8x4_splat(y_hi + 12));const float32x4_t v_yd = vec_splats(GGML_CPU_FP16_TO_FP32(a_ptr[l].d));- const float32x4_t v_xd = __lzs_f16cx4_load(b_ptr[l].d);+ const float32x4_t v_xd = ggml_vec_xl_f16x4_finite(b_ptr[l].d);const float32x4_t v_d = vec_mul(v_yd, v_xd);- v_sumf = vec_madd(vec_float(vxe_fold(v_sumi)), v_d, v_sumf);+ v_sumf = vec_madd(vec_float(vec_paddl_s16(v_sumi)), v_d, v_sumf);}vec_xst(v_sumf, 0, res_ptr + x * ncols_interleaved);@@ -192,24 +175,24 @@ void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voconst int8_t * y_lo = a_ptr[l].qs + 16 * k;const int8_t * y_hi = y_lo + qk / 2 * 4;- v_sumi0 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 0), v_sumi0);- v_sumi1 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 4), v_sumi1);- v_sumi2 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 8), v_sumi2);- v_sumi3 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 12), v_sumi3);+ v_sumi0 = ggml_vec_dot_s16(v_sumi0, v_xl, ggml_vec_xl_s8x4_splat(y_lo + 0));+ v_sumi1 = ggml_vec_dot_s16(v_sumi1, v_xl, ggml_vec_xl_s8x4_splat(y_lo + 4));+ v_sumi2 = ggml_vec_dot_s16(v_sumi2, v_xl, ggml_vec_xl_s8x4_splat(y_lo + 8));+ v_sumi3 = ggml_vec_dot_s16(v_sumi3, v_xl, ggml_vec_xl_s8x4_splat(y_lo + 12));- v_sumi0 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 0), v_sumi0);- v_sumi1 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 4), v_sumi1);- v_sumi2 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 8), v_sumi2);- v_sumi3 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 12), v_sumi3);+ v_sumi0 = ggml_vec_dot_s16(v_sumi0, v_xh, ggml_vec_xl_s8x4_splat(y_hi + 0));+ v_sumi1 = ggml_vec_dot_s16(v_sumi1, v_xh, ggml_vec_xl_s8x4_splat(y_hi + 4));+ v_sumi2 = ggml_vec_dot_s16(v_sumi2, v_xh, ggml_vec_xl_s8x4_splat(y_hi + 8));+ v_sumi3 = ggml_vec_dot_s16(v_sumi3, v_xh, ggml_vec_xl_s8x4_splat(y_hi + 12));}const float32x4_t v_yd = __lzs_f16cx4_load(a_ptr[l].d);- const float32x4_t v_xd = __lzs_f16cx4_load(b_ptr[l].d);+ const float32x4_t v_xd = ggml_vec_xl_f16x4_finite(b_ptr[l].d);- v_sumf[0] = vec_madd(vec_float(vxe_fold(v_sumi0)), vec_mul(v_xd, vec_splat(v_yd, 0)), v_sumf[0]);- v_sumf[1] = vec_madd(vec_float(vxe_fold(v_sumi1)), vec_mul(v_xd, vec_splat(v_yd, 1)), v_sumf[1]);- v_sumf[2] = vec_madd(vec_float(vxe_fold(v_sumi2)), vec_mul(v_xd, vec_splat(v_yd, 2)), v_sumf[2]);- v_sumf[3] = vec_madd(vec_float(vxe_fold(v_sumi3)), vec_mul(v_xd, vec_splat(v_yd, 3)), v_sumf[3]);+ v_sumf[0] = vec_madd(vec_float(vec_paddl_s16(v_sumi0)), vec_mul(v_xd, vec_splat(v_yd, 0)), v_sumf[0]);+ v_sumf[1] = vec_madd(vec_float(vec_paddl_s16(v_sumi1)), vec_mul(v_xd, vec_splat(v_yd, 1)), v_sumf[1]);+ v_sumf[2] = vec_madd(vec_float(vec_paddl_s16(v_sumi2)), vec_mul(v_xd, vec_splat(v_yd, 2)), v_sumf[2]);+ v_sumf[3] = vec_madd(vec_float(vec_paddl_s16(v_sumi3)), vec_mul(v_xd, vec_splat(v_yd, 3)), v_sumf[3]);}for (int m = 0; m < 4; m++) {diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.hindex 82345b2063..0d48c9a22e 100644--- a/ggml/src/ggml-cpu/ggml-cpu-impl.h+++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h@@ -443,6 +443,13 @@ inline static ggml_int8x16x4_t ggml_vec_xl_s8x4(const int8_t * ptr) {return res;}+// load 4 bytes and replicate them into all 4 word lanes+inline static int8x16_t ggml_vec_xl_s8x4_splat(const int8_t * ptr) {+ uint32_t tmp;+ memcpy(&tmp, ptr, sizeof(tmp));+ return (int8x16_t)vec_splats(tmp);+}+typedef struct ggml_int16x8x2_t {int16x8_t val[2];} ggml_int16x8x2_t;@@ -456,31 +463,17 @@ inline static ggml_int16x8x2_t ggml_vec_xl_s16x2(const int16_t * ptr) {return res;}-/*- ! WARNING: Very slow. Use vec_perm if possible. Refer to iq4_xs- ! or iq4_nl for example implementation.-*/-inline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) {- int8x16_t res;-- res[ 0] = a[b[ 0]];- res[ 1] = a[b[ 1]];- res[ 2] = a[b[ 2]];- res[ 3] = a[b[ 3]];- res[ 4] = a[b[ 4]];- res[ 5] = a[b[ 5]];- res[ 6] = a[b[ 6]];- res[ 7] = a[b[ 7]];- res[ 8] = a[b[ 8]];- res[ 9] = a[b[ 9]];- res[10] = a[b[10]];- res[11] = a[b[11]];- res[12] = a[b[12]];- res[13] = a[b[13]];- res[14] = a[b[14]];- res[15] = a[b[15]];-- return res;+// faster fp16 -> fp32 conversion using vector intrinsics+// WARNING: fp16 inf converts into 65536.0f. use this only for finite values.+inline static float32x4_t ggml_vec_xl_f16x4_finite(const ggml_fp16_t * x) {+ uint64_t tmp;+ memcpy(&tmp, x, sizeof(tmp));++ const uint64x2_t v_x = { tmp, tmp };+ const uint32x4_t v_xh = vec_unpackh((uint16x8_t)v_x);+ const uint32x4_t v_bits = vec_sl(vec_and(v_xh, vec_splats((uint32_t)0x7FFF)), 13);+ const float32x4_t v_abs = vec_mul((float32x4_t)v_bits, vec_splats(0x1.0p+112f));+ return (float32x4_t)vec_or((uint32x4_t)v_abs, vec_sl(vec_and(v_xh, vec_splats((uint32_t)0x8000)), 16));}inline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) {@@ -492,6 +485,11 @@ inline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) {return v_abo + v_abe;}+inline static int32x4_t vec_paddl_s16(int16x8_t a) {+ const int16x8_t v_ones = vec_splats((int16_t)1);+ return vec_mule(a, v_ones) + vec_mulo(a, v_ones);+}+/*** @see https://github.com/ggml-org/llama.cpp/pull/14037*/@@ -505,7 +503,11 @@ inline static int32_t vec_hsum_i32x4(int32x4_t v) {return v_temp[0] + v_temp[1];}-inline static int32x4_t ggml_vec_dot(int32x4_t acc, int8x16_t a, int8x16_t b) {+inline static int16x8_t ggml_vec_dot_s16(int16x8_t acc, int8x16_t a, int8x16_t b) {+ return vec_meadd(a, b, vec_moadd(a, b, acc));+}++inline static int32x4_t ggml_vec_dot_s32(int32x4_t acc, int8x16_t a, int8x16_t b) {const int16x8_t p = vec_mule(a, b) + vec_mulo(a, b);return acc + (vec_unpackh(p) + vec_unpackl(p));}