Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Fix quants for AARCH64 CPU inference
Justine Tunneydadb5052fb8f
diff --git a/llama.cpp/ggml-quants.c b/llama.cpp/ggml-quants.cindex b6d39088..d282d4c0 100644--- a/llama.cpp/ggml-quants.c+++ b/llama.cpp/ggml-quants.c@@ -441,6 +441,22 @@ inline static ggml_int8x16x4_t ggml_vld1q_s8_x4(const int8_t * ptr) {#define ggml_vld1q_s8_x4 vld1q_s8_x4#endif++#if !defined(__ARM_FEATURE_DOTPROD)++inline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b) {+ const int16x8_t p0 = vmull_s8(vget_low_s8 (a), vget_low_s8 (b));+ const int16x8_t p1 = vmull_s8(vget_high_s8(a), vget_high_s8(b));++ return vaddq_s32(acc, vaddq_s32(vpaddlq_s16(p0), vpaddlq_s16(p1)));+}++#else++#define ggml_vdotq_s32(a, b, c) vdotq_s32(a, b, c)++#endif+#endif#if defined(__ARM_NEON) || defined(__wasm_simd128__)diff --git a/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.incindex 25851025..c5ae1427 100644--- a/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc@@ -6,12 +6,10 @@ static void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void *const int nb = n / QK_K;#ifdef __ARM_NEON-const uint8x16_t m3 = vdupq_n_u8(0x3);const uint8x16_t m4 = vdupq_n_u8(0xF);-#if defined(__ARM_FEATURE_DOTPROD)- const int32x4_t vzero = vdupq_n_s32(0);-#endif++ const int32x4_t vzero = vdupq_n_s32(0);ggml_int8x16x2_t q2bytes;uint8_t aux[16];@@ -19,7 +17,6 @@ static void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void *float sum = 0;for (int i = 0; i < nb; ++i) {-const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);@@ -33,7 +30,7 @@ static void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void *const uint8x16_t mins = vshrq_n_u8(mins_and_scales, 4);const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);- const ggml_int16x8x2_t mins16 = {vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins)))};+ const ggml_int16x8x2_t mins16 = {{vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins)))}};const int32x4_t s0 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[0]), vget_low_s16 (q8sums.val[0])),vmull_s16(vget_high_s16(mins16.val[0]), vget_high_s16(q8sums.val[0])));const int32x4_t s1 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[1]), vget_low_s16 (q8sums.val[1])),@@ -45,20 +42,9 @@ static void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void *// We use this macro instead of a function call because for some reason// the code runs 2-3% slower, even if the function is declared inline-#if defined(__ARM_FEATURE_DOTPROD)#define MULTIPLY_ACCUM_WITH_SCALE(index)\- isum += vaddvq_s32(vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * aux[is+(index)];\- isum += vaddvq_s32(vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * aux[is+1+(index)];-#else-#define MULTIPLY_ACCUM_WITH_SCALE(index)\- {\- const int16x8_t p1 = vaddq_s16(vmull_s8(vget_low_s8 (q2bytes.val[0]), vget_low_s8 (q8bytes.val[0])),\- vmull_s8(vget_high_s8(q2bytes.val[0]), vget_high_s8(q8bytes.val[0])));\- const int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q2bytes.val[1]), vget_low_s8 (q8bytes.val[1])),\- vmull_s8(vget_high_s8(q2bytes.val[1]), vget_high_s8(q8bytes.val[1])));\- isum += vaddvq_s16(p1) * aux[is+(index)] + vaddvq_s16(p2) * aux[is+1+(index)];\- }-#endif+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * aux[is+(index)];\+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * aux[is+1+(index)];#define SHIFT_MULTIPLY_ACCUM_WITH_SCALE(shift, index)\q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\@@ -66,26 +52,23 @@ static void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void *q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[1], (shift)), m3));\MULTIPLY_ACCUM_WITH_SCALE((index));-for (int j = 0; j < QK_K/128; ++j) {-const ggml_uint8x16x2_t q2bits = ggml_vld1q_u8_x2(q2); q2 += 32;ggml_int8x16x2_t q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[0], m3));q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[1], m3));+MULTIPLY_ACCUM_WITH_SCALE(0);SHIFT_MULTIPLY_ACCUM_WITH_SCALE(2, 2);-SHIFT_MULTIPLY_ACCUM_WITH_SCALE(4, 4);-SHIFT_MULTIPLY_ACCUM_WITH_SCALE(6, 6);is += 8;}- sum += d * isum;+ sum += d * isum;}*s = sum;diff --git a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.incindex 389e3738..3019db38 100644--- a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc@@ -15,9 +15,7 @@ static void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void *uint32_t utmp[4];const uint8x16_t m3b = vdupq_n_u8(0x3);-#ifdef __ARM_FEATURE_DOTPRODconst int32x4_t vzero = vdupq_n_s32(0);-#endifconst uint8x16_t m0 = vdupq_n_u8(1);const uint8x16_t m1 = vshlq_n_u8(m0, 1);@@ -69,22 +67,11 @@ static void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void *q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 2), m3b)), vreinterpretq_s8_u8(q3h.val[2]));q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 2), m3b)), vreinterpretq_s8_u8(q3h.val[3]));-#if defined(__ARM_FEATURE_DOTPROD)- isum += vaddvq_s32(vdotq_s32(vzero, q3bytes.val[0], q8bytes_1.val[0])) * scale[0];- isum += vaddvq_s32(vdotq_s32(vzero, q3bytes.val[1], q8bytes_1.val[1])) * scale[1];- isum += vaddvq_s32(vdotq_s32(vzero, q3bytes.val[2], q8bytes_1.val[2])) * scale[2];- isum += vaddvq_s32(vdotq_s32(vzero, q3bytes.val[3], q8bytes_1.val[3])) * scale[3];-#else- int16x8_t p0 = vaddq_s16(vmull_s8(vget_low_s8 (q3bytes.val[0]), vget_low_s8 (q8bytes_1.val[0])),- vmull_s8(vget_high_s8(q3bytes.val[0]), vget_high_s8(q8bytes_1.val[0])));- int16x8_t p1 = vaddq_s16(vmull_s8(vget_low_s8 (q3bytes.val[1]), vget_low_s8 (q8bytes_1.val[1])),- vmull_s8(vget_high_s8(q3bytes.val[1]), vget_high_s8(q8bytes_1.val[1])));- int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q3bytes.val[2]), vget_low_s8 (q8bytes_1.val[2])),- vmull_s8(vget_high_s8(q3bytes.val[2]), vget_high_s8(q8bytes_1.val[2])));- int16x8_t p3 = vaddq_s16(vmull_s8(vget_low_s8 (q3bytes.val[3]), vget_low_s8 (q8bytes_1.val[3])),- vmull_s8(vget_high_s8(q3bytes.val[3]), vget_high_s8(q8bytes_1.val[3])));- isum += vaddvq_s16(p0) * scale[0] + vaddvq_s16(p1) * scale[1] + vaddvq_s16(p2) * scale[2] + vaddvq_s16(p3) * scale[3];-#endif+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_1.val[0])) * scale[0];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_1.val[1])) * scale[1];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_1.val[2])) * scale[2];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_1.val[3])) * scale[3];+scale += 4;q3h.val[0] = vbicq_u8(m2, qhbits.val[0]);@@ -97,22 +84,11 @@ static void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void *q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 6), m3b)), vreinterpretq_s8_u8(q3h.val[2]));q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 6), m3b)), vreinterpretq_s8_u8(q3h.val[3]));-#if defined(__ARM_FEATURE_DOTPROD)- isum += vaddvq_s32(vdotq_s32(vzero, q3bytes.val[0], q8bytes_2.val[0])) * scale[0];- isum += vaddvq_s32(vdotq_s32(vzero, q3bytes.val[1], q8bytes_2.val[1])) * scale[1];- isum += vaddvq_s32(vdotq_s32(vzero, q3bytes.val[2], q8bytes_2.val[2])) * scale[2];- isum += vaddvq_s32(vdotq_s32(vzero, q3bytes.val[3], q8bytes_2.val[3])) * scale[3];-#else- p0 = vaddq_s16(vmull_s8(vget_low_s8 (q3bytes.val[0]), vget_low_s8 (q8bytes_2.val[0])),- vmull_s8(vget_high_s8(q3bytes.val[0]), vget_high_s8(q8bytes_2.val[0])));- p1 = vaddq_s16(vmull_s8(vget_low_s8 (q3bytes.val[1]), vget_low_s8 (q8bytes_2.val[1])),- vmull_s8(vget_high_s8(q3bytes.val[1]), vget_high_s8(q8bytes_2.val[1])));- p2 = vaddq_s16(vmull_s8(vget_low_s8 (q3bytes.val[2]), vget_low_s8 (q8bytes_2.val[2])),- vmull_s8(vget_high_s8(q3bytes.val[2]), vget_high_s8(q8bytes_2.val[2])));- p3 = vaddq_s16(vmull_s8(vget_low_s8 (q3bytes.val[3]), vget_low_s8 (q8bytes_2.val[3])),- vmull_s8(vget_high_s8(q3bytes.val[3]), vget_high_s8(q8bytes_2.val[3])));- isum += vaddvq_s16(p0) * scale[0] + vaddvq_s16(p1) * scale[1] + vaddvq_s16(p2) * scale[2] + vaddvq_s16(p3) * scale[3];-#endif+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_2.val[0])) * scale[0];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_2.val[1])) * scale[1];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_2.val[2])) * scale[2];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_2.val[3])) * scale[3];+scale += 4;if (j == 0) {diff --git a/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q4_0_q8_0.incindex 4cf26245..dd7e2272 100644--- a/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc+++ b/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc@@ -43,32 +43,12 @@ static void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, const void * restrconst int8x16_t v1_1l = vld1q_s8(y1->qs);const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);-#if defined(__ARM_FEATURE_DOTPROD)// dot product into int32x4_t- const int32x4_t p_0 = vdotq_s32(vdotq_s32(vdupq_n_s32(0), v0_0ls, v1_0l), v0_0hs, v1_0h);- const int32x4_t p_1 = vdotq_s32(vdotq_s32(vdupq_n_s32(0), v0_1ls, v1_1l), v0_1hs, v1_1h);+ const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0ls, v1_0l), v0_0hs, v1_0h);+ const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1ls, v1_1l), v0_1hs, v1_1h);sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));-#else- const int16x8_t pl0l = vmull_s8(vget_low_s8 (v0_0ls), vget_low_s8 (v1_0l));- const int16x8_t pl0h = vmull_s8(vget_high_s8(v0_0ls), vget_high_s8(v1_0l));- const int16x8_t ph0l = vmull_s8(vget_low_s8 (v0_0hs), vget_low_s8 (v1_0h));- const int16x8_t ph0h = vmull_s8(vget_high_s8(v0_0hs), vget_high_s8(v1_0h));-- const int16x8_t pl1l = vmull_s8(vget_low_s8 (v0_1ls), vget_low_s8 (v1_1l));- const int16x8_t pl1h = vmull_s8(vget_high_s8(v0_1ls), vget_high_s8(v1_1l));- const int16x8_t ph1l = vmull_s8(vget_low_s8 (v0_1hs), vget_low_s8 (v1_1h));- const int16x8_t ph1h = vmull_s8(vget_high_s8(v0_1hs), vget_high_s8(v1_1h));-- const int32x4_t pl0 = vaddq_s32(vpaddlq_s16(pl0l), vpaddlq_s16(pl0h));- const int32x4_t ph0 = vaddq_s32(vpaddlq_s16(ph0l), vpaddlq_s16(ph0h));- const int32x4_t pl1 = vaddq_s32(vpaddlq_s16(pl1l), vpaddlq_s16(pl1h));- const int32x4_t ph1 = vaddq_s32(vpaddlq_s16(ph1l), vpaddlq_s16(ph1h));-- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(pl0, ph0)), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(pl1, ph1)), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));-#endif}*s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);diff --git a/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc b/llama.cpp/ggml_vec_dot_q4_1_q8_1.incindex 12ec2787..d48aff36 100644--- a/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc+++ b/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc@@ -41,32 +41,12 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, const void *const int8x16_t v1_1l = vld1q_s8(y1->qs);const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);-#if defined(__ARM_FEATURE_DOTPROD)// dot product into int32x4_t- const int32x4_t p_0 = vdotq_s32(vdotq_s32(vdupq_n_s32(0), v0_0l, v1_0l), v0_0h, v1_0h);- const int32x4_t p_1 = vdotq_s32(vdotq_s32(vdupq_n_s32(0), v0_1l, v1_1l), v0_1h, v1_1h);+ const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0l, v1_0l), v0_0h, v1_0h);+ const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1l, v1_1l), v0_1h, v1_1h);sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*y0->d);sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*y1->d);-#else- const int16x8_t pl0l = vmull_s8(vget_low_s8 (v0_0l), vget_low_s8 (v1_0l));- const int16x8_t pl0h = vmull_s8(vget_high_s8(v0_0l), vget_high_s8(v1_0l));- const int16x8_t ph0l = vmull_s8(vget_low_s8 (v0_0h), vget_low_s8 (v1_0h));- const int16x8_t ph0h = vmull_s8(vget_high_s8(v0_0h), vget_high_s8(v1_0h));-- const int16x8_t pl1l = vmull_s8(vget_low_s8 (v0_1l), vget_low_s8 (v1_1l));- const int16x8_t pl1h = vmull_s8(vget_high_s8(v0_1l), vget_high_s8(v1_1l));- const int16x8_t ph1l = vmull_s8(vget_low_s8 (v0_1h), vget_low_s8 (v1_1h));- const int16x8_t ph1h = vmull_s8(vget_high_s8(v0_1h), vget_high_s8(v1_1h));-- const int32x4_t pl0 = vaddq_s32(vpaddlq_s16(pl0l), vpaddlq_s16(pl0h));- const int32x4_t ph0 = vaddq_s32(vpaddlq_s16(ph0l), vpaddlq_s16(ph0h));- const int32x4_t pl1 = vaddq_s32(vpaddlq_s16(pl1l), vpaddlq_s16(pl1h));- const int32x4_t ph1 = vaddq_s32(vpaddlq_s16(ph1l), vpaddlq_s16(ph1h));-- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(pl0, ph0)), GGML_FP16_TO_FP32(x0->d)*y0->d);- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(pl1, ph1)), GGML_FP16_TO_FP32(x1->d)*y1->d);-#endif}*s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs;diff --git a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.incindex cfa128fa..e0933031 100644--- a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc@@ -13,11 +13,8 @@ static void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void *uint32_t utmp[4];#ifdef __ARM_NEON-const uint8x16_t m4b = vdupq_n_u8(0xf);-#ifdef __ARM_FEATURE_DOTPRODconst int32x4_t mzero = vdupq_n_s32(0);-#endifggml_int8x16x2_t q4bytes;ggml_int8x16x2_t q8bytes;@@ -54,44 +51,22 @@ static void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void *int32_t sumi2 = 0;for (int j = 0; j < QK_K/64; ++j) {-const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4); q4 += 32;-#ifdef __ARM_FEATURE_DOTPRODq8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));- const int32x4_t p1 = vdotq_s32(vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);+ const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);sumi1 += vaddvq_s32(p1) * scales[2*j+0];q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));- const int32x4_t p2 = vdotq_s32(vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);+ const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);sumi2 += vaddvq_s32(p2) * scales[2*j+1];-#else- q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;- q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));- q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));- const int16x8_t p0 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[0]), vget_low_s8 (q8bytes.val[0])),- vmull_s8(vget_high_s8(q4bytes.val[0]), vget_high_s8(q8bytes.val[0])));- const int16x8_t p1 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[1]), vget_low_s8 (q8bytes.val[1])),- vmull_s8(vget_high_s8(q4bytes.val[1]), vget_high_s8(q8bytes.val[1])));- sumi1 += vaddvq_s16(vaddq_s16(p0, p1)) * scales[2*j+0];-- q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;- q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));- q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));- const int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[0]), vget_low_s8 (q8bytes.val[0])),- vmull_s8(vget_high_s8(q4bytes.val[0]), vget_high_s8(q8bytes.val[0])));- const int16x8_t p3 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[1]), vget_low_s8 (q8bytes.val[1])),- vmull_s8(vget_high_s8(q4bytes.val[1]), vget_high_s8(q8bytes.val[1])));- sumi2 += vaddvq_s16(vaddq_s16(p2, p3)) * scales[2*j+1];--#endif}sumf += d * (sumi1 + sumi2);diff --git a/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q5_0_q8_0.incindex 430a9f16..54fae8ce 100644--- a/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc+++ b/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc@@ -68,32 +68,12 @@ static void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, const void *const int8x16_t v1_1l = vld1q_s8(y1->qs);const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);-#if defined(__ARM_FEATURE_DOTPROD)sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(- vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),- vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(- vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),- vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));-#else- const int16x8_t pl0l = vmull_s8(vget_low_s8 (v0_0lf), vget_low_s8 (v1_0l));- const int16x8_t pl0h = vmull_s8(vget_high_s8(v0_0lf), vget_high_s8(v1_0l));- const int16x8_t ph0l = vmull_s8(vget_low_s8 (v0_0hf), vget_low_s8 (v1_0h));- const int16x8_t ph0h = vmull_s8(vget_high_s8(v0_0hf), vget_high_s8(v1_0h));-- const int16x8_t pl1l = vmull_s8(vget_low_s8 (v0_1lf), vget_low_s8 (v1_1l));- const int16x8_t pl1h = vmull_s8(vget_high_s8(v0_1lf), vget_high_s8(v1_1l));- const int16x8_t ph1l = vmull_s8(vget_low_s8 (v0_1hf), vget_low_s8 (v1_1h));- const int16x8_t ph1h = vmull_s8(vget_high_s8(v0_1hf), vget_high_s8(v1_1h));-- const int32x4_t pl0 = vaddq_s32(vpaddlq_s16(pl0l), vpaddlq_s16(pl0h));- const int32x4_t ph0 = vaddq_s32(vpaddlq_s16(ph0l), vpaddlq_s16(ph0h));- const int32x4_t pl1 = vaddq_s32(vpaddlq_s16(pl1l), vpaddlq_s16(pl1h));- const int32x4_t ph1 = vaddq_s32(vpaddlq_s16(ph1l), vpaddlq_s16(ph1h));-- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(pl0, ph0)), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(pl1, ph1)), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));-#endif+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));}*s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);diff --git a/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc b/llama.cpp/ggml_vec_dot_q5_1_q8_1.incindex 8d6932fc..8d79b71a 100644--- a/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc+++ b/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc@@ -74,32 +74,12 @@ static void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void *const int8x16_t v1_1l = vld1q_s8(y1->qs);const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);-#if defined(__ARM_FEATURE_DOTPROD)sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(- vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),- vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*y0->d);+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),+ ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_FP16_TO_FP32(x0->d)*y0->d);sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(- vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),- vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*y1->d);-#else- const int16x8_t pl0l = vmull_s8(vget_low_s8 (v0_0lf), vget_low_s8 (v1_0l));- const int16x8_t pl0h = vmull_s8(vget_high_s8(v0_0lf), vget_high_s8(v1_0l));- const int16x8_t ph0l = vmull_s8(vget_low_s8 (v0_0hf), vget_low_s8 (v1_0h));- const int16x8_t ph0h = vmull_s8(vget_high_s8(v0_0hf), vget_high_s8(v1_0h));-- const int16x8_t pl1l = vmull_s8(vget_low_s8 (v0_1lf), vget_low_s8 (v1_1l));- const int16x8_t pl1h = vmull_s8(vget_high_s8(v0_1lf), vget_high_s8(v1_1l));- const int16x8_t ph1l = vmull_s8(vget_low_s8 (v0_1hf), vget_low_s8 (v1_1h));- const int16x8_t ph1h = vmull_s8(vget_high_s8(v0_1hf), vget_high_s8(v1_1h));-- const int32x4_t pl0 = vaddq_s32(vpaddlq_s16(pl0l), vpaddlq_s16(pl0h));- const int32x4_t ph0 = vaddq_s32(vpaddlq_s16(ph0l), vpaddlq_s16(ph0h));- const int32x4_t pl1 = vaddq_s32(vpaddlq_s16(pl1l), vpaddlq_s16(pl1h));- const int32x4_t ph1 = vaddq_s32(vpaddlq_s16(ph1l), vpaddlq_s16(ph1h));-- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(pl0, ph0)), GGML_FP16_TO_FP32(x0->d)*y0->d);- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(pl1, ph1)), GGML_FP16_TO_FP32(x1->d)*y1->d);-#endif+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),+ ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*y1->d);}*s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs0 + summs1;diff --git a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.incindex e1fae962..bd618e32 100644--- a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc@@ -14,13 +14,10 @@ static void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void *#ifdef __ARM_NEON-const uint8x16_t m4b = vdupq_n_u8(0xf);const uint8x16_t mone = vdupq_n_u8(1);const uint8x16_t mtwo = vdupq_n_u8(2);-#if defined(__ARM_FEATURE_DOTPROD)const int32x4_t mzero = vdupq_n_s32(0);-#endifggml_int8x16x4_t q5bytes;@@ -75,28 +72,11 @@ static void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void *q5bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[0], 4), q5h.val[2]));q5bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[1], 4), q5h.val[3]));-#if defined(__ARM_FEATURE_DOTPROD)-- sumi += vaddvq_s32(vdotq_s32(vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]), q5bytes.val[1], q8bytes.val[1])) * *scales++;- sumi += vaddvq_s32(vdotq_s32(vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]), q5bytes.val[3], q8bytes.val[3])) * *scales++;-#else-- const int16x8_t p0 = vaddq_s16(vmull_s8(vget_low_s8 (q5bytes.val[0]), vget_low_s8 (q8bytes.val[0])),- vmull_s8(vget_high_s8(q5bytes.val[0]), vget_high_s8(q8bytes.val[0])));- const int16x8_t p1 = vaddq_s16(vmull_s8(vget_low_s8 (q5bytes.val[1]), vget_low_s8 (q8bytes.val[1])),- vmull_s8(vget_high_s8(q5bytes.val[1]), vget_high_s8(q8bytes.val[1])));- sumi += vaddvq_s16(vaddq_s16(p0, p1)) * *scales++;-- const int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q5bytes.val[2]), vget_low_s8 (q8bytes.val[2])),- vmull_s8(vget_high_s8(q5bytes.val[2]), vget_high_s8(q8bytes.val[2])));- const int16x8_t p3 = vaddq_s16(vmull_s8(vget_low_s8 (q5bytes.val[3]), vget_low_s8 (q8bytes.val[3])),- vmull_s8(vget_high_s8(q5bytes.val[3]), vget_high_s8(q8bytes.val[3])));- sumi += vaddvq_s16(vaddq_s16(p2, p3)) * *scales++;-#endif+ sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]), q5bytes.val[1], q8bytes.val[1])) * *scales++;+ sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]), q5bytes.val[3], q8bytes.val[3])) * *scales++;}sumf += d * sumi - dmin * sumi_mins;-}*s = sumf;diff --git a/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.incindex 775a6be0..e5d8f6c9 100644--- a/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc@@ -7,13 +7,10 @@ static void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void *const int nb = n / QK_K;#ifdef __ARM_NEON-float sum = 0;const uint8x16_t m4b = vdupq_n_u8(0xF);-#if defined(__ARM_FEATURE_DOTPROD)const int32x4_t vzero = vdupq_n_s32(0);-#endif//const int8x16_t m32s = vdupq_n_s8(32);const uint8x16_t mone = vdupq_n_u8(3);@@ -33,7 +30,7 @@ static void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void *const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);const int8x16_t scales = vld1q_s8(scale);- const ggml_int16x8x2_t q6scales = {vmovl_s8(vget_low_s8(scales)), vmovl_s8(vget_high_s8(scales))};+ const ggml_int16x8x2_t q6scales = {{vmovl_s8(vget_low_s8(scales)), vmovl_s8(vget_high_s8(scales))}};const int32x4_t prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[0]), vget_low_s16 (q6scales.val[0])),vmull_s16(vget_high_s16(q8sums.val[0]), vget_high_s16(q6scales.val[0]))),@@ -65,31 +62,13 @@ static void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void *q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2]));q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3]));-#if defined(__ARM_FEATURE_DOTPROD)+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];- isum += vaddvq_s32(vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];scale += 4;-#else-- int16x8_t p0 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[0]), vget_low_s8 (q8bytes.val[0])),- vmull_s8(vget_high_s8(q6bytes.val[0]), vget_high_s8(q8bytes.val[0])));- int16x8_t p1 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[1]), vget_low_s8 (q8bytes.val[1])),- vmull_s8(vget_high_s8(q6bytes.val[1]), vget_high_s8(q8bytes.val[1])));- isum += vaddvq_s16(p0) * scale[0] + vaddvq_s16(p1) * scale[1];- scale += 2;-- int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[2]), vget_low_s8 (q8bytes.val[2])),- vmull_s8(vget_high_s8(q6bytes.val[2]), vget_high_s8(q8bytes.val[2])));- int16x8_t p3 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[3]), vget_low_s8 (q8bytes.val[3])),- vmull_s8(vget_high_s8(q6bytes.val[3]), vget_high_s8(q8bytes.val[3])));- isum += vaddvq_s16(p2) * scale[0] + vaddvq_s16(p3) * scale[1];- scale += 2;-#endif-q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;shifted = vshrq_n_u8(qhbits.val[0], 4);@@ -110,34 +89,11 @@ static void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void *q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2]));q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3]));-#if defined(__ARM_FEATURE_DOTPROD)-- isum += vaddvq_s32(vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];scale += 4;-- //for (int l = 0; l < 4; ++l) {- // const int32x4_t p = vdotq_s32(vzero, q6bytes.val[l], q8bytes.val[l]);- // isum += vaddvq_s32(p) * *scale++;- //}-#else- p0 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[0]), vget_low_s8 (q8bytes.val[0])),- vmull_s8(vget_high_s8(q6bytes.val[0]), vget_high_s8(q8bytes.val[0])));- p1 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[1]), vget_low_s8 (q8bytes.val[1])),- vmull_s8(vget_high_s8(q6bytes.val[1]), vget_high_s8(q8bytes.val[1])));- isum += vaddvq_s16(p0) * scale[0] + vaddvq_s16(p1) * scale[1];- scale += 2;-- p2 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[2]), vget_low_s8 (q8bytes.val[2])),- vmull_s8(vget_high_s8(q6bytes.val[2]), vget_high_s8(q8bytes.val[2])));- p3 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[3]), vget_low_s8 (q8bytes.val[3])),- vmull_s8(vget_high_s8(q6bytes.val[3]), vget_high_s8(q8bytes.val[3])));- isum += vaddvq_s16(p2) * scale[0] + vaddvq_s16(p3) * scale[1];- scale += 2;-#endif-}//sum += isum * d_all * y[i].d;sum += d_all * y[i].d * (isum - 32 * isum_mins);diff --git a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q8_0_q8_0.incindex e6dd15b8..1800acfe 100644--- a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc+++ b/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc@@ -30,34 +30,13 @@ static void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void *const int8x16_t y1_0 = vld1q_s8(y1->qs);const int8x16_t y1_1 = vld1q_s8(y1->qs + 16);-#if defined(__ARM_FEATURE_DOTPROD)sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(- vdotq_s32(vdupq_n_s32(0), x0_0, y0_0),- vdotq_s32(vdupq_n_s32(0), x0_1, y0_1))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));+ ggml_vdotq_s32(vdupq_n_s32(0), x0_0, y0_0),+ ggml_vdotq_s32(vdupq_n_s32(0), x0_1, y0_1))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(- vdotq_s32(vdupq_n_s32(0), x1_0, y1_0),- vdotq_s32(vdupq_n_s32(0), x1_1, y1_1))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));--#else- const int16x8_t p0_0 = vmull_s8(vget_low_s8 (x0_0), vget_low_s8 (y0_0));- const int16x8_t p0_1 = vmull_s8(vget_high_s8(x0_0), vget_high_s8(y0_0));- const int16x8_t p0_2 = vmull_s8(vget_low_s8 (x0_1), vget_low_s8 (y0_1));- const int16x8_t p0_3 = vmull_s8(vget_high_s8(x0_1), vget_high_s8(y0_1));-- const int16x8_t p1_0 = vmull_s8(vget_low_s8 (x1_0), vget_low_s8 (y1_0));- const int16x8_t p1_1 = vmull_s8(vget_high_s8(x1_0), vget_high_s8(y1_0));- const int16x8_t p1_2 = vmull_s8(vget_low_s8 (x1_1), vget_low_s8 (y1_1));- const int16x8_t p1_3 = vmull_s8(vget_high_s8(x1_1), vget_high_s8(y1_1));-- const int32x4_t p0 = vaddq_s32(vpaddlq_s16(p0_0), vpaddlq_s16(p0_1));- const int32x4_t p1 = vaddq_s32(vpaddlq_s16(p0_2), vpaddlq_s16(p0_3));- const int32x4_t p2 = vaddq_s32(vpaddlq_s16(p1_0), vpaddlq_s16(p1_1));- const int32x4_t p3 = vaddq_s32(vpaddlq_s16(p1_2), vpaddlq_s16(p1_3));-- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(p0, p1)), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(p2, p3)), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));-#endif+ ggml_vdotq_s32(vdupq_n_s32(0), x1_0, y1_0),+ ggml_vdotq_s32(vdupq_n_s32(0), x1_1, y1_1))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));}*s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);