Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Slightly speed up LLaVA runtime dispatch on Intel
Justine Tunneyb4dea04db7ef
diff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.hindex 5ec18a50..298a0c89 100644--- a/llama.cpp/ggml-impl.h+++ b/llama.cpp/ggml-impl.h@@ -88,6 +88,9 @@ extern "C" {#include <riscv_vector.h>#endif+#define GGML_FP16_TO_FP32__F16C(x) _cvtsh_ss(x)+#define GGML_FP32_TO_FP16__F16C(x) _cvtss_sh(x, 0)+#ifdef __F16C__#ifdef _MSC_VERdiff --git a/llama.cpp/ggml-quants.c b/llama.cpp/ggml-quants.cindex 91e31d23..96abc22a 100644--- a/llama.cpp/ggml-quants.c+++ b/llama.cpp/ggml-quants.c@@ -80,6 +80,70 @@ inline static int32_t vaddvq_s32(int32x4_t v) {#ifdef __x86_64__+#pragma GCC push_options+#pragma GCC target("avx2")+// Unpack 32 4-bit fields into 32 bytes+// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval+static inline __m256i bytes_from_nibbles_32_avx2(const uint8_t * rsi)+{+ const __m128i tmp = _mm_loadu_si128((const __m128i *)rsi);+ const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);+ const __m256i lowMask = _mm256_set1_epi8( 0xF );+ return _mm256_and_si256(lowMask, bytes);+}+#pragma GCC pop_options++#pragma GCC push_options+#pragma GCC target("avx")+// Unpack 32 4-bit fields into 32 bytes+// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval+static inline __m256i bytes_from_nibbles_32_avx(const uint8_t * rsi)+{+ // Load 16 bytes from memory+ __m128i tmpl = _mm_loadu_si128((const __m128i *)rsi);+ __m128i tmph = _mm_srli_epi16(tmpl, 4);+ const __m128i lowMask = _mm_set1_epi8(0xF);+ tmpl = _mm_and_si128(lowMask, tmpl);+ tmph = _mm_and_si128(lowMask, tmph);+ return MM256_SET_M128I(tmph, tmpl);+}+#pragma GCC pop_options++#pragma GCC push_options+#pragma GCC target("avx2")+// spread 32 bits to 32 bytes { 0x00, 0xFF }+static inline __m256i bytes_from_bits_32_avx2(const uint8_t * x) {+ uint32_t x32;+ memcpy(&x32, x, sizeof(uint32_t));+ const __m256i shuf_mask = _mm256_set_epi64x(+ 0x0303030303030303, 0x0202020202020202,+ 0x0101010101010101, 0x0000000000000000);+ __m256i bytes = _mm256_shuffle_epi8(_mm256_set1_epi32(x32), shuf_mask);+ const __m256i bit_mask = _mm256_set1_epi64x(0x7fbfdfeff7fbfdfe);+ bytes = _mm256_or_si256(bytes, bit_mask);+ return _mm256_cmpeq_epi8(bytes, _mm256_set1_epi64x(-1));+}+#pragma GCC pop_options++#pragma GCC push_options+#pragma GCC target("avx")+// spread 32 bits to 32 bytes { 0x00, 0xFF }+static inline __m256i bytes_from_bits_32_avx(const uint8_t * x) {+ uint32_t x32;+ memcpy(&x32, x, sizeof(uint32_t));+ const __m128i shuf_maskl = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);+ const __m128i shuf_maskh = _mm_set_epi64x(0x0303030303030303, 0x0202020202020202);+ __m128i bytesl = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskl);+ __m128i bytesh = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskh);+ const __m128i bit_mask = _mm_set1_epi64x(0x7fbfdfeff7fbfdfe);+ bytesl = _mm_or_si128(bytesl, bit_mask);+ bytesh = _mm_or_si128(bytesh, bit_mask);+ bytesl = _mm_cmpeq_epi8(bytesl, _mm_set1_epi64x(-1));+ bytesh = _mm_cmpeq_epi8(bytesh, _mm_set1_epi64x(-1));+ return MM256_SET_M128I(bytesh, bytesl);+}+#pragma GCC pop_options+// add int16_t pairwise and return as float vector__attribute__((__target__("avx")))static inline __m256 sum_i16_pairs_float_avx(const __m128i xh, const __m128i xl) {@@ -193,28 +257,6 @@ static inline int hsum_i32_4(const __m128i a) {}#if defined(__AVX2__) || defined(__AVX512F__)-// spread 32 bits to 32 bytes { 0x00, 0xFF }-static inline __m256i bytes_from_bits_32(const uint8_t * x) {- uint32_t x32;- memcpy(&x32, x, sizeof(uint32_t));- const __m256i shuf_mask = _mm256_set_epi64x(- 0x0303030303030303, 0x0202020202020202,- 0x0101010101010101, 0x0000000000000000);- __m256i bytes = _mm256_shuffle_epi8(_mm256_set1_epi32(x32), shuf_mask);- const __m256i bit_mask = _mm256_set1_epi64x(0x7fbfdfeff7fbfdfe);- bytes = _mm256_or_si256(bytes, bit_mask);- return _mm256_cmpeq_epi8(bytes, _mm256_set1_epi64x(-1));-}--// Unpack 32 4-bit fields into 32 bytes-// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval-static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)-{- const __m128i tmp = _mm_loadu_si128((const __m128i *)rsi);- const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);- const __m256i lowMask = _mm256_set1_epi8( 0xF );- return _mm256_and_si256(lowMask, bytes);-}static inline __m128i packNibbles( __m256i bytes ){@@ -237,34 +279,6 @@ static inline __m128i packNibbles( __m256i bytes )#endif}#elif defined(__AVX__)-// spread 32 bits to 32 bytes { 0x00, 0xFF }-static inline __m256i bytes_from_bits_32(const uint8_t * x) {- uint32_t x32;- memcpy(&x32, x, sizeof(uint32_t));- const __m128i shuf_maskl = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);- const __m128i shuf_maskh = _mm_set_epi64x(0x0303030303030303, 0x0202020202020202);- __m128i bytesl = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskl);- __m128i bytesh = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskh);- const __m128i bit_mask = _mm_set1_epi64x(0x7fbfdfeff7fbfdfe);- bytesl = _mm_or_si128(bytesl, bit_mask);- bytesh = _mm_or_si128(bytesh, bit_mask);- bytesl = _mm_cmpeq_epi8(bytesl, _mm_set1_epi64x(-1));- bytesh = _mm_cmpeq_epi8(bytesh, _mm_set1_epi64x(-1));- return MM256_SET_M128I(bytesh, bytesl);-}--// Unpack 32 4-bit fields into 32 bytes-// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval-static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)-{- // Load 16 bytes from memory- __m128i tmpl = _mm_loadu_si128((const __m128i *)rsi);- __m128i tmph = _mm_srli_epi16(tmpl, 4);- const __m128i lowMask = _mm_set1_epi8(0xF);- tmpl = _mm_and_si128(lowMask, tmpl);- tmph = _mm_and_si128(lowMask, tmph);- return MM256_SET_M128I(tmph, tmpl);-}static inline __m128i packNibbles( __m128i bytes1, __m128i bytes2 ){@@ -2339,315 +2353,50 @@ static inline __m128i get_scale_shuffle(int i) {}#endif-void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, const void * restrict vx, const void * restrict vy) {- const int qk = QK8_0;- const int nb = n / qk;-- assert(n % qk == 0);-- const block_q4_0 * restrict x = vx;- const block_q8_0 * restrict y = vy;--#if defined(__ARM_NEON)- float32x4_t sumv0 = vdupq_n_f32(0.0f);- float32x4_t sumv1 = vdupq_n_f32(0.0f);-- assert(nb % 2 == 0); // TODO: handle odd nb-- for (int i = 0; i < nb; i += 2) {- const block_q4_0 * restrict x0 = &x[i + 0];- const block_q4_0 * restrict x1 = &x[i + 1];- const block_q8_0 * restrict y0 = &y[i + 0];- const block_q8_0 * restrict y1 = &y[i + 1];-- const uint8x16_t m4b = vdupq_n_u8(0x0F);- const int8x16_t s8b = vdupq_n_s8(0x8);-- const uint8x16_t v0_0 = vld1q_u8(x0->qs);- const uint8x16_t v0_1 = vld1q_u8(x1->qs);-- // 4-bit -> 8-bit- const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));- const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));- const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));- const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));-- // sub 8- const int8x16_t v0_0ls = vsubq_s8(v0_0l, s8b);- const int8x16_t v0_0hs = vsubq_s8(v0_0h, s8b);- const int8x16_t v0_1ls = vsubq_s8(v0_1l, s8b);- const int8x16_t v0_1hs = vsubq_s8(v0_1h, s8b);-- // load y- const int8x16_t v1_0l = vld1q_s8(y0->qs);- const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);- const int8x16_t v1_1l = vld1q_s8(y1->qs);- const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);--#if defined(__ARM_FEATURE_DOTPROD)- // dot product into int32x4_t- const int32x4_t p_0 = vdotq_s32(vdotq_s32(vdupq_n_s32(0), v0_0ls, v1_0l), v0_0hs, v1_0h);- const int32x4_t p_1 = vdotq_s32(vdotq_s32(vdupq_n_s32(0), v0_1ls, v1_1l), v0_1hs, v1_1h);-- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));-#else- const int16x8_t pl0l = vmull_s8(vget_low_s8 (v0_0ls), vget_low_s8 (v1_0l));- const int16x8_t pl0h = vmull_s8(vget_high_s8(v0_0ls), vget_high_s8(v1_0l));- const int16x8_t ph0l = vmull_s8(vget_low_s8 (v0_0hs), vget_low_s8 (v1_0h));- const int16x8_t ph0h = vmull_s8(vget_high_s8(v0_0hs), vget_high_s8(v1_0h));-- const int16x8_t pl1l = vmull_s8(vget_low_s8 (v0_1ls), vget_low_s8 (v1_1l));- const int16x8_t pl1h = vmull_s8(vget_high_s8(v0_1ls), vget_high_s8(v1_1l));- const int16x8_t ph1l = vmull_s8(vget_low_s8 (v0_1hs), vget_low_s8 (v1_1h));- const int16x8_t ph1h = vmull_s8(vget_high_s8(v0_1hs), vget_high_s8(v1_1h));-- const int32x4_t pl0 = vaddq_s32(vpaddlq_s16(pl0l), vpaddlq_s16(pl0h));- const int32x4_t ph0 = vaddq_s32(vpaddlq_s16(ph0l), vpaddlq_s16(ph0h));- const int32x4_t pl1 = vaddq_s32(vpaddlq_s16(pl1l), vpaddlq_s16(pl1h));- const int32x4_t ph1 = vaddq_s32(vpaddlq_s16(ph1l), vpaddlq_s16(ph1h));+#ifdef __x86_64__+#define ggml_vec_dot_q4_0_q8_0 ggml_vec_dot_q4_0_q8_0_default+#include "ggml_vec_dot_q4_0_q8_0.inc"+#undef ggml_vec_dot_q4_0_q8_0- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(pl0, ph0)), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(pl1, ph1)), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));+#define ggml_vec_dot_q4_0_q8_0 __attribute__((__target__("avx"))) ggml_vec_dot_q4_0_q8_0_avx+#ifndef __AVX__+#define __UNDEF_AVX__+#define __AVX__#endif- }-- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);-#elif defined(__AVX2__)- // Initialize accumulator with zeros- __m256 acc = _mm256_setzero_ps();-- // Main loop- for (int i = 0; i < nb; ++i) {- /* Compute combined scale for the block */- const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );-- __m256i bx = bytes_from_nibbles_32(x[i].qs);-- // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.- const __m256i off = _mm256_set1_epi8( 8 );- bx = _mm256_sub_epi8( bx, off );-- __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);-- const __m256 q = mul_sum_i8_pairs_float_avx2(bx, by);-- /* Multiply q with scale and accumulate */- acc = _mm256_fmadd_ps( d, q, acc );- }-- *s = hsum_float_8(acc);-#elif defined(__AVX__)- // Initialize accumulator with zeros- __m256 acc = _mm256_setzero_ps();-- // Main loop- for (int i = 0; i < nb; ++i) {- // Compute combined scale for the block- const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );-- const __m128i lowMask = _mm_set1_epi8(0xF);- const __m128i off = _mm_set1_epi8(8);-- const __m128i tmp = _mm_loadu_si128((const __m128i *)x[i].qs);-- __m128i bx = _mm_and_si128(lowMask, tmp);- __m128i by = _mm_loadu_si128((const __m128i *)y[i].qs);- bx = _mm_sub_epi8(bx, off);- const __m128i i32_0 = mul_sum_i8_pairs(bx, by);-- bx = _mm_and_si128(lowMask, _mm_srli_epi64(tmp, 4));- by = _mm_loadu_si128((const __m128i *)(y[i].qs + 16));- bx = _mm_sub_epi8(bx, off);- const __m128i i32_1 = mul_sum_i8_pairs(bx, by);-- // Convert int32_t to float- __m256 p = _mm256_cvtepi32_ps(MM256_SET_M128I(i32_0, i32_1));-- // Apply the scale, and accumulate- acc = _mm256_add_ps(_mm256_mul_ps( d, p ), acc);- }-- *s = hsum_float_8(acc);-#elif defined(__SSSE3__)- // set constants- const __m128i lowMask = _mm_set1_epi8(0xF);- const __m128i off = _mm_set1_epi8(8);-- // Initialize accumulator with zeros- __m128 acc_0 = _mm_setzero_ps();- __m128 acc_1 = _mm_setzero_ps();- __m128 acc_2 = _mm_setzero_ps();- __m128 acc_3 = _mm_setzero_ps();-- // First round without accumulation- {- _mm_prefetch(&x[0] + sizeof(block_q4_0), _MM_HINT_T0);- _mm_prefetch(&y[0] + sizeof(block_q8_0), _MM_HINT_T0);-- // Compute combined scale for the block 0 and 1- const __m128 d_0_1 = _mm_set1_ps( GGML_FP16_TO_FP32(x[0].d) * GGML_FP16_TO_FP32(y[0].d) );-- const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[0].qs);-- __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);- __m128i by_0 = _mm_loadu_si128((const __m128i *)y[0].qs);- bx_0 = _mm_sub_epi8(bx_0, off);- const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);-- __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));- __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[0].qs + 16));- bx_1 = _mm_sub_epi8(bx_1, off);- const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);-- _mm_prefetch(&x[1] + sizeof(block_q4_0), _MM_HINT_T0);- _mm_prefetch(&y[1] + sizeof(block_q8_0), _MM_HINT_T0);-- // Compute combined scale for the block 2 and 3- const __m128 d_2_3 = _mm_set1_ps( GGML_FP16_TO_FP32(x[1].d) * GGML_FP16_TO_FP32(y[1].d) );-- const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[1].qs);-- __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);- __m128i by_2 = _mm_loadu_si128((const __m128i *)y[1].qs);- bx_2 = _mm_sub_epi8(bx_2, off);- const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);-- __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));- __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[1].qs + 16));- bx_3 = _mm_sub_epi8(bx_3, off);- const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);-- // Convert int32_t to float- __m128 p0 = _mm_cvtepi32_ps(i32_0);- __m128 p1 = _mm_cvtepi32_ps(i32_1);- __m128 p2 = _mm_cvtepi32_ps(i32_2);- __m128 p3 = _mm_cvtepi32_ps(i32_3);-- // Apply the scale- acc_0 = _mm_mul_ps( d_0_1, p0 );- acc_1 = _mm_mul_ps( d_0_1, p1 );- acc_2 = _mm_mul_ps( d_2_3, p2 );- acc_3 = _mm_mul_ps( d_2_3, p3 );- }-- assert(nb % 2 == 0); // TODO: handle odd nb-- // Main loop- for (int i = 2; i < nb; i+=2) {- _mm_prefetch(&x[i] + sizeof(block_q4_0), _MM_HINT_T0);- _mm_prefetch(&y[i] + sizeof(block_q8_0), _MM_HINT_T0);-- // Compute combined scale for the block 0 and 1- const __m128 d_0_1 = _mm_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );-- const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[i].qs);-- __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);- __m128i by_0 = _mm_loadu_si128((const __m128i *)y[i].qs);- bx_0 = _mm_sub_epi8(bx_0, off);- const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);-- __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));- __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[i].qs + 16));- bx_1 = _mm_sub_epi8(bx_1, off);- const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);-- _mm_prefetch(&x[i] + 2 * sizeof(block_q4_0), _MM_HINT_T0);- _mm_prefetch(&y[i] + 2 * sizeof(block_q8_0), _MM_HINT_T0);-- // Compute combined scale for the block 2 and 3- const __m128 d_2_3 = _mm_set1_ps( GGML_FP16_TO_FP32(x[i + 1].d) * GGML_FP16_TO_FP32(y[i + 1].d) );-- const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[i + 1].qs);-- __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);- __m128i by_2 = _mm_loadu_si128((const __m128i *)y[i + 1].qs);- bx_2 = _mm_sub_epi8(bx_2, off);- const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);-- __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));- __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[i + 1].qs + 16));- bx_3 = _mm_sub_epi8(bx_3, off);- const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);-- // Convert int32_t to float- __m128 p0 = _mm_cvtepi32_ps(i32_0);- __m128 p1 = _mm_cvtepi32_ps(i32_1);- __m128 p2 = _mm_cvtepi32_ps(i32_2);- __m128 p3 = _mm_cvtepi32_ps(i32_3);-- // Apply the scale- __m128 p0_d = _mm_mul_ps( d_0_1, p0 );- __m128 p1_d = _mm_mul_ps( d_0_1, p1 );- __m128 p2_d = _mm_mul_ps( d_2_3, p2 );- __m128 p3_d = _mm_mul_ps( d_2_3, p3 );-- // Acummulate- acc_0 = _mm_add_ps(p0_d, acc_0);- acc_1 = _mm_add_ps(p1_d, acc_1);- acc_2 = _mm_add_ps(p2_d, acc_2);- acc_3 = _mm_add_ps(p3_d, acc_3);- }-- *s = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);-#elif defined(__riscv_v_intrinsic)- float sumf = 0.0;-- size_t vl = __riscv_vsetvl_e8m1(qk/2);-- for (int i = 0; i < nb; i++) {- // load elements- vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);-- vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);- vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);-- // mask and store lower part of x, and then upper part- vuint8mf2_t x_a = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);- vuint8mf2_t x_l = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);-- vint8mf2_t x_ai = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);- vint8mf2_t x_li = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);-- // subtract offset- vint8mf2_t v0 = __riscv_vsub_vx_i8mf2(x_ai, 8, vl);- vint8mf2_t v1 = __riscv_vsub_vx_i8mf2(x_li, 8, vl);-- vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);- vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);-- vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);-- vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);- vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);+#include "ggml_vec_dot_q4_0_q8_0.inc"+#ifdef __UNDEF_AVX__+#undef __UNDEF_AVX__+#undef __AVX__+#endif+#undef ggml_vec_dot_q4_0_q8_0- int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);+#define ggml_vec_dot_q4_0_q8_0 __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q4_0_q8_0_avx2+#ifndef __AVX2__+#define __UNDEF_AVX2__+#define __AVX2__+#endif+#include "ggml_vec_dot_q4_0_q8_0.inc"+#ifdef __UNDEF_AVX2__+#undef __UNDEF_AVX2__+#undef __AVX2__+#endif+#undef ggml_vec_dot_q4_0_q8_0- sumf += sumi*GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d);+void ggml_vec_dot_q4_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ if (X86_HAVE(AVX2) && X86_HAVE(FMA)) {+ ggml_vec_dot_q4_0_q8_0_avx2(n, s, vx, vy);+ } else if (X86_HAVE(AVX)) {+ ggml_vec_dot_q4_0_q8_0_avx(n, s, vx, vy);+ } else {+ ggml_vec_dot_q4_0_q8_0_default(n, s, vx, vy);}+}- *s = sumf;#else- // scalar- float sumf = 0.0;-- for (int i = 0; i < nb; i++) {- int sumi = 0;-- for (int j = 0; j < qk/2; ++j) {- const int v0 = (x[i].qs[j] & 0x0F) - 8;- const int v1 = (x[i].qs[j] >> 4) - 8;-- sumi += (v0 * y[i].qs[j]) + (v1 * y[i].qs[j + qk/2]);- }-- sumf += sumi*GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d);- }-- *s = sumf;+#define static+#include "ggml_vec_dot_q4_0_q8_0.inc"+#undef static#endif-}void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {const int qk = QK8_1;@@ -2741,7 +2490,7 @@ void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, const void * restriconst __m256 d0d1 = _mm256_mul_ps( d0v, d1v );// Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes- const __m256i bx = bytes_from_nibbles_32(x[i].qs);+ const __m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);const __m256i by = _mm256_loadu_si256( (const __m256i *)y[i].qs );const __m256 xy = mul_sum_us8_pairs_float_avx2(bx, by);@@ -2978,8 +2727,8 @@ void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, const void * restri/* Compute combined scale for the block */const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));- __m256i bx = bytes_from_nibbles_32(x[i].qs);- __m256i bxhi = bytes_from_bits_32(x[i].qh);+ __m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);+ __m256i bxhi = bytes_from_bits_32_avx2(x[i].qh);bxhi = _mm256_andnot_si256(bxhi, _mm256_set1_epi8((char)0xF0));bx = _mm256_or_si256(bx, bxhi);@@ -3002,8 +2751,8 @@ void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, const void * restri/* Compute combined scale for the block */const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));- __m256i bx = bytes_from_nibbles_32(x[i].qs);- const __m256i bxhi = bytes_from_bits_32(x[i].qh);+ __m256i bx = bytes_from_nibbles_32_avx(x[i].qs);+ const __m256i bxhi = bytes_from_bits_32_avx(x[i].qh);__m128i bxhil = _mm256_castsi256_si128(bxhi);__m128i bxhih = _mm256_extractf128_si256(bxhi, 1);bxhil = _mm_andnot_si128(bxhil, mask);@@ -3297,8 +3046,8 @@ void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restrisumms += GGML_FP16_TO_FP32(x[i].m) * y[i].s;- __m256i bx = bytes_from_nibbles_32(x[i].qs);- __m256i bxhi = bytes_from_bits_32(x[i].qh);+ __m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);+ __m256i bxhi = bytes_from_bits_32_avx2(x[i].qh);bxhi = _mm256_and_si256(bxhi, _mm256_set1_epi8(0x10));bx = _mm256_or_si256(bx, bxhi);@@ -3324,8 +3073,8 @@ void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restrisumms += GGML_FP16_TO_FP32(x[i].m) * y[i].s;- __m256i bx = bytes_from_nibbles_32(x[i].qs);- const __m256i bxhi = bytes_from_bits_32(x[i].qh);+ __m256i bx = bytes_from_nibbles_32_avx(x[i].qs);+ const __m256i bxhi = bytes_from_bits_32_avx(x[i].qh);__m128i bxhil = _mm256_castsi256_si128(bxhi);__m128i bxhih = _mm256_extractf128_si256(bxhi, 1);bxhil = _mm_and_si128(bxhil, mask);@@ -3452,7 +3201,7 @@ void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restri#endif#undef ggml_vec_dot_q8_0_q8_0-#define ggml_vec_dot_q8_0_q8_0 __attribute__((__target__("avx2,fma"))) ggml_vec_dot_q8_0_q8_0_avx2+#define ggml_vec_dot_q8_0_q8_0 __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q8_0_q8_0_avx2#ifndef __AVX2__#define __UNDEF_AVX2__#define __AVX2__@@ -4886,7 +4635,7 @@ void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restri#endif#undef ggml_vec_dot_q6_K_q8_K-#define ggml_vec_dot_q6_K_q8_K __attribute__((__target__("avx2,fma"))) ggml_vec_dot_q6_K_q8_K_avx2+#define ggml_vec_dot_q6_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q6_K_q8_K_avx2#ifndef __AVX2__#define __UNDEF_AVX2__#define __AVX2__diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 85030082..187641d4 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -1005,7 +1005,67 @@ inline static void ggml_vec_neg_f32 (const int n, float * y, const float * x)inline static void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]*y[i]; }inline static void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]/y[i]; }+#ifdef __x86_64__+#pragma GCC push_options+#pragma GCC target("avx2")+#pragma GCC target("fma")+static void ggml_vec_dot_f32_avx2(const int n, float * restrict s, const float * restrict x, const float * restrict y) {+ float sumf = 0.0f;+ const int np = (n & ~(32 - 1));+ __m256 sum[(32/8)] = { _mm256_setzero_ps() };+ __m256 ax[(32/8)];+ __m256 ay[(32/8)];+ for (int i = 0; i < np; i += 32) {+ for (int j = 0; j < (32/8); j++) {+ ax[j] = _mm256_loadu_ps(x + i + j*8);+ ay[j] = _mm256_loadu_ps(y + i + j*8);+ sum[j] = _mm256_fmadd_ps(ax[j], ay[j], sum[j]);+ }+ }+ GGML_F32x8_REDUCE_AVX(sumf, sum);+ for (int i = np; i < n; ++i) {+ sumf += x[i]*y[i];+ }+ *s = sumf;+}+#pragma GCC pop_options+#endif /* __x86_64__ */++#ifdef __x86_64__+#pragma GCC push_options+#pragma GCC target("avx")+static void ggml_vec_dot_f32_avx(const int n, float * restrict s, const float * restrict x, const float * restrict y) {+ float sumf = 0.0f;+ const int np = (n & ~(32 - 1));+ __m256 sum[(32/8)] = { _mm256_setzero_ps() };+ __m256 ax[(32/8)];+ __m256 ay[(32/8)];+ for (int i = 0; i < np; i += 32) {+ for (int j = 0; j < (32/8); j++) {+ ax[j] = _mm256_loadu_ps(x + i + j*8);+ ay[j] = _mm256_loadu_ps(y + i + j*8);+ sum[j] = _mm256_add_ps(_mm256_mul_ps(ax[j], ay[j]), sum[j]);+ }+ }+ GGML_F32x8_REDUCE_AVX(sumf, sum);+ for (int i = np; i < n; ++i) {+ sumf += x[i]*y[i];+ }+ *s = sumf;+}+#pragma GCC pop_options+#endif /* __x86_64__ */+static void ggml_vec_dot_f32(const int n, float * restrict s, const float * restrict x, const float * restrict y) {++#ifdef __x86_64__+ if (X86_HAVE(AVX2) && X86_HAVE(FMA)) {+ return ggml_vec_dot_f32_avx2(n, s, x, y);+ } else if (X86_HAVE(AVX)) {+ return ggml_vec_dot_f32_avx(n, s, x, y);+ }+#endif+#ifdef GGML_SIMDfloat sumf = 0.0f;const int np = (n & ~(GGML_F32_STEP - 1));diff --git a/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.incindex 4ed31ad1..19dbd698 100644--- a/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc@@ -99,8 +99,8 @@ static void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void *for (int i = 0; i < nb; ++i) {- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);+ const float d = y[i].d * GGML_FP16_TO_FP32__F16C(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32__F16C(x[i].dmin);const uint8_t * restrict q2 = x[i].qs;const int8_t * restrict q8 = y[i].qs;diff --git a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.incindex 8f812434..66929626 100644--- a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc@@ -139,7 +139,7 @@ static void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void *for (int i = 0; i < nb; ++i) {- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float d = y[i].d * GGML_FP16_TO_FP32__F16C(x[i].d);const uint8_t * restrict q3 = x[i].qs;const int8_t * restrict q8 = y[i].qs;diff --git a/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q4_0_q8_0.incnew file mode 100644index 00000000..4cf26245--- /dev/null+++ b/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc@@ -0,0 +1,309 @@+static void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ const int qk = QK8_0;+ const int nb = n / qk;++ assert(n % qk == 0);++ const block_q4_0 * restrict x = vx;+ const block_q8_0 * restrict y = vy;++#if defined(__ARM_NEON)+ float32x4_t sumv0 = vdupq_n_f32(0.0f);+ float32x4_t sumv1 = vdupq_n_f32(0.0f);++ assert(nb % 2 == 0); // TODO: handle odd nb++ for (int i = 0; i < nb; i += 2) {+ const block_q4_0 * restrict x0 = &x[i + 0];+ const block_q4_0 * restrict x1 = &x[i + 1];+ const block_q8_0 * restrict y0 = &y[i + 0];+ const block_q8_0 * restrict y1 = &y[i + 1];++ const uint8x16_t m4b = vdupq_n_u8(0x0F);+ const int8x16_t s8b = vdupq_n_s8(0x8);++ const uint8x16_t v0_0 = vld1q_u8(x0->qs);+ const uint8x16_t v0_1 = vld1q_u8(x1->qs);++ // 4-bit -> 8-bit+ const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));+ const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));+ const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));+ const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));++ // sub 8+ const int8x16_t v0_0ls = vsubq_s8(v0_0l, s8b);+ const int8x16_t v0_0hs = vsubq_s8(v0_0h, s8b);+ const int8x16_t v0_1ls = vsubq_s8(v0_1l, s8b);+ const int8x16_t v0_1hs = vsubq_s8(v0_1h, s8b);++ // load y+ const int8x16_t v1_0l = vld1q_s8(y0->qs);+ const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);+ const int8x16_t v1_1l = vld1q_s8(y1->qs);+ const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);++#if defined(__ARM_FEATURE_DOTPROD)+ // dot product into int32x4_t+ const int32x4_t p_0 = vdotq_s32(vdotq_s32(vdupq_n_s32(0), v0_0ls, v1_0l), v0_0hs, v1_0h);+ const int32x4_t p_1 = vdotq_s32(vdotq_s32(vdupq_n_s32(0), v0_1ls, v1_1l), v0_1hs, v1_1h);++ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));+#else+ const int16x8_t pl0l = vmull_s8(vget_low_s8 (v0_0ls), vget_low_s8 (v1_0l));+ const int16x8_t pl0h = vmull_s8(vget_high_s8(v0_0ls), vget_high_s8(v1_0l));+ const int16x8_t ph0l = vmull_s8(vget_low_s8 (v0_0hs), vget_low_s8 (v1_0h));+ const int16x8_t ph0h = vmull_s8(vget_high_s8(v0_0hs), vget_high_s8(v1_0h));++ const int16x8_t pl1l = vmull_s8(vget_low_s8 (v0_1ls), vget_low_s8 (v1_1l));+ const int16x8_t pl1h = vmull_s8(vget_high_s8(v0_1ls), vget_high_s8(v1_1l));+ const int16x8_t ph1l = vmull_s8(vget_low_s8 (v0_1hs), vget_low_s8 (v1_1h));+ const int16x8_t ph1h = vmull_s8(vget_high_s8(v0_1hs), vget_high_s8(v1_1h));++ const int32x4_t pl0 = vaddq_s32(vpaddlq_s16(pl0l), vpaddlq_s16(pl0h));+ const int32x4_t ph0 = vaddq_s32(vpaddlq_s16(ph0l), vpaddlq_s16(ph0h));+ const int32x4_t pl1 = vaddq_s32(vpaddlq_s16(pl1l), vpaddlq_s16(pl1h));+ const int32x4_t ph1 = vaddq_s32(vpaddlq_s16(ph1l), vpaddlq_s16(ph1h));++ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(pl0, ph0)), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(pl1, ph1)), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));+#endif+ }++ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);+#elif defined(__AVX2__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();++ // Main loop+ for (int i = 0; i < nb; ++i) {+ /* Compute combined scale for the block */+ const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32__F16C(x[i].d) * GGML_FP16_TO_FP32__F16C(y[i].d) );++ __m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);++ // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.+ const __m256i off = _mm256_set1_epi8( 8 );+ bx = _mm256_sub_epi8( bx, off );++ __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);++ const __m256 q = mul_sum_i8_pairs_float_avx2(bx, by);++ /* Multiply q with scale and accumulate */+ acc = _mm256_fmadd_ps( d, q, acc );+ }++ *s = hsum_float_8(acc);+#elif defined(__AVX__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();++ // Main loop+ for (int i = 0; i < nb; ++i) {+ // Compute combined scale for the block+ const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );++ const __m128i lowMask = _mm_set1_epi8(0xF);+ const __m128i off = _mm_set1_epi8(8);++ const __m128i tmp = _mm_loadu_si128((const __m128i *)x[i].qs);++ __m128i bx = _mm_and_si128(lowMask, tmp);+ __m128i by = _mm_loadu_si128((const __m128i *)y[i].qs);+ bx = _mm_sub_epi8(bx, off);+ const __m128i i32_0 = mul_sum_i8_pairs(bx, by);++ bx = _mm_and_si128(lowMask, _mm_srli_epi64(tmp, 4));+ by = _mm_loadu_si128((const __m128i *)(y[i].qs + 16));+ bx = _mm_sub_epi8(bx, off);+ const __m128i i32_1 = mul_sum_i8_pairs(bx, by);++ // Convert int32_t to float+ __m256 p = _mm256_cvtepi32_ps(MM256_SET_M128I(i32_0, i32_1));++ // Apply the scale, and accumulate+ acc = _mm256_add_ps(_mm256_mul_ps( d, p ), acc);+ }++ *s = hsum_float_8(acc);+#elif defined(__SSSE3__)+ // set constants+ const __m128i lowMask = _mm_set1_epi8(0xF);+ const __m128i off = _mm_set1_epi8(8);++ // Initialize accumulator with zeros+ __m128 acc_0 = _mm_setzero_ps();+ __m128 acc_1 = _mm_setzero_ps();+ __m128 acc_2 = _mm_setzero_ps();+ __m128 acc_3 = _mm_setzero_ps();++ // First round without accumulation+ {+ _mm_prefetch(&x[0] + sizeof(block_q4_0), _MM_HINT_T0);+ _mm_prefetch(&y[0] + sizeof(block_q8_0), _MM_HINT_T0);++ // Compute combined scale for the block 0 and 1+ const __m128 d_0_1 = _mm_set1_ps( GGML_FP16_TO_FP32(x[0].d) * GGML_FP16_TO_FP32(y[0].d) );++ const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[0].qs);++ __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);+ __m128i by_0 = _mm_loadu_si128((const __m128i *)y[0].qs);+ bx_0 = _mm_sub_epi8(bx_0, off);+ const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);++ __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));+ __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[0].qs + 16));+ bx_1 = _mm_sub_epi8(bx_1, off);+ const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);++ _mm_prefetch(&x[1] + sizeof(block_q4_0), _MM_HINT_T0);+ _mm_prefetch(&y[1] + sizeof(block_q8_0), _MM_HINT_T0);++ // Compute combined scale for the block 2 and 3+ const __m128 d_2_3 = _mm_set1_ps( GGML_FP16_TO_FP32(x[1].d) * GGML_FP16_TO_FP32(y[1].d) );++ const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[1].qs);++ __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);+ __m128i by_2 = _mm_loadu_si128((const __m128i *)y[1].qs);+ bx_2 = _mm_sub_epi8(bx_2, off);+ const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);++ __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));+ __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[1].qs + 16));+ bx_3 = _mm_sub_epi8(bx_3, off);+ const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);++ // Convert int32_t to float+ __m128 p0 = _mm_cvtepi32_ps(i32_0);+ __m128 p1 = _mm_cvtepi32_ps(i32_1);+ __m128 p2 = _mm_cvtepi32_ps(i32_2);+ __m128 p3 = _mm_cvtepi32_ps(i32_3);++ // Apply the scale+ acc_0 = _mm_mul_ps( d_0_1, p0 );+ acc_1 = _mm_mul_ps( d_0_1, p1 );+ acc_2 = _mm_mul_ps( d_2_3, p2 );+ acc_3 = _mm_mul_ps( d_2_3, p3 );+ }++ assert(nb % 2 == 0); // TODO: handle odd nb++ // Main loop+ for (int i = 2; i < nb; i+=2) {+ _mm_prefetch(&x[i] + sizeof(block_q4_0), _MM_HINT_T0);+ _mm_prefetch(&y[i] + sizeof(block_q8_0), _MM_HINT_T0);++ // Compute combined scale for the block 0 and 1+ const __m128 d_0_1 = _mm_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );++ const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[i].qs);++ __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);+ __m128i by_0 = _mm_loadu_si128((const __m128i *)y[i].qs);+ bx_0 = _mm_sub_epi8(bx_0, off);+ const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);++ __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));+ __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[i].qs + 16));+ bx_1 = _mm_sub_epi8(bx_1, off);+ const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);++ _mm_prefetch(&x[i] + 2 * sizeof(block_q4_0), _MM_HINT_T0);+ _mm_prefetch(&y[i] + 2 * sizeof(block_q8_0), _MM_HINT_T0);++ // Compute combined scale for the block 2 and 3+ const __m128 d_2_3 = _mm_set1_ps( GGML_FP16_TO_FP32(x[i + 1].d) * GGML_FP16_TO_FP32(y[i + 1].d) );++ const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[i + 1].qs);++ __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);+ __m128i by_2 = _mm_loadu_si128((const __m128i *)y[i + 1].qs);+ bx_2 = _mm_sub_epi8(bx_2, off);+ const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);++ __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));+ __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[i + 1].qs + 16));+ bx_3 = _mm_sub_epi8(bx_3, off);+ const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);++ // Convert int32_t to float+ __m128 p0 = _mm_cvtepi32_ps(i32_0);+ __m128 p1 = _mm_cvtepi32_ps(i32_1);+ __m128 p2 = _mm_cvtepi32_ps(i32_2);+ __m128 p3 = _mm_cvtepi32_ps(i32_3);++ // Apply the scale+ __m128 p0_d = _mm_mul_ps( d_0_1, p0 );+ __m128 p1_d = _mm_mul_ps( d_0_1, p1 );+ __m128 p2_d = _mm_mul_ps( d_2_3, p2 );+ __m128 p3_d = _mm_mul_ps( d_2_3, p3 );++ // Acummulate+ acc_0 = _mm_add_ps(p0_d, acc_0);+ acc_1 = _mm_add_ps(p1_d, acc_1);+ acc_2 = _mm_add_ps(p2_d, acc_2);+ acc_3 = _mm_add_ps(p3_d, acc_3);+ }++ *s = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);+#elif defined(__riscv_v_intrinsic)+ float sumf = 0.0;++ size_t vl = __riscv_vsetvl_e8m1(qk/2);++ for (int i = 0; i < nb; i++) {+ // load elements+ vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);++ vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);+ vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);++ // mask and store lower part of x, and then upper part+ vuint8mf2_t x_a = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);+ vuint8mf2_t x_l = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);++ vint8mf2_t x_ai = __riscv_vreinterpret_v_u8mf2_i8mf2(x_a);+ vint8mf2_t x_li = __riscv_vreinterpret_v_u8mf2_i8mf2(x_l);++ // subtract offset+ vint8mf2_t v0 = __riscv_vsub_vx_i8mf2(x_ai, 8, vl);+ vint8mf2_t v1 = __riscv_vsub_vx_i8mf2(x_li, 8, vl);++ vint16m1_t vec_mul1 = __riscv_vwmul_vv_i16m1(v0, y0, vl);+ vint16m1_t vec_mul2 = __riscv_vwmul_vv_i16m1(v1, y1, vl);++ vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);++ vint32m1_t vs1 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul1, vec_zero, vl);+ vint32m1_t vs2 = __riscv_vwredsum_vs_i16m1_i32m1(vec_mul2, vs1, vl);++ int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);++ sumf += sumi*GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d);+ }++ *s = sumf;+#else+ // scalar+ float sumf = 0.0;++ for (int i = 0; i < nb; i++) {+ int sumi = 0;++ for (int j = 0; j < qk/2; ++j) {+ const int v0 = (x[i].qs[j] & 0x0F) - 8;+ const int v1 = (x[i].qs[j] >> 4) - 8;++ sumi += (v0 * y[i].qs[j]) + (v1 * y[i].qs[j + qk/2]);+ }++ sumf += sumi*GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d);+ }++ *s = sumf;+#endif+}diff --git a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.incindex e300b026..4bbc5061 100644--- a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc@@ -117,8 +117,8 @@ static void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void *const int8_t * restrict q8 = y[i].qs;#if QK_K == 256- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);+ const float d = y[i].d * GGML_FP16_TO_FP32__F16C(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32__F16C(x[i].dmin);memcpy(utmp, x[i].scales, 12);utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);diff --git a/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.incindex a8dda0fb..8a32dc31 100644--- a/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc@@ -155,7 +155,7 @@ static void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void *for (int i = 0; i < nb; ++i) {- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float d = y[i].d * GGML_FP16_TO_FP32__F16C(x[i].d);const uint8_t * restrict q4 = x[i].ql;const uint8_t * restrict qh = x[i].qh;diff --git a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q8_0_q8_0.incindex 0cc80e4d..d5c067dc 100644--- a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc+++ b/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc@@ -68,7 +68,11 @@ static void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void *// Main loopfor (int i = 0; i < nb; ++i) {// Compute combined scale for the block+#ifdef __AVX2__+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32__F16C(x[i].d) * GGML_FP16_TO_FP32__F16C(y[i].d));+#elseconst __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));+#endif__m256i bx = _mm256_loadu_si256((const __m256i *)x[i].qs);__m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);