Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Modify GGML to use runtime dispatching
Justine Tunney680852ca9813
diff --git a/build/config.mk b/build/config.mkindex c68bb3ad..5eac771d 100644--- a/build/config.mk+++ b/build/config.mk@@ -9,7 +9,7 @@ CXX = $(PREFIX)c++ARFLAGS = rcsDCCFLAGS = -g -O3-TARGET_ARCH = -Xx86_64,-msse3,-mavx,-mavx2,-mf16c,-mfma+TARGET_ARCH = -Xx86_64,-mssse3CPPFLAGS += -iquote. -mcosmoTMPDIR = o//tmpdiff --git a/llama.cpp/ggml-quants.c b/llama.cpp/ggml-quants.cindex 740be6dc..a93635ed 100644--- a/llama.cpp/ggml-quants.c+++ b/llama.cpp/ggml-quants.c@@ -1,3 +1,5 @@+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi#include "ggml-quants.h"#include "ggml-impl.h"@@ -5,6 +7,7 @@#include <string.h>#include <assert.h>#include <float.h>+#include <cosmo.h>#ifdef __ARM_NEON@@ -66,6 +69,97 @@ inline static int32_t vaddvq_s32(int32x4_t v) {#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)+// horizontally add 8 floats+#define hsum_float_8(x) ({ \+ __m128 res = _mm256_extractf128_ps(x, 1); \+ res = _mm_add_ps(res, _mm256_castps256_ps128(x)); \+ res = _mm_add_ps(res, _mm_movehl_ps(res, res)); \+ res = _mm_add_ss(res, _mm_movehdup_ps(res)); \+ _mm_cvtss_f32(res); \+})++#ifdef __x86_64__++// add int16_t pairwise and return as float vector+__attribute__((__target__("avx")))+static inline __m256 sum_i16_pairs_float_avx(const __m128i xh, const __m128i xl) {+ const __m128i ones = _mm_set1_epi16(1);+ const __m128i summed_pairsl = _mm_madd_epi16(ones, xl);+ const __m128i summed_pairsh = _mm_madd_epi16(ones, xh);+ const __m256i summed_pairs = MM256_SET_M128I(summed_pairsh, summed_pairsl);+ return _mm256_cvtepi32_ps(summed_pairs);+}++// add int16_t pairwise and return as float vector+__attribute__((__target__("avx2,fma")))+static inline __m256 sum_i16_pairs_float_avx2(const __m256i x) {+ const __m256i ones = _mm256_set1_epi16(1);+ const __m256i summed_pairs = _mm256_madd_epi16(ones, x);+ return _mm256_cvtepi32_ps(summed_pairs);+}++__attribute__((__target__("avx")))+static inline __m256 mul_sum_us8_pairs_float_avx(const __m256i ax, const __m256i sy) {+ const __m128i axl = _mm256_castsi256_si128(ax);+ const __m128i axh = _mm256_extractf128_si256(ax, 1);+ const __m128i syl = _mm256_castsi256_si128(sy);+ const __m128i syh = _mm256_extractf128_si256(sy, 1);+ // Perform multiplication and create 16-bit values+ const __m128i dotl = _mm_maddubs_epi16(axl, syl);+ const __m128i doth = _mm_maddubs_epi16(axh, syh);+ return sum_i16_pairs_float_avx(doth, dotl);+}++__attribute__((__target__("avx2,fma")))+static inline __m256 mul_sum_us8_pairs_float_avx2(const __m256i ax, const __m256i sy) {+#if __AVXVNNI__+ const __m256i zero = _mm256_setzero_si256();+ const __m256i summed_pairs = _mm256_dpbusd_epi32(zero, ax, sy);+ return _mm256_cvtepi32_ps(summed_pairs);+#else+ // Perform multiplication and create 16-bit values+ const __m256i dot = _mm256_maddubs_epi16(ax, sy);+ return sum_i16_pairs_float_avx2(dot);+#endif+}++// multiply int8_t, add results pairwise twice and return as float vector+__attribute__((__target__("avx")))+static inline __m256 mul_sum_i8_pairs_float_avx(const __m256i x, const __m256i y) {+ const __m128i xl = _mm256_castsi256_si128(x);+ const __m128i xh = _mm256_extractf128_si256(x, 1);+ const __m128i yl = _mm256_castsi256_si128(y);+ const __m128i yh = _mm256_extractf128_si256(y, 1);+ // Get absolute values of x vectors+ const __m128i axl = _mm_sign_epi8(xl, xl);+ const __m128i axh = _mm_sign_epi8(xh, xh);+ // Sign the values of the y vectors+ const __m128i syl = _mm_sign_epi8(yl, xl);+ const __m128i syh = _mm_sign_epi8(yh, xh);+ // Perform multiplication and create 16-bit values+ const __m128i dotl = _mm_maddubs_epi16(axl, syl);+ const __m128i doth = _mm_maddubs_epi16(axh, syh);+ return sum_i16_pairs_float_avx(doth, dotl);+}++// multiply int8_t, add results pairwise twice and return as float vector+__attribute__((__target__("avx2,fma")))+static inline __m256 mul_sum_i8_pairs_float_avx2(const __m256i x, const __m256i y) {+#if __AVXVNNIINT8__+ const __m256i zero = _mm256_setzero_si256();+ const __m256i summed_pairs = _mm256_dpbssd_epi32(zero, x, y);+ return _mm256_cvtepi32_ps(summed_pairs);+#else+ // Get absolute values of x vectors+ const __m256i ax = _mm256_sign_epi8(x, x);+ // Sign the values of the y vectors+ const __m256i sy = _mm256_sign_epi8(y, x);+ return mul_sum_us8_pairs_float_avx2(ax, sy);+#endif+}++#endif /* __x86_64__ */+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)// multiply int8_t, add results pairwise twicestatic inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {@@ -80,14 +174,6 @@ static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {}#if __AVX__ || __AVX2__ || __AVX512F__-// horizontally add 8 floats-static inline float hsum_float_8(const __m256 x) {- __m128 res = _mm256_extractf128_ps(x, 1);- res = _mm_add_ps(res, _mm256_castps256_ps128(x));- res = _mm_add_ps(res, _mm_movehl_ps(res, res));- res = _mm_add_ss(res, _mm_movehdup_ps(res));- return _mm_cvtss_f32(res);-}// horizontally add 8 int32_tstatic inline int hsum_i32_8(const __m256i a) {@@ -130,40 +216,6 @@ static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)return _mm256_and_si256(lowMask, bytes);}-// add int16_t pairwise and return as float vector-static inline __m256 sum_i16_pairs_float(const __m256i x) {- const __m256i ones = _mm256_set1_epi16(1);- const __m256i summed_pairs = _mm256_madd_epi16(ones, x);- return _mm256_cvtepi32_ps(summed_pairs);-}--static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {-#if __AVXVNNI__- const __m256i zero = _mm256_setzero_si256();- const __m256i summed_pairs = _mm256_dpbusd_epi32(zero, ax, sy);- return _mm256_cvtepi32_ps(summed_pairs);-#else- // Perform multiplication and create 16-bit values- const __m256i dot = _mm256_maddubs_epi16(ax, sy);- return sum_i16_pairs_float(dot);-#endif-}--// multiply int8_t, add results pairwise twice and return as float vector-static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {-#if __AVXVNNIINT8__- const __m256i zero = _mm256_setzero_si256();- const __m256i summed_pairs = _mm256_dpbssd_epi32(zero, x, y);- return _mm256_cvtepi32_ps(summed_pairs);-#else- // Get absolute values of x vectors- const __m256i ax = _mm256_sign_epi8(x, x);- // Sign the values of the y vectors- const __m256i sy = _mm256_sign_epi8(y, x);- return mul_sum_us8_pairs_float(ax, sy);-#endif-}-static inline __m128i packNibbles( __m256i bytes ){// Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh@@ -214,44 +266,6 @@ static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)return MM256_SET_M128I(tmph, tmpl);}-// add int16_t pairwise and return as float vector-static inline __m256 sum_i16_pairs_float(const __m128i xh, const __m128i xl) {- const __m128i ones = _mm_set1_epi16(1);- const __m128i summed_pairsl = _mm_madd_epi16(ones, xl);- const __m128i summed_pairsh = _mm_madd_epi16(ones, xh);- const __m256i summed_pairs = MM256_SET_M128I(summed_pairsh, summed_pairsl);- return _mm256_cvtepi32_ps(summed_pairs);-}--static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {- const __m128i axl = _mm256_castsi256_si128(ax);- const __m128i axh = _mm256_extractf128_si256(ax, 1);- const __m128i syl = _mm256_castsi256_si128(sy);- const __m128i syh = _mm256_extractf128_si256(sy, 1);- // Perform multiplication and create 16-bit values- const __m128i dotl = _mm_maddubs_epi16(axl, syl);- const __m128i doth = _mm_maddubs_epi16(axh, syh);- return sum_i16_pairs_float(doth, dotl);-}--// multiply int8_t, add results pairwise twice and return as float vector-static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {- const __m128i xl = _mm256_castsi256_si128(x);- const __m128i xh = _mm256_extractf128_si256(x, 1);- const __m128i yl = _mm256_castsi256_si128(y);- const __m128i yh = _mm256_extractf128_si256(y, 1);- // Get absolute values of x vectors- const __m128i axl = _mm_sign_epi8(xl, xl);- const __m128i axh = _mm_sign_epi8(xh, xh);- // Sign the values of the y vectors- const __m128i syl = _mm_sign_epi8(yl, xl);- const __m128i syh = _mm_sign_epi8(yh, xh);- // Perform multiplication and create 16-bit values- const __m128i dotl = _mm_maddubs_epi16(axl, syl);- const __m128i doth = _mm_maddubs_epi16(axh, syh);- return sum_i16_pairs_float(doth, dotl);-}-static inline __m128i packNibbles( __m128i bytes1, __m128i bytes2 ){// Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh@@ -2283,9 +2297,9 @@ void quantize_row_q8_K(const float * restrict x, void * restrict y, int k) {//// Helper functions//-#if __AVX__ || __AVX2__ || __AVX512F__-+#if __x86_64__// shuffles to pick the required scales in dot products+__attribute__((__target__("avx")))static inline __m256i get_scale_shuffle_q3k(int i) {static const uint8_t k_shuffle[128] = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,@@ -2295,6 +2309,7 @@ static inline __m256i get_scale_shuffle_q3k(int i) {};return _mm256_loadu_si256((const __m256i*)k_shuffle + i);}+__attribute__((__target__("avx")))static inline __m256i get_scale_shuffle_k4(int i) {static const uint8_t k_shuffle[256] = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,@@ -2308,6 +2323,7 @@ static inline __m256i get_scale_shuffle_k4(int i) {};return _mm256_loadu_si256((const __m256i*)k_shuffle + i);}+__attribute__((__target__("avx")))static inline __m128i get_scale_shuffle(int i) {static const uint8_t k_shuffle[128] = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,@@ -2414,7 +2430,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, const void * restrict vx,__m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);- const __m256 q = mul_sum_i8_pairs_float(bx, by);+ const __m256 q = mul_sum_i8_pairs_float_avx2(bx, by);/* Multiply q with scale and accumulate */acc = _mm256_fmadd_ps( d, q, acc );@@ -2728,7 +2744,7 @@ void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, const void * restriconst __m256i bx = bytes_from_nibbles_32(x[i].qs);const __m256i by = _mm256_loadu_si256( (const __m256i *)y[i].qs );- const __m256 xy = mul_sum_us8_pairs_float(bx, by);+ const __m256 xy = mul_sum_us8_pairs_float_avx2(bx, by);// Accumulate d0*d1*x*y#if defined(__AVX2__)@@ -2969,7 +2985,7 @@ void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, const void * restri__m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);- const __m256 q = mul_sum_i8_pairs_float(bx, by);+ const __m256 q = mul_sum_i8_pairs_float_avx2(bx, by);/* Multiply q with scale and accumulate */acc = _mm256_fmadd_ps(d, q, acc);@@ -3000,7 +3016,7 @@ void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, const void * restriconst __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);- const __m256 q = mul_sum_i8_pairs_float(bx, by);+ const __m256 q = mul_sum_i8_pairs_float_avx(bx, by);/* Multiply q with scale and accumulate */acc = _mm256_add_ps(_mm256_mul_ps(d, q), acc);@@ -3289,7 +3305,7 @@ void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restriconst __m256 dy = _mm256_set1_ps(y[i].d);const __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);- const __m256 q = mul_sum_us8_pairs_float(bx, by);+ const __m256 q = mul_sum_us8_pairs_float_avx2(bx, by);acc = _mm256_fmadd_ps(q, _mm256_mul_ps(dx, dy), acc);}@@ -3323,7 +3339,7 @@ void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restriconst __m256 dy = _mm256_set1_ps(y[i].d);const __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);- const __m256 q = mul_sum_us8_pairs_float(bx, by);+ const __m256 q = mul_sum_us8_pairs_float_avx(bx, by);acc = _mm256_add_ps(_mm256_mul_ps(q, _mm256_mul_ps(dx, dy)), acc);}@@ -3418,128 +3434,52 @@ void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, const void * restri#endif}-void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {- const int qk = QK8_0;- const int nb = n / qk;-- assert(n % qk == 0);-- const block_q8_0 * restrict x = vx;- const block_q8_0 * restrict y = vy;--#if defined(__ARM_NEON)- float32x4_t sumv0 = vdupq_n_f32(0.0f);- float32x4_t sumv1 = vdupq_n_f32(0.0f);-- assert(nb % 2 == 0); // TODO: handle odd nb-- for (int i = 0; i < nb; i += 2) {- const block_q8_0 * restrict x0 = &x[i + 0];- const block_q8_0 * restrict x1 = &x[i + 1];- const block_q8_0 * restrict y0 = &y[i + 0];- const block_q8_0 * restrict y1 = &y[i + 1];-- const int8x16_t x0_0 = vld1q_s8(x0->qs);- const int8x16_t x0_1 = vld1q_s8(x0->qs + 16);- const int8x16_t x1_0 = vld1q_s8(x1->qs);- const int8x16_t x1_1 = vld1q_s8(x1->qs + 16);-- // load y- const int8x16_t y0_0 = vld1q_s8(y0->qs);- const int8x16_t y0_1 = vld1q_s8(y0->qs + 16);- const int8x16_t y1_0 = vld1q_s8(y1->qs);- const int8x16_t y1_1 = vld1q_s8(y1->qs + 16);--#if defined(__ARM_FEATURE_DOTPROD)- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(- vdotq_s32(vdupq_n_s32(0), x0_0, y0_0),- vdotq_s32(vdupq_n_s32(0), x0_1, y0_1))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));+#ifdef __x86_64__- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(- vdotq_s32(vdupq_n_s32(0), x1_0, y1_0),- vdotq_s32(vdupq_n_s32(0), x1_1, y1_1))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));+#define ggml_vec_dot_q8_0_q8_0 ggml_vec_dot_q8_0_q8_0_default+#include "ggml_vec_dot_q8_0_q8_0.inc"+#undef ggml_vec_dot_q8_0_q8_0-#else- const int16x8_t p0_0 = vmull_s8(vget_low_s8 (x0_0), vget_low_s8 (y0_0));- const int16x8_t p0_1 = vmull_s8(vget_high_s8(x0_0), vget_high_s8(y0_0));- const int16x8_t p0_2 = vmull_s8(vget_low_s8 (x0_1), vget_low_s8 (y0_1));- const int16x8_t p0_3 = vmull_s8(vget_high_s8(x0_1), vget_high_s8(y0_1));-- const int16x8_t p1_0 = vmull_s8(vget_low_s8 (x1_0), vget_low_s8 (y1_0));- const int16x8_t p1_1 = vmull_s8(vget_high_s8(x1_0), vget_high_s8(y1_0));- const int16x8_t p1_2 = vmull_s8(vget_low_s8 (x1_1), vget_low_s8 (y1_1));- const int16x8_t p1_3 = vmull_s8(vget_high_s8(x1_1), vget_high_s8(y1_1));-- const int32x4_t p0 = vaddq_s32(vpaddlq_s16(p0_0), vpaddlq_s16(p0_1));- const int32x4_t p1 = vaddq_s32(vpaddlq_s16(p0_2), vpaddlq_s16(p0_3));- const int32x4_t p2 = vaddq_s32(vpaddlq_s16(p1_0), vpaddlq_s16(p1_1));- const int32x4_t p3 = vaddq_s32(vpaddlq_s16(p1_2), vpaddlq_s16(p1_3));-- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(p0, p1)), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(p2, p3)), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));+#define ggml_vec_dot_q8_0_q8_0 __attribute__((__target__("avx"))) ggml_vec_dot_q8_0_q8_0_avx+#ifndef __AVX__+#define __UNDEF_AVX__+#define __AVX__#endif- }-- *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);-#elif defined(__AVX2__) || defined(__AVX__)- // Initialize accumulator with zeros- __m256 acc = _mm256_setzero_ps();-- // Main loop- for (int i = 0; i < nb; ++i) {- // Compute combined scale for the block- const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));- __m256i bx = _mm256_loadu_si256((const __m256i *)x[i].qs);- __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);-- const __m256 q = mul_sum_i8_pairs_float(bx, by);-- // Multiply q with scale and accumulate-#if defined(__AVX2__)- acc = _mm256_fmadd_ps( d, q, acc );-#else- acc = _mm256_add_ps( _mm256_mul_ps( d, q ), acc );+#include "ggml_vec_dot_q8_0_q8_0.inc"+#ifdef __UNDEF_AVX__+#undef __UNDEF_AVX__+#undef __AVX__#endif- }+#undef ggml_vec_dot_q8_0_q8_0- *s = hsum_float_8(acc);-#elif defined(__riscv_v_intrinsic)- float sumf = 0.0;- size_t vl = __riscv_vsetvl_e8m1(qk);-- for (int i = 0; i < nb; i++) {- // load elements- vint8m1_t bx = __riscv_vle8_v_i8m1(x[i].qs, vl);- vint8m1_t by = __riscv_vle8_v_i8m1(y[i].qs, vl);-- vint16m2_t vw_mul = __riscv_vwmul_vv_i16m2(bx, by, vl);-- vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, vl);- vint32m1_t v_sum = __riscv_vwredsum_vs_i16m2_i32m1(vw_mul, v_zero, vl);-- int sumi = __riscv_vmv_x_s_i32m1_i32(v_sum);+#define ggml_vec_dot_q8_0_q8_0 __attribute__((__target__("avx2,fma"))) ggml_vec_dot_q8_0_q8_0_avx2+#ifndef __AVX2__+#define __UNDEF_AVX2__+#define __AVX2__+#endif+#include "ggml_vec_dot_q8_0_q8_0.inc"+#ifdef __UNDEF_AVX2__+#undef __UNDEF_AVX2__+#undef __AVX2__+#endif+#undef ggml_vec_dot_q8_0_q8_0- sumf += sumi*(GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d));+void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ if (X86_HAVE(AVX2) && X86_HAVE(FMA)) {+ ggml_vec_dot_q8_0_q8_0_avx2(n, s, vx, vy);+ } else if (X86_HAVE(AVX)) {+ ggml_vec_dot_q8_0_q8_0_avx(n, s, vx, vy);+ } else {+ ggml_vec_dot_q8_0_q8_0_default(n, s, vx, vy);}+}- *s = sumf;#else- // scalar- float sumf = 0.0;-- for (int i = 0; i < nb; i++) {- int sumi = 0;-- for (int j = 0; j < qk; j++) {- sumi += x[i].qs[j]*y[i].qs[j];- }-- sumf += sumi*(GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d));- }-- *s = sumf;+#define static+#include "ggml_vec_dot_q8_0_q8_0.inc"+#undef static#endif-}+#if QK_K == 256void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {@@ -5113,6 +5053,54 @@ void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restri#endif#if QK_K == 256++#ifdef __x86_64__++#define ggml_vec_dot_q4_K_q8_K ggml_vec_dot_q4_K_q8_K_default+#include "ggml_vec_dot_q4_K_q8_K_256.inc"+#undef ggml_vec_dot_q4_K_q8_K++#define ggml_vec_dot_q4_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q4_K_q8_K_avx+#ifndef __AVX__+#define __UNDEF_AVX__+#define __AVX__+#endif+#include "ggml_vec_dot_q4_K_q8_K_256.inc"+#ifdef __UNDEF_AVX__+#undef __UNDEF_AVX__+#undef __AVX__+#endif+#undef ggml_vec_dot_q4_K_q8_K++#define ggml_vec_dot_q4_K_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_q4_K_q8_K_avx2+#ifndef __AVX2__+#define __UNDEF_AVX2__+#define __AVX2__+#endif+#include "ggml_vec_dot_q4_K_q8_K_256.inc"+#ifdef __UNDEF_AVX2__+#undef __UNDEF_AVX2__+#undef __AVX2__+#endif+#undef ggml_vec_dot_q4_K_q8_K++void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {+ ggml_vec_dot_q4_K_q8_K_avx2(n, s, vx, vy);+ } else if (X86_HAVE(AVX)) {+ ggml_vec_dot_q4_K_q8_K_avx(n, s, vx, vy);+ } else {+ ggml_vec_dot_q4_K_q8_K_default(n, s, vx, vy);+ }+}++#else+#define static+#include "ggml_vec_dot_q4_K_q8_K_256.inc"+#undef static+#endif++#elsevoid ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {assert(n % QK_K == 0);@@ -5121,413 +5109,32 @@ void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restriconst int nb = n / QK_K;- static const uint32_t kmask1 = 0x3f3f3f3f;- static const uint32_t kmask2 = 0x0f0f0f0f;- static const uint32_t kmask3 = 0x03030303;-- uint32_t utmp[4];-#ifdef __ARM_NEONconst uint8x16_t m4b = vdupq_n_u8(0xf);+#ifdef __ARM_FEATURE_DOTPRODconst int32x4_t mzero = vdupq_n_s32(0);#endif- int8x16x2_t q4bytes;- int8x16x2_t q8bytes;-float sumf = 0;- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);-- const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));-- memcpy(utmp, x[i].scales, 12);-- uint32x2_t mins8 = { 0 };- mins8 = vset_lane_u32(utmp[1] & kmask1, mins8, 0);- mins8 = vset_lane_u32(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), mins8, 1);+ int8x16x2_t q4bytes;+ int8x16x4_t q8bytes;- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[0] &= kmask1;+ float sum_mins = 0.f;- const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8)));- const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),- vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));- sumf -= dmin * vaddvq_s32(prod);+ uint16_t aux16[2];+ const uint8_t * restrict scales = (const uint8_t *)aux16;- const uint8_t * scales = (const uint8_t *)utmp;+ for (int i = 0; i < nb; ++i) {const uint8_t * restrict q4 = x[i].qs;const int8_t * restrict q8 = y[i].qs;- int32_t sumi1 = 0;- int32_t sumi2 = 0;-- for (int j = 0; j < QK_K/64; ++j) {-- const uint8x16x2_t q4bits = vld1q_u8_x2(q4); q4 += 32;--#ifdef __ARM_FEATURE_DOTPROD- q8bytes = vld1q_s8_x2(q8); q8 += 32;- q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));- q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));-- const int32x4_t p1 = vdotq_s32(vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);- sumi1 += vaddvq_s32(p1) * scales[2*j+0];-- q8bytes = vld1q_s8_x2(q8); q8 += 32;- q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));- q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));-- const int32x4_t p2 = vdotq_s32(vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);-- sumi2 += vaddvq_s32(p2) * scales[2*j+1];-#else- q8bytes = vld1q_s8_x2(q8); q8 += 32;- q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));- q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));- const int16x8_t p0 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[0]), vget_low_s8 (q8bytes.val[0])),- vmull_s8(vget_high_s8(q4bytes.val[0]), vget_high_s8(q8bytes.val[0])));- const int16x8_t p1 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[1]), vget_low_s8 (q8bytes.val[1])),- vmull_s8(vget_high_s8(q4bytes.val[1]), vget_high_s8(q8bytes.val[1])));- sumi1 += vaddvq_s16(vaddq_s16(p0, p1)) * scales[2*j+0];-- q8bytes = vld1q_s8_x2(q8); q8 += 32;- q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));- q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));- const int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[0]), vget_low_s8 (q8bytes.val[0])),- vmull_s8(vget_high_s8(q4bytes.val[0]), vget_high_s8(q8bytes.val[0])));- const int16x8_t p3 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[1]), vget_low_s8 (q8bytes.val[1])),- vmull_s8(vget_high_s8(q4bytes.val[1]), vget_high_s8(q8bytes.val[1])));- sumi2 += vaddvq_s16(vaddq_s16(p2, p3)) * scales[2*j+1];--#endif- }-- sumf += d * (sumi1 + sumi2);-- }-- *s = sumf;--#elif defined __AVX2__-- const __m256i m4 = _mm256_set1_epi8(0xF);-- __m256 acc = _mm256_setzero_ps();- __m128 acc_m = _mm_setzero_ps();-- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);-- memcpy(utmp, x[i].scales, 12);- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);- const uint32_t uaux = utmp[1] & kmask1;- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[2] = uaux;- utmp[0] &= kmask1;-- const uint8_t * restrict q4 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;-- const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));-- const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);- const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));- const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);- acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m);-- const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);- const __m256i scales = MM256_SET_M128I(sc128, sc128);-- __m256i sumi = _mm256_setzero_si256();-- for (int j = 0; j < QK_K/64; ++j) {-- const __m256i scale_l = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));- const __m256i scale_h = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));-- const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;- const __m256i q4l = _mm256_and_si256(q4bits, m4);- const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4);-- const __m256i q8l = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- __m256i p16l = _mm256_maddubs_epi16(q4l, q8l);- p16l = _mm256_madd_epi16(scale_l, p16l);-- const __m256i q8h = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- __m256i p16h = _mm256_maddubs_epi16(q4h, q8h);- p16h = _mm256_madd_epi16(scale_h, p16h);- const __m256i sumj = _mm256_add_epi32(p16l, p16h);-- sumi = _mm256_add_epi32(sumi, sumj);- }-- __m256 vd = _mm256_set1_ps(d);- acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);-- }-- acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));- acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));-- *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);--#elif defined __AVX__-- const __m128i m4 = _mm_set1_epi8(0xF);- const __m128i m2 = _mm_set1_epi8(0x2);-- __m256 acc = _mm256_setzero_ps();- __m128 acc_m = _mm_setzero_ps();-- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);-- const uint8_t * restrict q4 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;-- memcpy(utmp, x[i].scales, 12);- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);- const uint32_t uaux = utmp[1] & kmask1;- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[2] = uaux;- utmp[0] &= kmask1;-- const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);- const __m128i scales = _mm_cvtepu8_epi16(utmps);- const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));-- const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);- const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);- const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);- const __m128i prod = _mm_madd_epi16(mins, q8s);- acc_m = _mm_add_ps(_mm_mul_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod)), acc_m);-- __m128i sumi_0 = _mm_setzero_si128();- __m128i sumi_1 = _mm_setzero_si128();-- __m128i shuffle = _mm_set1_epi16(0x0100);- for (int j = 0; j < QK_K/64; ++j) {-- const __m128i scale_l = _mm_shuffle_epi8(scales, shuffle);- shuffle = _mm_add_epi16(shuffle, m2);- const __m128i scale_h = _mm_shuffle_epi8(scales, shuffle);- shuffle = _mm_add_epi16(shuffle, m2);-- __m128i q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;- const __m128i q4l_0 = _mm_and_si128(q4bits, m4);- const __m128i q4h_0 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);- q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;- const __m128i q4l_1 = _mm_and_si128(q4bits, m4);- const __m128i q4h_1 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);-- const __m128i q8l_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- __m128i p16l = _mm_maddubs_epi16(q4l_0, q8l_0);- p16l = _mm_madd_epi16(scale_l, p16l);- sumi_0 = _mm_add_epi32(sumi_0, p16l);- const __m128i q8l_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- p16l = _mm_maddubs_epi16(q4l_1, q8l_1);- p16l = _mm_madd_epi16(scale_l, p16l);- sumi_1 = _mm_add_epi32(sumi_1, p16l);-- const __m128i q8h_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- __m128i p16h = _mm_maddubs_epi16(q4h_0, q8h_0);- p16h = _mm_madd_epi16(scale_h, p16h);- sumi_0 = _mm_add_epi32(sumi_0, p16h);- const __m128i q8h_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- p16h = _mm_maddubs_epi16(q4h_1, q8h_1);- p16h = _mm_madd_epi16(scale_h, p16h);- sumi_1 = _mm_add_epi32(sumi_1, p16h);-- }-- __m256 vd = _mm256_set1_ps(d);- __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);- acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);-- }-- acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));- acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));-- *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);--#elif defined __riscv_v_intrinsic-- const uint8_t * scales = (const uint8_t*)&utmp[0];- const uint8_t * mins = (const uint8_t*)&utmp[2];-- float sumf = 0;-- for (int i = 0; i < nb; ++i) {-- size_t vl = 8;-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);-- vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl);- vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl);- vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl);-- memcpy(utmp, x[i].scales, 12);- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);- const uint32_t uaux = utmp[1] & kmask1;- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[2] = uaux;- utmp[0] &= kmask1;-- vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl);- vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl));- vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl);-- vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);- sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);-- const uint8_t * restrict q4 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;-- vl = 32;-- int32_t sum_1 = 0;- int32_t sum_2 = 0;-- vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);-- for (int j = 0; j < QK_K/64; ++j) {- // load Q4- vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl);-- // load Q8 and multiply it with lower Q4 nibble- vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);- vint8m1_t q4_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl));- vint16m2_t qv_0 = __riscv_vwmul_vv_i16m2(q4_0, q8_0, vl);- vint16m1_t vs_0 = __riscv_vredsum_vs_i16m2_i16m1(qv_0, vzero, vl);-- sum_1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[2*j+0];-- // load Q8 and multiply it with upper Q4 nibble- vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl);- vint8m1_t q4_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl));- vint16m2_t qv_1 = __riscv_vwmul_vv_i16m2(q4_1, q8_1, vl);- vint16m1_t vs_1 = __riscv_vredsum_vs_i16m2_i16m1(qv_1, vzero, vl);-- sum_2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[2*j+1];-- q4 += 32; q8 += 64;-- }-- sumf += d*(sum_1 + sum_2);-- }-- *s = sumf;--#else--- const uint8_t * scales = (const uint8_t*)&utmp[0];- const uint8_t * mins = (const uint8_t*)&utmp[2];-- int8_t aux8[QK_K];- int16_t aux16[8];- float sums [8];- int32_t aux32[8];- memset(sums, 0, 8*sizeof(float));-- float sumf = 0;- for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q4 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;- memset(aux32, 0, 8*sizeof(int32_t));- int8_t * restrict a = aux8;- for (int j = 0; j < QK_K/64; ++j) {- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);- a += 32;- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);- a += 32; q4 += 32;- }- memcpy(utmp, x[i].scales, 12);- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);- const uint32_t uaux = utmp[1] & kmask1;- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);- utmp[2] = uaux;- utmp[0] &= kmask1;-- int sumi = 0;- for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];- a = aux8;- int is = 0;- for (int j = 0; j < QK_K/32; ++j) {- int32_t scale = scales[is++];- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- }- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];- const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;- sumf -= dmin * sumi;- }- for (int l = 0; l < 8; ++l) sumf += sums[l];- *s = sumf;-#endif-}-#else-void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {- assert(n % QK_K == 0);-- const block_q4_K * restrict x = vx;- const block_q8_K * restrict y = vy;-- const int nb = n / QK_K;--#ifdef __ARM_NEON-- const uint8x16_t m4b = vdupq_n_u8(0xf);--#ifdef __ARM_FEATURE_DOTPROD- const int32x4_t mzero = vdupq_n_s32(0);-#endif-- float sumf = 0;-- int8x16x2_t q4bytes;- int8x16x4_t q8bytes;-- float sum_mins = 0.f;-- uint16_t aux16[2];- const uint8_t * restrict scales = (const uint8_t *)aux16;-- for (int i = 0; i < nb; ++i) {-- const uint8_t * restrict q4 = x[i].qs;- const int8_t * restrict q8 = y[i].qs;-- const uint16_t * restrict a = (const uint16_t *)x[i].scales;- aux16[0] = a[0] & 0x0f0f;- aux16[1] = (a[0] >> 4) & 0x0f0f;+ const uint16_t * restrict a = (const uint16_t *)x[i].scales;+ aux16[0] = a[0] & 0x0f0f;+ aux16[1] = (a[0] >> 4) & 0x0f0f;const int32_t summi = scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]);sum_mins += y[i].d * (float)x[i].d[1] * summi;@@ -6491,479 +6098,52 @@ void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restri#if QK_K == 256-void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {- assert(n % QK_K == 0);-- const block_q6_K * restrict x = vx;- const block_q8_K * restrict y = vy;-- const int nb = n / QK_K;-#ifdef __ARM_NEON+#ifdef __x86_64__- float sum = 0;+#define ggml_vec_dot_q6_K_q8_K ggml_vec_dot_q6_K_q8_K_default+#include "ggml_vec_dot_q6_K_q8_K_256.inc"+#undef ggml_vec_dot_q6_K_q8_K- const uint8x16_t m4b = vdupq_n_u8(0xF);-#if defined(__ARM_FEATURE_DOTPROD)- const int32x4_t vzero = vdupq_n_s32(0);+#define ggml_vec_dot_q6_K_q8_K __attribute__((__target__("avx"))) ggml_vec_dot_q6_K_q8_K_avx+#ifndef __AVX__+#define __UNDEF_AVX__+#define __AVX__#endif- //const int8x16_t m32s = vdupq_n_s8(32);-- const uint8x16_t mone = vdupq_n_u8(3);-- int8x16x4_t q6bytes;- uint8x16x4_t q6h;-- for (int i = 0; i < nb; ++i) {-- const float d_all = GGML_FP16_TO_FP32(x[i].d);-- const uint8_t * restrict q6 = x[i].ql;- const uint8_t * restrict qh = x[i].qh;- const int8_t * restrict q8 = y[i].qs;-- const int8_t * restrict scale = x[i].scales;-- const int16x8x2_t q8sums = vld1q_s16_x2(y[i].bsums);- const int8x16_t scales = vld1q_s8(scale);- const int16x8x2_t q6scales = {vmovl_s8(vget_low_s8(scales)), vmovl_s8(vget_high_s8(scales))};-- const int32x4_t prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[0]), vget_low_s16 (q6scales.val[0])),- vmull_s16(vget_high_s16(q8sums.val[0]), vget_high_s16(q6scales.val[0]))),- vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[1]), vget_low_s16 (q6scales.val[1])),- vmull_s16(vget_high_s16(q8sums.val[1]), vget_high_s16(q6scales.val[1]))));- int32_t isum_mins = vaddvq_s32(prod);-- int32_t isum = 0;-- for (int j = 0; j < QK_K/128; ++j) {-- uint8x16x2_t qhbits = vld1q_u8_x2(qh); qh += 32;- uint8x16x4_t q6bits = vld1q_u8_x4(q6); q6 += 64;- int8x16x4_t q8bytes = vld1q_s8_x4(q8); q8 += 64;-- q6h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);- q6h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);- uint8x16_t shifted = vshrq_n_u8(qhbits.val[0], 2);- q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);- shifted = vshrq_n_u8(qhbits.val[1], 2);- q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);-- //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0])), m32s);- //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1])), m32s);- //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2])), m32s);- //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3])), m32s);- q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0]));- q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1]));- q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2]));- q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3]));--#if defined(__ARM_FEATURE_DOTPROD)-- isum += vaddvq_s32(vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];- scale += 4;--#else-- int16x8_t p0 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[0]), vget_low_s8 (q8bytes.val[0])),- vmull_s8(vget_high_s8(q6bytes.val[0]), vget_high_s8(q8bytes.val[0])));- int16x8_t p1 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[1]), vget_low_s8 (q8bytes.val[1])),- vmull_s8(vget_high_s8(q6bytes.val[1]), vget_high_s8(q8bytes.val[1])));- isum += vaddvq_s16(p0) * scale[0] + vaddvq_s16(p1) * scale[1];- scale += 2;-- int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[2]), vget_low_s8 (q8bytes.val[2])),- vmull_s8(vget_high_s8(q6bytes.val[2]), vget_high_s8(q8bytes.val[2])));- int16x8_t p3 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[3]), vget_low_s8 (q8bytes.val[3])),- vmull_s8(vget_high_s8(q6bytes.val[3]), vget_high_s8(q8bytes.val[3])));- isum += vaddvq_s16(p2) * scale[0] + vaddvq_s16(p3) * scale[1];- scale += 2;+#include "ggml_vec_dot_q6_K_q8_K_256.inc"+#ifdef __UNDEF_AVX__+#undef __UNDEF_AVX__+#undef __AVX__#endif+#undef ggml_vec_dot_q6_K_q8_K- q8bytes = vld1q_s8_x4(q8); q8 += 64;-- shifted = vshrq_n_u8(qhbits.val[0], 4);- q6h.val[0] = vshlq_n_u8(vandq_u8(mone, shifted), 4);- shifted = vshrq_n_u8(qhbits.val[1], 4);- q6h.val[1] = vshlq_n_u8(vandq_u8(mone, shifted), 4);- shifted = vshrq_n_u8(qhbits.val[0], 6);- q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);- shifted = vshrq_n_u8(qhbits.val[1], 6);- q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);-- //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0])), m32s);- //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1])), m32s);- //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2])), m32s);- //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3])), m32s);- q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0]));- q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1]));- q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2]));- q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3]));--#if defined(__ARM_FEATURE_DOTPROD)-- isum += vaddvq_s32(vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];- scale += 4;-- //for (int l = 0; l < 4; ++l) {- // const int32x4_t p = vdotq_s32(vzero, q6bytes.val[l], q8bytes.val[l]);- // isum += vaddvq_s32(p) * *scale++;- //}-#else- p0 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[0]), vget_low_s8 (q8bytes.val[0])),- vmull_s8(vget_high_s8(q6bytes.val[0]), vget_high_s8(q8bytes.val[0])));- p1 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[1]), vget_low_s8 (q8bytes.val[1])),- vmull_s8(vget_high_s8(q6bytes.val[1]), vget_high_s8(q8bytes.val[1])));- isum += vaddvq_s16(p0) * scale[0] + vaddvq_s16(p1) * scale[1];- scale += 2;-- p2 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[2]), vget_low_s8 (q8bytes.val[2])),- vmull_s8(vget_high_s8(q6bytes.val[2]), vget_high_s8(q8bytes.val[2])));- p3 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[3]), vget_low_s8 (q8bytes.val[3])),- vmull_s8(vget_high_s8(q6bytes.val[3]), vget_high_s8(q8bytes.val[3])));- isum += vaddvq_s16(p2) * scale[0] + vaddvq_s16(p3) * scale[1];- scale += 2;+#define ggml_vec_dot_q6_K_q8_K __attribute__((__target__("avx2,fma"))) ggml_vec_dot_q6_K_q8_K_avx2+#ifndef __AVX2__+#define __UNDEF_AVX2__+#define __AVX2__#endif+#include "ggml_vec_dot_q6_K_q8_K_256.inc"+#ifdef __UNDEF_AVX2__+#undef __UNDEF_AVX2__+#undef __AVX2__+#endif+#undef ggml_vec_dot_q6_K_q8_K- }- //sum += isum * d_all * y[i].d;- sum += d_all * y[i].d * (isum - 32 * isum_mins);-- }- *s = sum;--#elif defined __AVX2__-- const __m256i m4 = _mm256_set1_epi8(0xF);- const __m256i m2 = _mm256_set1_epi8(3);- const __m256i m32s = _mm256_set1_epi8(32);-- __m256 acc = _mm256_setzero_ps();-- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);-- const uint8_t * restrict q4 = x[i].ql;- const uint8_t * restrict qh = x[i].qh;- const int8_t * restrict q8 = y[i].qs;-- const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);-- __m256i sumi = _mm256_setzero_si256();-- int is = 0;-- for (int j = 0; j < QK_K/128; ++j) {-- const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0));- const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1));- const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2));- const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3));- is += 4;-- const __m256i q4bits1 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;- const __m256i q4bits2 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;- const __m256i q4bitsH = _mm256_loadu_si256((const __m256i*)qh); qh += 32;-- const __m256i q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bitsH, m2), 4);- const __m256i q4h_1 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 2), m2), 4);- const __m256i q4h_2 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 4), m2), 4);- const __m256i q4h_3 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 6), m2), 4);-- const __m256i q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0);- const __m256i q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1);- const __m256i q4_2 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_2);- const __m256i q4_3 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m4), q4h_3);-- const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;- const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;-- __m256i q8s_0 = _mm256_maddubs_epi16(m32s, q8_0);- __m256i q8s_1 = _mm256_maddubs_epi16(m32s, q8_1);- __m256i q8s_2 = _mm256_maddubs_epi16(m32s, q8_2);- __m256i q8s_3 = _mm256_maddubs_epi16(m32s, q8_3);-- __m256i p16_0 = _mm256_maddubs_epi16(q4_0, q8_0);- __m256i p16_1 = _mm256_maddubs_epi16(q4_1, q8_1);- __m256i p16_2 = _mm256_maddubs_epi16(q4_2, q8_2);- __m256i p16_3 = _mm256_maddubs_epi16(q4_3, q8_3);-- p16_0 = _mm256_sub_epi16(p16_0, q8s_0);- p16_1 = _mm256_sub_epi16(p16_1, q8s_1);- p16_2 = _mm256_sub_epi16(p16_2, q8s_2);- p16_3 = _mm256_sub_epi16(p16_3, q8s_3);-- p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0);- p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1);- p16_2 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_2), p16_2);- p16_3 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_3), p16_3);-- sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));- sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_2, p16_3));-- }-- acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);- }-- *s = hsum_float_8(acc);--#elif defined __AVX__-- const __m128i m4 = _mm_set1_epi8(0xF);- const __m128i m3 = _mm_set1_epi8(3);- const __m128i m32s = _mm_set1_epi8(32);- const __m128i m2 = _mm_set1_epi8(2);-- __m256 acc = _mm256_setzero_ps();-- for (int i = 0; i < nb; ++i) {-- const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);-- const uint8_t * restrict q4 = x[i].ql;- const uint8_t * restrict qh = x[i].qh;- const int8_t * restrict q8 = y[i].qs;-- const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);-- __m128i sumi_0 = _mm_setzero_si128();- __m128i sumi_1 = _mm_setzero_si128();-- __m128i shuffle = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);- for (int j = 0; j < QK_K/128; ++j) {-- const __m128i q4bitsH_0 = _mm_loadu_si128((const __m128i*)qh); qh += 16;- const __m128i q4bitsH_1 = _mm_loadu_si128((const __m128i*)qh); qh += 16;-- const __m128i q4h_0 = _mm_slli_epi16(_mm_and_si128(q4bitsH_0, m3), 4);- const __m128i q4h_1 = _mm_slli_epi16(_mm_and_si128(q4bitsH_1, m3), 4);- const __m128i q4h_2 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 2), m3), 4);- const __m128i q4h_3 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 2), m3), 4);- const __m128i q4h_4 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 4), m3), 4);- const __m128i q4h_5 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 4), m3), 4);- const __m128i q4h_6 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 6), m3), 4);- const __m128i q4h_7 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 6), m3), 4);-- const __m128i q4bits1_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;- const __m128i q4bits1_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;- const __m128i q4bits2_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;- const __m128i q4bits2_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;-- const __m128i q4_0 = _mm_or_si128(_mm_and_si128(q4bits1_0, m4), q4h_0);- const __m128i q4_1 = _mm_or_si128(_mm_and_si128(q4bits1_1, m4), q4h_1);- const __m128i q4_2 = _mm_or_si128(_mm_and_si128(q4bits2_0, m4), q4h_2);- const __m128i q4_3 = _mm_or_si128(_mm_and_si128(q4bits2_1, m4), q4h_3);- const __m128i q4_4 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_0, 4), m4), q4h_4);- const __m128i q4_5 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_1, 4), m4), q4h_5);- const __m128i q4_6 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_0, 4), m4), q4h_6);- const __m128i q4_7 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_1, 4), m4), q4h_7);-- const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;- const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;-- __m128i q8s_0 = _mm_maddubs_epi16(m32s, q8_0);- __m128i q8s_1 = _mm_maddubs_epi16(m32s, q8_1);- __m128i q8s_2 = _mm_maddubs_epi16(m32s, q8_2);- __m128i q8s_3 = _mm_maddubs_epi16(m32s, q8_3);- __m128i q8s_4 = _mm_maddubs_epi16(m32s, q8_4);- __m128i q8s_5 = _mm_maddubs_epi16(m32s, q8_5);- __m128i q8s_6 = _mm_maddubs_epi16(m32s, q8_6);- __m128i q8s_7 = _mm_maddubs_epi16(m32s, q8_7);-- __m128i p16_0 = _mm_maddubs_epi16(q4_0, q8_0);- __m128i p16_1 = _mm_maddubs_epi16(q4_1, q8_1);- __m128i p16_2 = _mm_maddubs_epi16(q4_2, q8_2);- __m128i p16_3 = _mm_maddubs_epi16(q4_3, q8_3);- __m128i p16_4 = _mm_maddubs_epi16(q4_4, q8_4);- __m128i p16_5 = _mm_maddubs_epi16(q4_5, q8_5);- __m128i p16_6 = _mm_maddubs_epi16(q4_6, q8_6);- __m128i p16_7 = _mm_maddubs_epi16(q4_7, q8_7);-- p16_0 = _mm_sub_epi16(p16_0, q8s_0);- p16_1 = _mm_sub_epi16(p16_1, q8s_1);- p16_2 = _mm_sub_epi16(p16_2, q8s_2);- p16_3 = _mm_sub_epi16(p16_3, q8s_3);- p16_4 = _mm_sub_epi16(p16_4, q8s_4);- p16_5 = _mm_sub_epi16(p16_5, q8s_5);- p16_6 = _mm_sub_epi16(p16_6, q8s_6);- p16_7 = _mm_sub_epi16(p16_7, q8s_7);-- const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);- shuffle = _mm_add_epi8(shuffle, m2);- const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);- shuffle = _mm_add_epi8(shuffle, m2);- const __m128i scale_2 = _mm_shuffle_epi8(scales, shuffle);- shuffle = _mm_add_epi8(shuffle, m2);- const __m128i scale_3 = _mm_shuffle_epi8(scales, shuffle);- shuffle = _mm_add_epi8(shuffle, m2);-- p16_0 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_0), p16_0);- p16_1 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_0, scale_0)), p16_1);- p16_2 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_1), p16_2);- p16_3 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_1, scale_1)), p16_3);- p16_4 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_2), p16_4);- p16_5 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_2, scale_2)), p16_5);- p16_6 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_3), p16_6);- p16_7 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_3, scale_3)), p16_7);-- sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));- sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));- sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_4, p16_6));- sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_5, p16_7));-- }-- __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);- acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);- }-- *s = hsum_float_8(acc);--#elif defined __riscv_v_intrinsic-- float sumf = 0;- for (int i = 0; i < nb; ++i) {-- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;-- const uint8_t * restrict q6 = x[i].ql;- const uint8_t * restrict qh = x[i].qh;- const int8_t * restrict q8 = y[i].qs;-- const int8_t * restrict scale = x[i].scales;-- size_t vl;-- vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);-- int sum_t = 0;- int is = 0;-- for (int j = 0; j < QK_K/128; ++j) {-- vl = 32;-- // load qh- vuint8m1_t qh_x = __riscv_vle8_v_u8m1(qh, vl);-- // load Q6- vuint8m1_t q6_0 = __riscv_vle8_v_u8m1(q6, vl);- vuint8m1_t q6_1 = __riscv_vle8_v_u8m1(q6+32, vl);-- vuint8m1_t q6a_0 = __riscv_vand_vx_u8m1(q6_0, 0x0F, vl);- vuint8m1_t q6a_1 = __riscv_vand_vx_u8m1(q6_1, 0x0F, vl);- vuint8m1_t q6s_0 = __riscv_vsrl_vx_u8m1(q6_0, 0x04, vl);- vuint8m1_t q6s_1 = __riscv_vsrl_vx_u8m1(q6_1, 0x04, vl);-- vuint8m1_t qh_0 = __riscv_vand_vx_u8m1(qh_x, 0x03, vl);- vuint8m1_t qh_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x2, vl), 0x03 , vl);- vuint8m1_t qh_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x4, vl), 0x03 , vl);- vuint8m1_t qh_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x6, vl), 0x03 , vl);-- vuint8m1_t qhi_0 = __riscv_vor_vv_u8m1(q6a_0, __riscv_vsll_vx_u8m1(qh_0, 0x04, vl), vl);- vuint8m1_t qhi_1 = __riscv_vor_vv_u8m1(q6a_1, __riscv_vsll_vx_u8m1(qh_1, 0x04, vl), vl);- vuint8m1_t qhi_2 = __riscv_vor_vv_u8m1(q6s_0, __riscv_vsll_vx_u8m1(qh_2, 0x04, vl), vl);- vuint8m1_t qhi_3 = __riscv_vor_vv_u8m1(q6s_1, __riscv_vsll_vx_u8m1(qh_3, 0x04, vl), vl);-- vint8m1_t a_0 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_0), 32, vl);- vint8m1_t a_1 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_1), 32, vl);- vint8m1_t a_2 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_2), 32, vl);- vint8m1_t a_3 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_3), 32, vl);-- // load Q8 and take product- vint16m2_t va_q_0 = __riscv_vwmul_vv_i16m2(a_0, __riscv_vle8_v_i8m1(q8, vl), vl);- vint16m2_t va_q_1 = __riscv_vwmul_vv_i16m2(a_1, __riscv_vle8_v_i8m1(q8+32, vl), vl);- vint16m2_t va_q_2 = __riscv_vwmul_vv_i16m2(a_2, __riscv_vle8_v_i8m1(q8+64, vl), vl);- vint16m2_t va_q_3 = __riscv_vwmul_vv_i16m2(a_3, __riscv_vle8_v_i8m1(q8+96, vl), vl);-- vl = 16;-- vint32m2_t vaux_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 0), scale[is+0], vl);- vint32m2_t vaux_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 1), scale[is+1], vl);- vint32m2_t vaux_2 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 0), scale[is+2], vl);- vint32m2_t vaux_3 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 1), scale[is+3], vl);- vint32m2_t vaux_4 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 0), scale[is+4], vl);- vint32m2_t vaux_5 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 1), scale[is+5], vl);- vint32m2_t vaux_6 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 0), scale[is+6], vl);- vint32m2_t vaux_7 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 1), scale[is+7], vl);-- vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_0, vaux_1, vl), vzero, vl);- vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_2, vaux_3, vl), isum0, vl);- vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_4, vaux_5, vl), isum1, vl);- vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_6, vaux_7, vl), isum2, vl);-- sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);-- q6 += 64; qh += 32; q8 += 128; is=8;-- }-- sumf += d * sum_t;-+void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ if (X86_HAVE(AVX2) && X86_HAVE(FMA)) {+ ggml_vec_dot_q6_K_q8_K_avx2(n, s, vx, vy);+ } else if (X86_HAVE(AVX)) {+ ggml_vec_dot_q6_K_q8_K_avx(n, s, vx, vy);+ } else {+ ggml_vec_dot_q6_K_q8_K_default(n, s, vx, vy);}-- *s = sumf;+}#else-- int8_t aux8[QK_K];- int16_t aux16[8];- float sums [8];- int32_t aux32[8];- memset(sums, 0, 8*sizeof(float));-- float sumf = 0;- for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q4 = x[i].ql;- const uint8_t * restrict qh = x[i].qh;- const int8_t * restrict q8 = y[i].qs;- memset(aux32, 0, 8*sizeof(int32_t));- int8_t * restrict a = aux8;- for (int j = 0; j < QK_K; j += 128) {- for (int l = 0; l < 32; ++l) {- a[l + 0] = (int8_t)((q4[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;- a[l + 32] = (int8_t)((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;- a[l + 64] = (int8_t)((q4[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;- a[l + 96] = (int8_t)((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;- }- a += 128;- q4 += 64;- qh += 32;- }- a = aux8;- int is = 0;- for (int j = 0; j < QK_K/16; ++j) {- int scale = x[i].scales[is++];- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- }- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];- }- for (int l = 0; l < 8; ++l) sumf += sums[l];- *s = sumf;+#define static+#include "ggml_vec_dot_q6_K_q8_K_256.inc"+#undef static#endif-}#elsediff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 991821d0..9d66e7c1 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -1,4 +1,4 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi#define _CRT_SECURE_NO_DEPRECATE // Disables ridiculous "unsafe" warnigns on Windows#define _USE_MATH_DEFINES // For M_PI on MSVC@@ -13,6 +13,7 @@#include <alloca.h>#endif+#include <cosmo.h>#include <fcntl.h>#include <assert.h>#include <errno.h>@@ -271,24 +272,33 @@ void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int n) {}}+#if defined(__x86_64__) && !defined(__F16C__)+#pragma GCC push_options+#pragma GCC target("f16c")+#endifvoid ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int n) {int i = 0;-#if defined(__F16C__)- for (; i + 7 < n; i += 8) {- __m256 x_vec = _mm256_loadu_ps(x + i);- __m128i y_vec = _mm256_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);- _mm_storeu_si128((__m128i *)(y + i), y_vec);- }- for(; i + 3 < n; i += 4) {- __m128 x_vec = _mm_loadu_ps(x + i);- __m128i y_vec = _mm_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);- _mm_storel_epi64((__m128i *)(y + i), y_vec);+#ifdef __x86_64__+ if (X86_HAVE(F16C)) {+ for (; i + 7 < n; i += 8) {+ __m256 x_vec = _mm256_loadu_ps(x + i);+ __m128i y_vec = _mm256_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);+ _mm_storeu_si128((__m128i *)(y + i), y_vec);+ }+ for(; i + 3 < n; i += 4) {+ __m128 x_vec = _mm_loadu_ps(x + i);+ __m128i y_vec = _mm_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);+ _mm_storel_epi64((__m128i *)(y + i), y_vec);+ }}#endiffor (; i < n; i++) {y[i] = GGML_FP32_TO_FP16(x[i]);}}+#if defined(__x86_64__) && !defined(__F16C__)+#pragma GCC pop_options+#endif//// timingdiff --git a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.incnew file mode 100644index 00000000..0157f32b--- /dev/null+++ b/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc@@ -0,0 +1,380 @@+static void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ assert(n % QK_K == 0);++ const block_q4_K * restrict x = vx;+ const block_q8_K * restrict y = vy;++ const int nb = n / QK_K;++ static const uint32_t kmask1 = 0x3f3f3f3f;+ static const uint32_t kmask2 = 0x0f0f0f0f;+ static const uint32_t kmask3 = 0x03030303;++ uint32_t utmp[4];++#ifdef __ARM_NEON++ const uint8x16_t m4b = vdupq_n_u8(0xf);+#ifdef __ARM_FEATURE_DOTPROD+ const int32x4_t mzero = vdupq_n_s32(0);+#endif++ int8x16x2_t q4bytes;+ int8x16x2_t q8bytes;++ float sumf = 0;++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));++ memcpy(utmp, x[i].scales, 12);++ uint32x2_t mins8 = { 0 };+ mins8 = vset_lane_u32(utmp[1] & kmask1, mins8, 0);+ mins8 = vset_lane_u32(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), mins8, 1);++ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[0] &= kmask1;++ const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8)));+ const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),+ vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));+ sumf -= dmin * vaddvq_s32(prod);++ const uint8_t * scales = (const uint8_t *)utmp;++ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ int32_t sumi1 = 0;+ int32_t sumi2 = 0;++ for (int j = 0; j < QK_K/64; ++j) {++ const uint8x16x2_t q4bits = vld1q_u8_x2(q4); q4 += 32;++#ifdef __ARM_FEATURE_DOTPROD+ q8bytes = vld1q_s8_x2(q8); q8 += 32;+ q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));+ q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));++ const int32x4_t p1 = vdotq_s32(vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);+ sumi1 += vaddvq_s32(p1) * scales[2*j+0];++ q8bytes = vld1q_s8_x2(q8); q8 += 32;+ q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));+ q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));++ const int32x4_t p2 = vdotq_s32(vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);++ sumi2 += vaddvq_s32(p2) * scales[2*j+1];+#else+ q8bytes = vld1q_s8_x2(q8); q8 += 32;+ q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));+ q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));+ const int16x8_t p0 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[0]), vget_low_s8 (q8bytes.val[0])),+ vmull_s8(vget_high_s8(q4bytes.val[0]), vget_high_s8(q8bytes.val[0])));+ const int16x8_t p1 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[1]), vget_low_s8 (q8bytes.val[1])),+ vmull_s8(vget_high_s8(q4bytes.val[1]), vget_high_s8(q8bytes.val[1])));+ sumi1 += vaddvq_s16(vaddq_s16(p0, p1)) * scales[2*j+0];++ q8bytes = vld1q_s8_x2(q8); q8 += 32;+ q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));+ q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));+ const int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[0]), vget_low_s8 (q8bytes.val[0])),+ vmull_s8(vget_high_s8(q4bytes.val[0]), vget_high_s8(q8bytes.val[0])));+ const int16x8_t p3 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[1]), vget_low_s8 (q8bytes.val[1])),+ vmull_s8(vget_high_s8(q4bytes.val[1]), vget_high_s8(q8bytes.val[1])));+ sumi2 += vaddvq_s16(vaddq_s16(p2, p3)) * scales[2*j+1];++#endif+ }++ sumf += d * (sumi1 + sumi2);++ }++ *s = sumf;++#elif defined __AVX2__++ const __m256i m4 = _mm256_set1_epi8(0xF);++ __m256 acc = _mm256_setzero_ps();+ __m128 acc_m = _mm_setzero_ps();++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * _cvtsh_ss(x[i].d);+ const float dmin = -y[i].d * _cvtsh_ss(x[i].dmin);++ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;++ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));++ const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);+ const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));+ const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);+ acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m);++ const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0);+ const __m256i scales = MM256_SET_M128I(sc128, sc128);++ __m256i sumi = _mm256_setzero_si256();++ for (int j = 0; j < QK_K/64; ++j) {++ const __m256i scale_l = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));+ const __m256i scale_h = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));++ const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;+ const __m256i q4l = _mm256_and_si256(q4bits, m4);+ const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4);++ const __m256i q8l = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ __m256i p16l = _mm256_maddubs_epi16(q4l, q8l);+ p16l = _mm256_madd_epi16(scale_l, p16l);++ const __m256i q8h = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ __m256i p16h = _mm256_maddubs_epi16(q4h, q8h);+ p16h = _mm256_madd_epi16(scale_h, p16h);+ const __m256i sumj = _mm256_add_epi32(p16l, p16h);++ sumi = _mm256_add_epi32(sumi, sumj);+ }++ __m256 vd = _mm256_set1_ps(d);+ acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);++ }++ acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));+ acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));++ *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);++#elif defined __AVX__++ const __m128i m4 = _mm_set1_epi8(0xF);+ const __m128i m2 = _mm_set1_epi8(0x2);++ __m256 acc = _mm256_setzero_ps();+ __m128 acc_m = _mm_setzero_ps();++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;++ const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);+ const __m128i scales = _mm_cvtepu8_epi16(utmps);+ const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));++ const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);+ const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);+ const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);+ const __m128i prod = _mm_madd_epi16(mins, q8s);+ acc_m = _mm_add_ps(_mm_mul_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod)), acc_m);++ __m128i sumi_0 = _mm_setzero_si128();+ __m128i sumi_1 = _mm_setzero_si128();++ __m128i shuffle = _mm_set1_epi16(0x0100);+ for (int j = 0; j < QK_K/64; ++j) {++ const __m128i scale_l = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi16(shuffle, m2);+ const __m128i scale_h = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi16(shuffle, m2);++ __m128i q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4l_0 = _mm_and_si128(q4bits, m4);+ const __m128i q4h_0 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);+ q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4l_1 = _mm_and_si128(q4bits, m4);+ const __m128i q4h_1 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);++ const __m128i q8l_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ __m128i p16l = _mm_maddubs_epi16(q4l_0, q8l_0);+ p16l = _mm_madd_epi16(scale_l, p16l);+ sumi_0 = _mm_add_epi32(sumi_0, p16l);+ const __m128i q8l_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ p16l = _mm_maddubs_epi16(q4l_1, q8l_1);+ p16l = _mm_madd_epi16(scale_l, p16l);+ sumi_1 = _mm_add_epi32(sumi_1, p16l);++ const __m128i q8h_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ __m128i p16h = _mm_maddubs_epi16(q4h_0, q8h_0);+ p16h = _mm_madd_epi16(scale_h, p16h);+ sumi_0 = _mm_add_epi32(sumi_0, p16h);+ const __m128i q8h_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ p16h = _mm_maddubs_epi16(q4h_1, q8h_1);+ p16h = _mm_madd_epi16(scale_h, p16h);+ sumi_1 = _mm_add_epi32(sumi_1, p16h);++ }++ __m256 vd = _mm256_set1_ps(d);+ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);+ acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);++ }++ acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));+ acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));++ *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);++#elif defined __riscv_v_intrinsic++ const uint8_t * scales = (const uint8_t*)&utmp[0];+ const uint8_t * mins = (const uint8_t*)&utmp[2];++ float sumf = 0;++ for (int i = 0; i < nb; ++i) {++ size_t vl = 8;++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl);+ vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl);+ vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl);++ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;++ vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl);+ vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl));+ vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl);++ vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);+ sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);++ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ vl = 32;++ int32_t sum_1 = 0;+ int32_t sum_2 = 0;++ vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);++ for (int j = 0; j < QK_K/64; ++j) {+ // load Q4+ vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl);++ // load Q8 and multiply it with lower Q4 nibble+ vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);+ vint8m1_t q4_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl));+ vint16m2_t qv_0 = __riscv_vwmul_vv_i16m2(q4_0, q8_0, vl);+ vint16m1_t vs_0 = __riscv_vredsum_vs_i16m2_i16m1(qv_0, vzero, vl);++ sum_1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[2*j+0];++ // load Q8 and multiply it with upper Q4 nibble+ vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl);+ vint8m1_t q4_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl));+ vint16m2_t qv_1 = __riscv_vwmul_vv_i16m2(q4_1, q8_1, vl);+ vint16m1_t vs_1 = __riscv_vredsum_vs_i16m2_i16m1(qv_1, vzero, vl);++ sum_2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[2*j+1];++ q4 += 32; q8 += 64;++ }++ sumf += d*(sum_1 + sum_2);++ }++ *s = sumf;++#else+++ const uint8_t * scales = (const uint8_t*)&utmp[0];+ const uint8_t * mins = (const uint8_t*)&utmp[2];++ int8_t aux8[QK_K];+ int16_t aux16[8];+ float sums [8];+ int32_t aux32[8];+ memset(sums, 0, 8*sizeof(float));++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+ memset(aux32, 0, 8*sizeof(int32_t));+ int8_t * restrict a = aux8;+ for (int j = 0; j < QK_K/64; ++j) {+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);+ a += 32;+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);+ a += 32; q4 += 32;+ }+ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;++ int sumi = 0;+ for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];+ a = aux8;+ int is = 0;+ for (int j = 0; j < QK_K/32; ++j) {+ int32_t scale = scales[is++];+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ }+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];+ const float dmin = GGML_FP16_TO_FP32(x[i].dmin) * y[i].d;+ sumf -= dmin * sumi;+ }+ for (int l = 0; l < 8; ++l) sumf += sums[l];+ *s = sumf;+#endif+}diff --git a/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.incnew file mode 100644index 00000000..a8dda0fb--- /dev/null+++ b/llama.cpp/ggml_vec_dot_q6_K_q8_K_256.inc@@ -0,0 +1,473 @@+static void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ assert(n % QK_K == 0);++ const block_q6_K * restrict x = vx;+ const block_q8_K * restrict y = vy;++ const int nb = n / QK_K;++#ifdef __ARM_NEON++ float sum = 0;++ const uint8x16_t m4b = vdupq_n_u8(0xF);+#if defined(__ARM_FEATURE_DOTPROD)+ const int32x4_t vzero = vdupq_n_s32(0);+#endif+ //const int8x16_t m32s = vdupq_n_s8(32);++ const uint8x16_t mone = vdupq_n_u8(3);++ int8x16x4_t q6bytes;+ uint8x16x4_t q6h;++ for (int i = 0; i < nb; ++i) {++ const float d_all = GGML_FP16_TO_FP32(x[i].d);++ const uint8_t * restrict q6 = x[i].ql;+ const uint8_t * restrict qh = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;++ const int8_t * restrict scale = x[i].scales;++ const int16x8x2_t q8sums = vld1q_s16_x2(y[i].bsums);+ const int8x16_t scales = vld1q_s8(scale);+ const int16x8x2_t q6scales = {vmovl_s8(vget_low_s8(scales)), vmovl_s8(vget_high_s8(scales))};++ const int32x4_t prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[0]), vget_low_s16 (q6scales.val[0])),+ vmull_s16(vget_high_s16(q8sums.val[0]), vget_high_s16(q6scales.val[0]))),+ vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[1]), vget_low_s16 (q6scales.val[1])),+ vmull_s16(vget_high_s16(q8sums.val[1]), vget_high_s16(q6scales.val[1]))));+ int32_t isum_mins = vaddvq_s32(prod);++ int32_t isum = 0;++ for (int j = 0; j < QK_K/128; ++j) {++ uint8x16x2_t qhbits = vld1q_u8_x2(qh); qh += 32;+ uint8x16x4_t q6bits = vld1q_u8_x4(q6); q6 += 64;+ int8x16x4_t q8bytes = vld1q_s8_x4(q8); q8 += 64;++ q6h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);+ q6h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);+ uint8x16_t shifted = vshrq_n_u8(qhbits.val[0], 2);+ q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);+ shifted = vshrq_n_u8(qhbits.val[1], 2);+ q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);++ //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0])), m32s);+ //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1])), m32s);+ //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2])), m32s);+ //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3])), m32s);+ q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0]));+ q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1]));+ q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2]));+ q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3]));++#if defined(__ARM_FEATURE_DOTPROD)++ isum += vaddvq_s32(vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] ++ vaddvq_s32(vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] ++ vaddvq_s32(vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] ++ vaddvq_s32(vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];+ scale += 4;++#else++ int16x8_t p0 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[0]), vget_low_s8 (q8bytes.val[0])),+ vmull_s8(vget_high_s8(q6bytes.val[0]), vget_high_s8(q8bytes.val[0])));+ int16x8_t p1 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[1]), vget_low_s8 (q8bytes.val[1])),+ vmull_s8(vget_high_s8(q6bytes.val[1]), vget_high_s8(q8bytes.val[1])));+ isum += vaddvq_s16(p0) * scale[0] + vaddvq_s16(p1) * scale[1];+ scale += 2;++ int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[2]), vget_low_s8 (q8bytes.val[2])),+ vmull_s8(vget_high_s8(q6bytes.val[2]), vget_high_s8(q8bytes.val[2])));+ int16x8_t p3 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[3]), vget_low_s8 (q8bytes.val[3])),+ vmull_s8(vget_high_s8(q6bytes.val[3]), vget_high_s8(q8bytes.val[3])));+ isum += vaddvq_s16(p2) * scale[0] + vaddvq_s16(p3) * scale[1];+ scale += 2;+#endif++ q8bytes = vld1q_s8_x4(q8); q8 += 64;++ shifted = vshrq_n_u8(qhbits.val[0], 4);+ q6h.val[0] = vshlq_n_u8(vandq_u8(mone, shifted), 4);+ shifted = vshrq_n_u8(qhbits.val[1], 4);+ q6h.val[1] = vshlq_n_u8(vandq_u8(mone, shifted), 4);+ shifted = vshrq_n_u8(qhbits.val[0], 6);+ q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);+ shifted = vshrq_n_u8(qhbits.val[1], 6);+ q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);++ //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0])), m32s);+ //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1])), m32s);+ //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2])), m32s);+ //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3])), m32s);+ q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0]));+ q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1]));+ q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2]));+ q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3]));++#if defined(__ARM_FEATURE_DOTPROD)++ isum += vaddvq_s32(vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] ++ vaddvq_s32(vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] ++ vaddvq_s32(vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] ++ vaddvq_s32(vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];+ scale += 4;++ //for (int l = 0; l < 4; ++l) {+ // const int32x4_t p = vdotq_s32(vzero, q6bytes.val[l], q8bytes.val[l]);+ // isum += vaddvq_s32(p) * *scale++;+ //}+#else+ p0 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[0]), vget_low_s8 (q8bytes.val[0])),+ vmull_s8(vget_high_s8(q6bytes.val[0]), vget_high_s8(q8bytes.val[0])));+ p1 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[1]), vget_low_s8 (q8bytes.val[1])),+ vmull_s8(vget_high_s8(q6bytes.val[1]), vget_high_s8(q8bytes.val[1])));+ isum += vaddvq_s16(p0) * scale[0] + vaddvq_s16(p1) * scale[1];+ scale += 2;++ p2 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[2]), vget_low_s8 (q8bytes.val[2])),+ vmull_s8(vget_high_s8(q6bytes.val[2]), vget_high_s8(q8bytes.val[2])));+ p3 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[3]), vget_low_s8 (q8bytes.val[3])),+ vmull_s8(vget_high_s8(q6bytes.val[3]), vget_high_s8(q8bytes.val[3])));+ isum += vaddvq_s16(p2) * scale[0] + vaddvq_s16(p3) * scale[1];+ scale += 2;+#endif++ }+ //sum += isum * d_all * y[i].d;+ sum += d_all * y[i].d * (isum - 32 * isum_mins);++ }+ *s = sum;++#elif defined __AVX2__++ const __m256i m4 = _mm256_set1_epi8(0xF);+ const __m256i m2 = _mm256_set1_epi8(3);+ const __m256i m32s = _mm256_set1_epi8(32);++ __m256 acc = _mm256_setzero_ps();++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);++ const uint8_t * restrict q4 = x[i].ql;+ const uint8_t * restrict qh = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;++ const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);++ __m256i sumi = _mm256_setzero_si256();++ int is = 0;++ for (int j = 0; j < QK_K/128; ++j) {++ const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0));+ const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1));+ const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2));+ const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3));+ is += 4;++ const __m256i q4bits1 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;+ const __m256i q4bits2 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;+ const __m256i q4bitsH = _mm256_loadu_si256((const __m256i*)qh); qh += 32;++ const __m256i q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bitsH, m2), 4);+ const __m256i q4h_1 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 2), m2), 4);+ const __m256i q4h_2 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 4), m2), 4);+ const __m256i q4h_3 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 6), m2), 4);++ const __m256i q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0);+ const __m256i q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1);+ const __m256i q4_2 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_2);+ const __m256i q4_3 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m4), q4h_3);++ const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;+ const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;++ __m256i q8s_0 = _mm256_maddubs_epi16(m32s, q8_0);+ __m256i q8s_1 = _mm256_maddubs_epi16(m32s, q8_1);+ __m256i q8s_2 = _mm256_maddubs_epi16(m32s, q8_2);+ __m256i q8s_3 = _mm256_maddubs_epi16(m32s, q8_3);++ __m256i p16_0 = _mm256_maddubs_epi16(q4_0, q8_0);+ __m256i p16_1 = _mm256_maddubs_epi16(q4_1, q8_1);+ __m256i p16_2 = _mm256_maddubs_epi16(q4_2, q8_2);+ __m256i p16_3 = _mm256_maddubs_epi16(q4_3, q8_3);++ p16_0 = _mm256_sub_epi16(p16_0, q8s_0);+ p16_1 = _mm256_sub_epi16(p16_1, q8s_1);+ p16_2 = _mm256_sub_epi16(p16_2, q8s_2);+ p16_3 = _mm256_sub_epi16(p16_3, q8s_3);++ p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0);+ p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1);+ p16_2 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_2), p16_2);+ p16_3 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_3), p16_3);++ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));+ sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_2, p16_3));++ }++ acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);+ }++ *s = hsum_float_8(acc);++#elif defined __AVX__++ const __m128i m4 = _mm_set1_epi8(0xF);+ const __m128i m3 = _mm_set1_epi8(3);+ const __m128i m32s = _mm_set1_epi8(32);+ const __m128i m2 = _mm_set1_epi8(2);++ __m256 acc = _mm256_setzero_ps();++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);++ const uint8_t * restrict q4 = x[i].ql;+ const uint8_t * restrict qh = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;++ const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);++ __m128i sumi_0 = _mm_setzero_si128();+ __m128i sumi_1 = _mm_setzero_si128();++ __m128i shuffle = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);+ for (int j = 0; j < QK_K/128; ++j) {++ const __m128i q4bitsH_0 = _mm_loadu_si128((const __m128i*)qh); qh += 16;+ const __m128i q4bitsH_1 = _mm_loadu_si128((const __m128i*)qh); qh += 16;++ const __m128i q4h_0 = _mm_slli_epi16(_mm_and_si128(q4bitsH_0, m3), 4);+ const __m128i q4h_1 = _mm_slli_epi16(_mm_and_si128(q4bitsH_1, m3), 4);+ const __m128i q4h_2 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 2), m3), 4);+ const __m128i q4h_3 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 2), m3), 4);+ const __m128i q4h_4 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 4), m3), 4);+ const __m128i q4h_5 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 4), m3), 4);+ const __m128i q4h_6 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_0, 6), m3), 4);+ const __m128i q4h_7 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH_1, 6), m3), 4);++ const __m128i q4bits1_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4bits1_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4bits2_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;+ const __m128i q4bits2_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;++ const __m128i q4_0 = _mm_or_si128(_mm_and_si128(q4bits1_0, m4), q4h_0);+ const __m128i q4_1 = _mm_or_si128(_mm_and_si128(q4bits1_1, m4), q4h_1);+ const __m128i q4_2 = _mm_or_si128(_mm_and_si128(q4bits2_0, m4), q4h_2);+ const __m128i q4_3 = _mm_or_si128(_mm_and_si128(q4bits2_1, m4), q4h_3);+ const __m128i q4_4 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_0, 4), m4), q4h_4);+ const __m128i q4_5 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_1, 4), m4), q4h_5);+ const __m128i q4_6 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_0, 4), m4), q4h_6);+ const __m128i q4_7 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_1, 4), m4), q4h_7);++ const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;+ const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;++ __m128i q8s_0 = _mm_maddubs_epi16(m32s, q8_0);+ __m128i q8s_1 = _mm_maddubs_epi16(m32s, q8_1);+ __m128i q8s_2 = _mm_maddubs_epi16(m32s, q8_2);+ __m128i q8s_3 = _mm_maddubs_epi16(m32s, q8_3);+ __m128i q8s_4 = _mm_maddubs_epi16(m32s, q8_4);+ __m128i q8s_5 = _mm_maddubs_epi16(m32s, q8_5);+ __m128i q8s_6 = _mm_maddubs_epi16(m32s, q8_6);+ __m128i q8s_7 = _mm_maddubs_epi16(m32s, q8_7);++ __m128i p16_0 = _mm_maddubs_epi16(q4_0, q8_0);+ __m128i p16_1 = _mm_maddubs_epi16(q4_1, q8_1);+ __m128i p16_2 = _mm_maddubs_epi16(q4_2, q8_2);+ __m128i p16_3 = _mm_maddubs_epi16(q4_3, q8_3);+ __m128i p16_4 = _mm_maddubs_epi16(q4_4, q8_4);+ __m128i p16_5 = _mm_maddubs_epi16(q4_5, q8_5);+ __m128i p16_6 = _mm_maddubs_epi16(q4_6, q8_6);+ __m128i p16_7 = _mm_maddubs_epi16(q4_7, q8_7);++ p16_0 = _mm_sub_epi16(p16_0, q8s_0);+ p16_1 = _mm_sub_epi16(p16_1, q8s_1);+ p16_2 = _mm_sub_epi16(p16_2, q8s_2);+ p16_3 = _mm_sub_epi16(p16_3, q8s_3);+ p16_4 = _mm_sub_epi16(p16_4, q8s_4);+ p16_5 = _mm_sub_epi16(p16_5, q8s_5);+ p16_6 = _mm_sub_epi16(p16_6, q8s_6);+ p16_7 = _mm_sub_epi16(p16_7, q8s_7);++ const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi8(shuffle, m2);+ const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi8(shuffle, m2);+ const __m128i scale_2 = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi8(shuffle, m2);+ const __m128i scale_3 = _mm_shuffle_epi8(scales, shuffle);+ shuffle = _mm_add_epi8(shuffle, m2);++ p16_0 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_0), p16_0);+ p16_1 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_0, scale_0)), p16_1);+ p16_2 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_1), p16_2);+ p16_3 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_1, scale_1)), p16_3);+ p16_4 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_2), p16_4);+ p16_5 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_2, scale_2)), p16_5);+ p16_6 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_3), p16_6);+ p16_7 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_3, scale_3)), p16_7);++ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));+ sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_4, p16_6));+ sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_5, p16_7));++ }++ __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);+ acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);+ }++ *s = hsum_float_8(acc);++#elif defined __riscv_v_intrinsic++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {++ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;++ const uint8_t * restrict q6 = x[i].ql;+ const uint8_t * restrict qh = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;++ const int8_t * restrict scale = x[i].scales;++ size_t vl;++ vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);++ int sum_t = 0;+ int is = 0;++ for (int j = 0; j < QK_K/128; ++j) {++ vl = 32;++ // load qh+ vuint8m1_t qh_x = __riscv_vle8_v_u8m1(qh, vl);++ // load Q6+ vuint8m1_t q6_0 = __riscv_vle8_v_u8m1(q6, vl);+ vuint8m1_t q6_1 = __riscv_vle8_v_u8m1(q6+32, vl);++ vuint8m1_t q6a_0 = __riscv_vand_vx_u8m1(q6_0, 0x0F, vl);+ vuint8m1_t q6a_1 = __riscv_vand_vx_u8m1(q6_1, 0x0F, vl);+ vuint8m1_t q6s_0 = __riscv_vsrl_vx_u8m1(q6_0, 0x04, vl);+ vuint8m1_t q6s_1 = __riscv_vsrl_vx_u8m1(q6_1, 0x04, vl);++ vuint8m1_t qh_0 = __riscv_vand_vx_u8m1(qh_x, 0x03, vl);+ vuint8m1_t qh_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x2, vl), 0x03 , vl);+ vuint8m1_t qh_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x4, vl), 0x03 , vl);+ vuint8m1_t qh_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x6, vl), 0x03 , vl);++ vuint8m1_t qhi_0 = __riscv_vor_vv_u8m1(q6a_0, __riscv_vsll_vx_u8m1(qh_0, 0x04, vl), vl);+ vuint8m1_t qhi_1 = __riscv_vor_vv_u8m1(q6a_1, __riscv_vsll_vx_u8m1(qh_1, 0x04, vl), vl);+ vuint8m1_t qhi_2 = __riscv_vor_vv_u8m1(q6s_0, __riscv_vsll_vx_u8m1(qh_2, 0x04, vl), vl);+ vuint8m1_t qhi_3 = __riscv_vor_vv_u8m1(q6s_1, __riscv_vsll_vx_u8m1(qh_3, 0x04, vl), vl);++ vint8m1_t a_0 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_0), 32, vl);+ vint8m1_t a_1 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_1), 32, vl);+ vint8m1_t a_2 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_2), 32, vl);+ vint8m1_t a_3 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_3), 32, vl);++ // load Q8 and take product+ vint16m2_t va_q_0 = __riscv_vwmul_vv_i16m2(a_0, __riscv_vle8_v_i8m1(q8, vl), vl);+ vint16m2_t va_q_1 = __riscv_vwmul_vv_i16m2(a_1, __riscv_vle8_v_i8m1(q8+32, vl), vl);+ vint16m2_t va_q_2 = __riscv_vwmul_vv_i16m2(a_2, __riscv_vle8_v_i8m1(q8+64, vl), vl);+ vint16m2_t va_q_3 = __riscv_vwmul_vv_i16m2(a_3, __riscv_vle8_v_i8m1(q8+96, vl), vl);++ vl = 16;++ vint32m2_t vaux_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 0), scale[is+0], vl);+ vint32m2_t vaux_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 1), scale[is+1], vl);+ vint32m2_t vaux_2 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 0), scale[is+2], vl);+ vint32m2_t vaux_3 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 1), scale[is+3], vl);+ vint32m2_t vaux_4 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 0), scale[is+4], vl);+ vint32m2_t vaux_5 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 1), scale[is+5], vl);+ vint32m2_t vaux_6 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 0), scale[is+6], vl);+ vint32m2_t vaux_7 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 1), scale[is+7], vl);++ vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_0, vaux_1, vl), vzero, vl);+ vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_2, vaux_3, vl), isum0, vl);+ vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_4, vaux_5, vl), isum1, vl);+ vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_6, vaux_7, vl), isum2, vl);++ sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);++ q6 += 64; qh += 32; q8 += 128; is=8;++ }++ sumf += d * sum_t;++ }++ *s = sumf;++#else++ int8_t aux8[QK_K];+ int16_t aux16[8];+ float sums [8];+ int32_t aux32[8];+ memset(sums, 0, 8*sizeof(float));++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const uint8_t * restrict q4 = x[i].ql;+ const uint8_t * restrict qh = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;+ memset(aux32, 0, 8*sizeof(int32_t));+ int8_t * restrict a = aux8;+ for (int j = 0; j < QK_K; j += 128) {+ for (int l = 0; l < 32; ++l) {+ a[l + 0] = (int8_t)((q4[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;+ a[l + 32] = (int8_t)((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;+ a[l + 64] = (int8_t)((q4[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;+ a[l + 96] = (int8_t)((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;+ }+ a += 128;+ q4 += 64;+ qh += 32;+ }+ a = aux8;+ int is = 0;+ for (int j = 0; j < QK_K/16; ++j) {+ int scale = x[i].scales[is++];+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ }+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];+ }+ for (int l = 0; l < 8; ++l) sumf += sums[l];+ *s = sumf;+#endif+}diff --git a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q8_0_q8_0.incnew file mode 100644index 00000000..0cc80e4d--- /dev/null+++ b/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc@@ -0,0 +1,123 @@+static void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ const int qk = QK8_0;+ const int nb = n / qk;++ assert(n % qk == 0);++ const block_q8_0 * restrict x = vx;+ const block_q8_0 * restrict y = vy;++#if defined(__ARM_NEON)+ float32x4_t sumv0 = vdupq_n_f32(0.0f);+ float32x4_t sumv1 = vdupq_n_f32(0.0f);++ assert(nb % 2 == 0); // TODO: handle odd nb++ for (int i = 0; i < nb; i += 2) {+ const block_q8_0 * restrict x0 = &x[i + 0];+ const block_q8_0 * restrict x1 = &x[i + 1];+ const block_q8_0 * restrict y0 = &y[i + 0];+ const block_q8_0 * restrict y1 = &y[i + 1];++ const int8x16_t x0_0 = vld1q_s8(x0->qs);+ const int8x16_t x0_1 = vld1q_s8(x0->qs + 16);+ const int8x16_t x1_0 = vld1q_s8(x1->qs);+ const int8x16_t x1_1 = vld1q_s8(x1->qs + 16);++ // load y+ const int8x16_t y0_0 = vld1q_s8(y0->qs);+ const int8x16_t y0_1 = vld1q_s8(y0->qs + 16);+ const int8x16_t y1_0 = vld1q_s8(y1->qs);+ const int8x16_t y1_1 = vld1q_s8(y1->qs + 16);++#if defined(__ARM_FEATURE_DOTPROD)+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(+ vdotq_s32(vdupq_n_s32(0), x0_0, y0_0),+ vdotq_s32(vdupq_n_s32(0), x0_1, y0_1))), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));++ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(+ vdotq_s32(vdupq_n_s32(0), x1_0, y1_0),+ vdotq_s32(vdupq_n_s32(0), x1_1, y1_1))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));++#else+ const int16x8_t p0_0 = vmull_s8(vget_low_s8 (x0_0), vget_low_s8 (y0_0));+ const int16x8_t p0_1 = vmull_s8(vget_high_s8(x0_0), vget_high_s8(y0_0));+ const int16x8_t p0_2 = vmull_s8(vget_low_s8 (x0_1), vget_low_s8 (y0_1));+ const int16x8_t p0_3 = vmull_s8(vget_high_s8(x0_1), vget_high_s8(y0_1));++ const int16x8_t p1_0 = vmull_s8(vget_low_s8 (x1_0), vget_low_s8 (y1_0));+ const int16x8_t p1_1 = vmull_s8(vget_high_s8(x1_0), vget_high_s8(y1_0));+ const int16x8_t p1_2 = vmull_s8(vget_low_s8 (x1_1), vget_low_s8 (y1_1));+ const int16x8_t p1_3 = vmull_s8(vget_high_s8(x1_1), vget_high_s8(y1_1));++ const int32x4_t p0 = vaddq_s32(vpaddlq_s16(p0_0), vpaddlq_s16(p0_1));+ const int32x4_t p1 = vaddq_s32(vpaddlq_s16(p0_2), vpaddlq_s16(p0_3));+ const int32x4_t p2 = vaddq_s32(vpaddlq_s16(p1_0), vpaddlq_s16(p1_1));+ const int32x4_t p3 = vaddq_s32(vpaddlq_s16(p1_2), vpaddlq_s16(p1_3));++ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(p0, p1)), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(p2, p3)), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));+#endif+ }++ *s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);+#elif defined(__AVX2__) || defined(__AVX__)+ // Initialize accumulator with zeros+ __m256 acc = _mm256_setzero_ps();++ // Main loop+ for (int i = 0; i < nb; ++i) {+ // Compute combined scale for the block+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));+ __m256i bx = _mm256_loadu_si256((const __m256i *)x[i].qs);+ __m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);++#if defined(__AVX2__)+ const __m256 q = mul_sum_i8_pairs_float_avx2(bx, by);+ // Multiply q with scale and accumulate+ acc = _mm256_fmadd_ps( d, q, acc );+#else+ const __m256 q = mul_sum_i8_pairs_float_avx(bx, by);+ // Multiply q with scale and accumulate+ acc = _mm256_add_ps( _mm256_mul_ps( d, q ), acc );+#endif+ }++ *s = hsum_float_8(acc);+#elif defined(__riscv_v_intrinsic)+ float sumf = 0.0;+ size_t vl = __riscv_vsetvl_e8m1(qk);++ for (int i = 0; i < nb; i++) {+ // load elements+ vint8m1_t bx = __riscv_vle8_v_i8m1(x[i].qs, vl);+ vint8m1_t by = __riscv_vle8_v_i8m1(y[i].qs, vl);++ vint16m2_t vw_mul = __riscv_vwmul_vv_i16m2(bx, by, vl);++ vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, vl);+ vint32m1_t v_sum = __riscv_vwredsum_vs_i16m2_i32m1(vw_mul, v_zero, vl);++ int sumi = __riscv_vmv_x_s_i32m1_i32(v_sum);++ sumf += sumi*(GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d));+ }++ *s = sumf;+#else+ // scalar+ float sumf = 0.0;++ for (int i = 0; i < nb; i++) {+ int sumi = 0;++ for (int j = 0; j < qk; j++) {+ sumi += x[i].qs[j]*y[i].qs[j];+ }++ sumf += sumi*(GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d));+ }++ *s = sumf;+#endif+}diff --git a/llama.cpp/server/BUILD.mk b/llama.cpp/server/BUILD.mkindex 50f213cf..79d206b2 100644--- a/llama.cpp/server/BUILD.mk+++ b/llama.cpp/server/BUILD.mk@@ -13,7 +13,6 @@ LLAMA_CPP_SERVER_ASSETS := \o/$(MODE)/llama.cpp/server/server: \o/$(MODE)/llama.cpp/server/server.o \- o/$(MODE)/llama.cpp/server/lib/lib.a \o/$(MODE)/llama.cpp/llava/llava.a \o/$(MODE)/llama.cpp/llama.cpp.a \$(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)@@ -21,9 +20,6 @@ o/$(MODE)/llama.cpp/server/server: \# strip llama.cpp/server/ directory prefix off name in zip archiveo/$(MODE)/llama.cpp/server/.args.zip.o: private ZIPOBJ_FLAGS += -B-include llama.cpp/server/lib/BUILD.mk-.PHONY: o/$(MODE)/llama.cpp/servero/$(MODE)/llama.cpp/server: \- o/$(MODE)/llama.cpp/server/server \- o/$(MODE)/llama.cpp/server/lib+ o/$(MODE)/llama.cpp/server/serverdiff --git a/llama.cpp/server/lib/BUILD.mk b/llama.cpp/server/lib/BUILD.mkdeleted file mode 100644index c36f2088..00000000--- a/llama.cpp/server/lib/BUILD.mk+++ /dev/null@@ -1,17 +0,0 @@-#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐-#───vi: set et ft=make ts=8 tw=8 fenc=utf-8 :vi───────────────────────┘--PKGS += LLAMA_CPP_SERVER_LIB--LLAMA_CPP_SERVER_LIB_FILES := $(wildcard llama.cpp/server/lib/*)-LLAMA_CPP_SERVER_LIB_HDRS = $(filter %.h,$(LLAMA_CPP_SERVER_LIB_FILES))-LLAMA_CPP_SERVER_LIB_SRCS = $(filter %.cc,$(LLAMA_CPP_SERVER_LIB_FILES))-LLAMA_CPP_SERVER_LIB_OBJS = $(LLAMA_CPP_SERVER_LIB_SRCS:%.cc=o/$(MODE)/%.o)--.PHONY: tool/args/args.h--o/$(MODE)/llama.cpp/server/lib/lib.a: $(LLAMA_CPP_SERVER_LIB_OBJS)--.PHONY: o/$(MODE)/llama.cpp/server/lib-o/$(MODE)/llama.cpp/server/lib: \- o/$(MODE)/llama.cpp/server/lib/lib.adiff --git a/llama.cpp/server/lib/lib.h b/llama.cpp/server/lib/lib.hdeleted file mode 100644index c32dbbc0..00000000--- a/llama.cpp/server/lib/lib.h+++ /dev/null@@ -1,6 +0,0 @@-#ifndef LLAMA_CPP_SERVER_LIB_H_-#define LLAMA_CPP_SERVER_LIB_H_--bool LaunchBrowser(const char *);--#endif /* LLAMA_CPP_SERVER_LIB_H_ */diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex 8c49116d..4a8c52a0 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -6,13 +6,10 @@#include "llama.cpp/llama.h"#include "llama.cpp/grammar-parser.h"#include "llama.cpp/llava/clip.h"-#include "llama.cpp/server/lib/lib.h"#include "llama.cpp/stb_image.h"+#include "llamafile/llamafile.h"-#ifndef NDEBUG-// crash the server in debug mode, otherwise send an http 500 error#define CPPHTTPLIB_NO_EXCEPTIONS 1-#endif#include "httplib.h"#include "json.h"@@ -2542,7 +2539,7 @@ int main(int argc, char **argv)snprintf(url, sizeof(url), "http://%s:%d/",sparams.hostname == "0.0.0.0" ? "127.0.0.1" : sparams.hostname.c_str(),sparams.port);- LaunchBrowser(url);+ llamafile_launch_browser(url);}tinyprint(2, "loading weights...\n", NULL);diff --git a/llamafile/check_cpu.c b/llamafile/check_cpu.cindex d0b8cd36..abc4286f 100644--- a/llamafile/check_cpu.c+++ b/llamafile/check_cpu.c@@ -1,4 +1,4 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi//// Copyright 2023 Mozilla Foundationdiff --git a/llamafile/cuda.c b/llamafile/cuda.cindex 4a7e11c6..678a0d45 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -366,7 +366,6 @@ static bool ImportCudaImpl(void) {// runtime link dynamic shared objectvoid *lib;- tinyprint(2, "linking ", dso, "...\n", NULL);lib = cosmo_dlopen(dso, RTLD_LAZY);if (!lib) {tinyprint(2, Dlerror(), ": failed to load library\n", NULL);diff --git a/llama.cpp/server/lib/launchbrowser.cc b/llamafile/launch_browser.csimilarity index 52%rename from llama.cpp/server/lib/launchbrowser.ccrename to llamafile/launch_browser.cindex 8fe14fc6..40c3a591 100644--- a/llama.cpp/server/lib/launchbrowser.cc+++ b/llamafile/launch_browser.c@@ -1,6 +1,21 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi-#include "llama.cpp/server/lib/lib.h"+//+// Copyright 2023 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "llamafile.h"#include <cosmo.h>#include <spawn.h>#include <errno.h>@@ -8,14 +23,17 @@#include <string.h>#include <sys/wait.h>-static void ReportFailure(const char *url,- const char *cmd,- const char *reason) {+static void report_failure(const char *url,+ const char *cmd,+ const char *reason) {tinyprint(2, "failed to open ", url, " in a browser tab using ", cmd,": ", reason, "\n", NULL);}-bool LaunchBrowser(const char *url) {+/**+ * Opens browser tab on host system.+ */+bool llamafile_launch_browser(const char *url) {// determine which command opens browser tabconst char *cmd;@@ -37,7 +55,7 @@ bool LaunchBrowser(const char *url) {err = posix_spawnp(&pid, cmd, NULL, &sa, args, environ);posix_spawnattr_destroy(&sa);if (err) {- ReportFailure(url, cmd, strerror(err));+ report_failure(url, cmd, strerror(err));return false;}@@ -46,12 +64,12 @@ bool LaunchBrowser(const char *url) {int ws;while (waitpid(pid, &ws, 0) == -1) {if (errno != EINTR) {- ReportFailure(url, cmd, strerror(errno));+ report_failure(url, cmd, strerror(errno));return false;}}if (ws) {- ReportFailure(url, cmd, "process exited with non-zero status");+ report_failure(url, cmd, "process exited with non-zero status");return false;}diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex 2809ffa4..ed975a55 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -23,6 +23,7 @@ bool llamafile_extract(const char *, const char *);int llamafile_is_file_newer_than(const char *, const char *);void llamafile_schlep(const void *, size_t);void llamafile_get_app_dir(char *, size_t);+bool llamafile_launch_browser(const char *);#ifdef __cplusplus}