Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add support for AVX512
Justine Tunneye629e752fd8b
diff --git a/README.md b/README.mdindex f016aadf..a6765b0d 100644--- a/README.md+++ b/README.md@@ -343,9 +343,8 @@ llamafile supports the following CPUs:print an error and refuse to run. This means that if you have an IntelCPU, it needs to be Intel Sandybridge or newer (circa 2011+), and ifyou have an AMD CPU, then it needs to be Bulldozer or newer (circa- 2011+). Support for AVX2, FMA, F16C, and VNNI are conditionally- enabled at runtime if you have a newer CPU. There's no support for- AVX512 runtime dispatching yet.+ 2011+). Support for AVX512, AVX2, FMA, F16C, and VNNI are+ conditionally enabled at runtime if you have a newer CPU.- **ARM64** microprocessors must have ARMv8a+. This means everythingfrom Apple Silicon to 64-bit Raspberry Pis will work, provided yourdiff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.hindex 324b7217..e68b7287 100644--- a/llama.cpp/ggml-impl.h+++ b/llama.cpp/ggml-impl.h@@ -101,9 +101,6 @@ typedef uint16_t ggml_fp16_internal_t;#include <riscv_vector.h>#endif-#define GGML_FP16_TO_FP32__F16C(x) _cvtsh_ss(x)-#define GGML_FP32_TO_FP16__F16C(x) _cvtss_sh(x, 0)-#ifdef __F16C__#ifdef _MSC_VERdiff --git a/llama.cpp/ggml-quants.c b/llama.cpp/ggml-quants.cindex d8ef1ec6..e9bec78b 100644--- a/llama.cpp/ggml-quants.c+++ b/llama.cpp/ggml-quants.c@@ -155,18 +155,9 @@ static inline __m256 sum_i16_pairs_float_avx2(const __m256i x) {__attribute__((__target__("avx2,fma,f16c")))static inline __m256 mul_sum_us8_pairs_float_avx2(const __m256i ax, const __m256i sy) {- if (X86_HAVE(AVXVNNI)) {- // {vex} vpdpbusd %ymm1,%ymm2,%ymm0- register __m256i ymm2 asm("ymm2") = ax;- register __m256i ymm1 asm("ymm1") = sy;- register __m256i ymm0 asm("ymm0") = _mm256_setzero_si256();- asm(".byte\t0xc4,0xe2,0x6d,0x50,0xc1" : "+x"(ymm0) : "x"(ymm1), "x"(ymm2));- return _mm256_cvtepi32_ps(ymm0);- } else {- // Perform multiplication and create 16-bit values- const __m256i dot = _mm256_maddubs_epi16(ax, sy);- return sum_i16_pairs_float_avx2(dot);- }+ // Perform multiplication and create 16-bit values+ const __m256i dot = _mm256_maddubs_epi16(ax, sy);+ return sum_i16_pairs_float_avx2(dot);}__attribute__((__target__("avx")))@@ -967,7 +958,7 @@ void quantize_row_q8_1_reference(const float * restrict x, block_q8_1 * restrictconst float d = amax / ((1 << 7) - 1);const float id = d ? 1.0f/d : 0.0f;- y[i].d = d;+ y[i].d = GGML_FP32_TO_FP16(d);int sum = 0;@@ -982,7 +973,7 @@ void quantize_row_q8_1_reference(const float * restrict x, block_q8_1 * restrictsum += y[i].qs[QK8_1/2 + j];}- y[i].s = sum*d;+ y[i].s = GGML_FP32_TO_FP16(sum*d);}}@@ -1010,7 +1001,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {const float d = amax / ((1 << 7) - 1);const float id = d ? 1.0f/d : 0.0f;- y[i].d = d;+ y[i].d = GGML_FP32_TO_FP16(d);int32x4_t accv = vdupq_n_s32(0);@@ -1026,7 +1017,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {accv = vaddq_s32(accv, vi);}- y[i].s = d * vaddvq_s32(accv);+ y[i].s = GGML_FP32_TO_FP16(d * vaddvq_s32(accv));}#elif defined(__wasm_simd128__)for (int i = 0; i < nb; i++) {@@ -1049,7 +1040,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {const float d = amax / ((1 << 7) - 1);const float id = d ? 1.0f/d : 0.0f;- y[i].d = d;+ y[i].d = GGML_FP32_TO_FP16(d);v128_t accv = wasm_i32x4_splat(0);@@ -1065,10 +1056,11 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {accv = wasm_i32x4_add(accv, vi);}- y[i].s = d * (wasm_i32x4_extract_lane(accv, 0) +- wasm_i32x4_extract_lane(accv, 1) +- wasm_i32x4_extract_lane(accv, 2) +- wasm_i32x4_extract_lane(accv, 3));+ y[i].s = GGML_FP32_TO_FP16(+ d * (wasm_i32x4_extract_lane(accv, 0) ++ wasm_i32x4_extract_lane(accv, 1) ++ wasm_i32x4_extract_lane(accv, 2) ++ wasm_i32x4_extract_lane(accv, 3)));}#elif defined(__AVX2__) || defined(__AVX__)for (int i = 0; i < nb; i++) {@@ -1093,7 +1085,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {// Quantize these floatsconst float d = maxScalar / 127.f;- y[i].d = d;+ y[i].d = GGML_FP32_TO_FP16(d);const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f;const __m256 mul = _mm256_set1_ps( id );@@ -1117,7 +1109,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {#if defined(__AVX2__)// Compute the sum of the quants and set y[i].s- y[i].s = d * hsum_i32_8(_mm256_add_epi32(_mm256_add_epi32(i0, i1), _mm256_add_epi32(i2, i3)));+ y[i].s = GGML_FP32_TO_FP16(d * hsum_i32_8(_mm256_add_epi32(_mm256_add_epi32(i0, i1), _mm256_add_epi32(i2, i3))));// Convert int32 to int16i0 = _mm256_packs_epi32( i0, i1 ); // 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15@@ -1147,7 +1139,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {// Compute the sum of the quants and set y[i].sconst __m128i s0 = _mm_add_epi32(_mm_add_epi32(ni0, ni1), _mm_add_epi32(ni2, ni3));const __m128i s1 = _mm_add_epi32(_mm_add_epi32(ni4, ni5), _mm_add_epi32(ni6, ni7));- y[i].s = d * hsum_i32_4(_mm_add_epi32(s0, s1));+ y[i].s = GGML_FP32_TO_FP16(d * hsum_i32_4(_mm_add_epi32(s0, s1)));// Convert int32 to int16ni0 = _mm_packs_epi32( ni0, ni1 );@@ -1178,7 +1170,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {const float d = amax / ((1 << 7) - 1);const float id = d ? 1.0f/d : 0.0f;- y[i].d = d;+ y[i].d = GGML_FP32_TO_FP16(d);vfloat32m4_t x0 = __riscv_vfmul_vf_f32m4(v_x, id, vl);@@ -1195,7 +1187,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {// set y[i].sint sum = __riscv_vmv_x_s_i16m1_i16(vwrs);- y[i].s = sum*d;+ y[i].s = GGML_FP32_TO_FP16(sum*d);}#elseGGML_UNUSED(nb);diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 9a04d838..efea3060 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -305,7 +305,7 @@ static void ggml_fp32_to_fp16_row_f16c(const float * x, ggml_fp16_t * y, int n)_mm_storel_epi64((__m128i *)(y + i), y_vec);}for (; i < n; i++) {- y[i] = GGML_FP32_TO_FP16__F16C(x[i]);+ y[i] = GGML_FP32_TO_FP16(x[i]);}}#pragma GCC pop_options@@ -10874,6 +10874,25 @@ static void ggml_compute_forward_mul_mat(}}}++#if 0+ if (dst->type == GGML_TYPE_F32) {+ for (int j = 0; j < ne1; ++j) {+ for (int i = 0; i < ne0; ++i) {+ if (isnan(*(float *)((char *)dst->data + nb1*j + nb0*i))) {+ fprintf(stderr, "error: matmul(%s, %s/%s, %s, m=%d, n=%d, k=%d) produced nan at (%d,%d)\n",+ ggml_type_name(src0->type),+ ggml_type_name(src1->type),+ ggml_type_name(vec_dot_type),+ ggml_type_name(dst->type),+ ne01, ne11, ne00,+ j, i);+ exit(1);+ }+ }+ }+ }+#endif}// ggml_compute_forward_mul_mat_iddiff --git a/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.incindex ebbaa970..ccb38cf9 100644--- a/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc@@ -82,8 +82,8 @@ static void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, size_t bs, cfor (int i = 0; i < nb; ++i) {- const float d = y[i].d * GGML_FP16_TO_FP32__F16C(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32__F16C(x[i].dmin);+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);const uint8_t * restrict q2 = x[i].qs;const int8_t * restrict q8 = y[i].qs;diff --git a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.incindex aaf501f2..a8fc9901 100644--- a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc@@ -115,7 +115,7 @@ static void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, size_t bs, cfor (int i = 0; i < nb; ++i) {- const float d = y[i].d * GGML_FP16_TO_FP32__F16C(x[i].d);+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);const uint8_t * restrict q3 = x[i].qs;const int8_t * restrict q8 = y[i].qs;diff --git a/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q4_0_q8_0.incindex 822395d5..8b07eabd 100644--- a/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc+++ b/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc@@ -59,7 +59,7 @@ void ggml_vec_dot_q4_0_q8_0(const int n, float * restrict s, size_t bs, const vo// Main loopfor (int i = 0; i < nb; ++i) {/* Compute combined scale for the block */- const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32__F16C(x[i].d) * GGML_FP16_TO_FP32__F16C(y[i].d) );+ const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );__m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);diff --git a/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc b/llama.cpp/ggml_vec_dot_q4_1_q8_1.incindex 07ab3038..ecab2852 100644--- a/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc+++ b/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc@@ -3,10 +3,88 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, cconst int nb = n / qk;assert(n % qk == 0);+#if defined(__ARM_FEATURE_MATMUL_INT8)+ assert((nrc == 2) || (nrc == 1));+#else+ assert(nrc == 1);+#endif+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);const block_q4_1 * restrict x = vx;const block_q8_1 * restrict y = vy;+#if defined(__ARM_FEATURE_MATMUL_INT8)+ if (nrc == 2) {+ const block_q4_1 * restrict vx0 = vx;+ const block_q4_1 * restrict vx1 = vx + bx;+ const block_q8_1 * restrict vy0 = vy;+ const block_q8_1 * restrict vy1 = vy + by;++ float32x4_t sumv0 = vdupq_n_f32(0.0f);+ float32x4_t summs0 = vdupq_n_f32(0.0f);++ for (int i = 0; i < nb; i++) {+ const block_q4_1 * restrict b_x0 = &vx0[i];+ const block_q4_1 * restrict b_x1 = &vx1[i];+ const block_q8_1 * restrict b_y0 = &vy0[i];+ const block_q8_1 * restrict b_y1 = &vy1[i];++ float32x4_t summs_t = {GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y0->s),+ GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y0->s),+ GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y1->s),+ GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y1->s)};+ summs0 += summs_t;++ const uint8x16_t m4b = vdupq_n_u8(0x0F);++ const uint8x16_t v0_0 = vld1q_u8(b_x0->qs);+ const uint8x16_t v0_1 = vld1q_u8(b_x1->qs);++ // 4-bit -> 8-bit+ const int8x16_t x0_l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));+ const int8x16_t x0_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));+ const int8x16_t x1_l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));+ const int8x16_t x1_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));++ // load y+ const int8x16_t y0_l = vld1q_s8(b_y0->qs);+ const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);+ const int8x16_t y1_l = vld1q_s8(b_y1->qs);+ const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);++ // mmla into int32x4_t+ float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*b_y0->d,+ GGML_FP16_TO_FP32(b_x0->d)*b_y1->d,+ GGML_FP16_TO_FP32(b_x1->d)*b_y0->d,+ GGML_FP16_TO_FP32(b_x1->d)*b_y1->d};++ int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));+ int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));++ int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));+ int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));++ int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));+ int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));++ int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));+ int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));+ sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),+ l1, r1)), l2, r2)), l3, r3))), scale);+ }++ float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);+ float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);+ sumv2 = sumv2 + summs0;++ vst1_f32(s, vget_low_f32(sumv2));+ vst1_f32(s + bs, vget_high_f32(sumv2));+ return;+ }+#endif// TODO: add WASM SIMD#if defined(__ARM_NEON)float32x4_t sumv0 = vdupq_n_f32(0.0f);@@ -22,7 +100,7 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, cconst block_q8_1 * restrict y0 = &y[i + 0];const block_q8_1 * restrict y1 = &y[i + 1];- summs += GGML_FP16_TO_FP32(x0->m) * y0->s + GGML_FP16_TO_FP32(x1->m) * y1->s;+ summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s) + GGML_FP16_TO_FP32(x1->m) * GGML_FP16_TO_FP32(y1->s);const uint8x16_t m4b = vdupq_n_u8(0x0F);@@ -45,8 +123,8 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, cconst int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0l, v1_0l), v0_0h, v1_0h);const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1l, v1_1l), v0_1h, v1_1h);- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*y0->d);- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*y1->d);+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));}*s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs;@@ -58,18 +136,10 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, c// Main loopfor (int i = 0; i < nb; ++i) {-#ifdef __AVX2__- const float d0 = GGML_FP16_TO_FP32__F16C(x[i].d);-#elseconst float d0 = GGML_FP16_TO_FP32(x[i].d);-#endif- const float d1 = y[i].d;+ const float d1 = GGML_FP16_TO_FP32(y[i].d);-#ifdef __AVX2__- summs += GGML_FP16_TO_FP32__F16C(x[i].m) * y[i].s;-#else- summs += GGML_FP16_TO_FP32(x[i].m) * y[i].s;-#endif+ summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);const __m256 d0v = _mm256_set1_ps( d0 );const __m256 d1v = _mm256_set1_ps( d1 );@@ -78,25 +148,22 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, cconst __m256 d0d1 = _mm256_mul_ps( d0v, d1v );// Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes+#if defined(__AVX2__)- const __m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);+ const __m256i qx = bytes_from_nibbles_32_avx2(x[i].qs);#else- const __m256i bx = bytes_from_nibbles_32_avx(x[i].qs);+ const __m256i qx = bytes_from_nibbles_32_avx(x[i].qs);#endif- const __m256i by = _mm256_loadu_si256( (const __m256i *)y[i].qs );+ const __m256i qy = _mm256_loadu_si256( (const __m256i *)y[i].qs );#if defined(__AVX2__)- const __m256 xy = mul_sum_us8_pairs_float_avx2(bx, by);+ const __m256 xy = mul_sum_us8_pairs_float_avx2(qx, qy);#else- const __m256 xy = mul_sum_us8_pairs_float_avx(bx, by);+ const __m256 xy = mul_sum_us8_pairs_float_avx(qx, qy);#endif// Accumulate d0*d1*x*y-#if defined(__AVX2__)- acc = _mm256_fmadd_ps( d0d1, xy, acc );-#elseacc = _mm256_add_ps( _mm256_mul_ps( d0d1, xy ), acc );-#endif}*s = hsum_float_8(acc) + summs;@@ -129,7 +196,7 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, cint sumi = __riscv_vmv_x_s_i32m1_i32(vs2);- sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s;+ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);}*s = sumf;@@ -147,7 +214,7 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, csumi += (v0 * y[i].qs[j]) + (v1 * y[i].qs[j + qk/2]);}- sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s;+ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);}*s = sumf;diff --git a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.incindex 589fe471..d8fa48e5 100644--- a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc@@ -84,8 +84,8 @@ static void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, size_t bs, cfor (int i = 0; i < nb; ++i) {- const float d = y[i].d * GGML_FP16_TO_FP32__F16C(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32__F16C(x[i].dmin);+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);memcpy(utmp, x[i].scales, 12);utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);diff --git a/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q5_0_q8_0.incindex 8ba7bae9..527ca5c3 100644--- a/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc+++ b/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc@@ -145,7 +145,7 @@ static void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, size_t bs, c// Main loopfor (int i = 0; i < nb; i++) {/* Compute combined scale for the block */- const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32__F16C(x[i].d) * GGML_FP16_TO_FP32__F16C(y[i].d));+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));__m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);__m256i bxhi = bytes_from_bits_32_avx2(x[i].qh);diff --git a/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc b/llama.cpp/ggml_vec_dot_q5_1_q8_1.incindex bc77430a..2255fddb 100644--- a/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc+++ b/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc@@ -156,9 +156,9 @@ static void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, size_t bs, c// Main loopfor (int i = 0; i < nb; i++) {- const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32__F16C(x[i].d));+ const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d));- summs += GGML_FP16_TO_FP32__F16C(x[i].m) * y[i].s;+ summs += GGML_FP16_TO_FP32(x[i].m) * y[i].s;__m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);__m256i bxhi = bytes_from_bits_32_avx2(x[i].qh);diff --git a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.incindex eb8852fa..61d829bf 100644--- a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc+++ b/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc@@ -96,8 +96,8 @@ static void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, size_t bs, cconst int8_t * restrict q8 = y[i].qs;#if QK_K == 256- const float d = y[i].d * GGML_FP16_TO_FP32__F16C(x[i].d);- const float dmin = -y[i].d * GGML_FP16_TO_FP32__F16C(x[i].dmin);+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);memcpy(utmp, x[i].scales, 12);utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);diff --git a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q8_0_q8_0.incindex c90ee156..29455a66 100644--- a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc+++ b/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc@@ -47,11 +47,7 @@ static void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, size_t bs, c// Main loopfor (int i = 0; i < nb; ++i) {// Compute combined scale for the block-#ifdef __AVX2__- const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32__F16C(x[i].d) * GGML_FP16_TO_FP32__F16C(y[i].d));-#elseconst __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));-#endif__m256i bx = _mm256_loadu_si256((const __m256i *)x[i].qs);__m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);@@ -62,11 +58,7 @@ static void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, size_t bs, c#endif// Multiply q with scale and accumulate-#if defined(__AVX2__)- acc = _mm256_fmadd_ps( d, q, acc );-#elseacc = _mm256_add_ps( _mm256_mul_ps( d, q ), acc );-#endif}*s = hsum_float_8(acc);diff --git a/llama.cpp/quantize/quantize.cpp b/llama.cpp/quantize/quantize.cppindex 4fc2d565..9b7b9cae 100644--- a/llama.cpp/quantize/quantize.cpp+++ b/llama.cpp/quantize/quantize.cpp@@ -219,6 +219,7 @@ int main(int argc, char ** argv) {}llama_model_quantize_params params = llama_model_quantize_default_params();+ params.nthread = 8;int arg_idx = 1;std::string imatrix_file;diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 81e26a78..92379c6e 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -17,18 +17,6 @@ LLAMAFILE_OBJS = \$(LLAMAFILE_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \$(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o)-o/$(MODE)/llamafile/sgemm_sss_avx.o: private TARGET_ARCH += -Xx86_64-mavx-o/$(MODE)/llamafile/sgemm_sss_fma.o: private TARGET_ARCH += -Xx86_64-mfma-o/$(MODE)/llamafile/sgemm_sss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f-o/$(MODE)/llamafile/sgemm_hss_f16c.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mf16c-o/$(MODE)/llamafile/sgemm_hss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f-o/$(MODE)/llamafile/sgemm_qqs_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni-o/$(MODE)/llamafile/sgemm_qqs_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni-o/$(MODE)/llamafile/sgemm_qqs_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma-o/$(MODE)/llamafile/sgemm_eqs_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni-o/$(MODE)/llamafile/sgemm_eqs_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni-o/$(MODE)/llamafile/sgemm_eqs_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma-o/$(MODE)/llamafile/zipalign: \o/$(MODE)/llamafile/zipalign.o \o/$(MODE)/llamafile/help.o \@@ -52,6 +40,24 @@ o/$(MODE)/llamafile: \o/$(MODE)/llamafile/zipcheck \o/$(MODE)/llamafile/addnl+################################################################################+# microarchitectures++o/$(MODE)/llamafile/sgemm_sss_avx.o: private TARGET_ARCH += -Xx86_64-mavx+o/$(MODE)/llamafile/sgemm_sss_fma.o: private TARGET_ARCH += -Xx86_64-mfma+o/$(MODE)/llamafile/sgemm_sss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f+o/$(MODE)/llamafile/sgemm_hss_f16c.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mf16c+o/$(MODE)/llamafile/sgemm_hss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f+o/$(MODE)/llamafile/sgemm_q0q0s_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni+o/$(MODE)/llamafile/sgemm_q0q0s_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni+o/$(MODE)/llamafile/sgemm_q0q0s_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma+o/$(MODE)/llamafile/sgemm_e0q0s_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni+o/$(MODE)/llamafile/sgemm_e0q0s_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni+o/$(MODE)/llamafile/sgemm_e0q0s_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma+o/$(MODE)/llamafile/sgemm_e1q1s_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni+o/$(MODE)/llamafile/sgemm_e1q1s_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni+o/$(MODE)/llamafile/sgemm_e1q1s_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma+################################################################################# testingdiff --git a/llamafile/kernel.h b/llamafile/kernel.hindex 5bdca973..dced6c4d 100644--- a/llamafile/kernel.h+++ b/llamafile/kernel.h@@ -1,5 +1,11 @@#pragma once+#ifdef __AVX512F__+#define VECTOR_REGISTERS 32+#else+#define VECTOR_REGISTERS 16+#endif+#define BEGIN_KERNEL(RM, RN) \int ytiles = (m - m0) / RM; \int xtiles = (n - n0) / RN; \@@ -16,3 +22,5 @@int j = n0 + job % xtiles * RN;#define END_KERNEL() }++#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppindex 3fde8dd0..54ba1b21 100644--- a/llamafile/sgemm.cpp+++ b/llamafile/sgemm.cpp@@ -60,34 +60,51 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,if (Btype != GGML_TYPE_Q8_0)return false;if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))- return llamafile_sgemm_qqs_avx512vnni(m, n, k, (const block_q8_0 *)A, lda,- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,- nth, task);+ return llamafile_sgemm_q0q0s_avx512vnni(m, n, k, (const block_q8_0 *)A, lda,+ (const block_q8_0 *)B, ldb, (float *)C, ldc,+ ith, nth, task);if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI) && !(k % 32))- return llamafile_sgemm_qqs_avxvnni(m, n, k, (const block_q8_0 *)A, lda,- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,- nth, task);+ return llamafile_sgemm_q0q0s_avxvnni(m, n, k, (const block_q8_0 *)A, lda,+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,+ nth, task);if (X86_HAVE(FMA) && X86_HAVE(AVX2) && !(k % 32))- return llamafile_sgemm_qqs_fma(m, n, k, (const block_q8_0 *)A, lda,- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,- task);+ return llamafile_sgemm_q0q0s_fma(m, n, k, (const block_q8_0 *)A, lda,+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,+ task);return false;case GGML_TYPE_Q4_0:if (Btype != GGML_TYPE_Q8_0)return false;if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))- return llamafile_sgemm_eqs_avx512vnni(m, n, k, (const block_q4_0 *)A, lda,- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,- nth, task);+ return llamafile_sgemm_e0q0s_avx512vnni(m, n, k, (const block_q4_0 *)A, lda,+ (const block_q8_0 *)B, ldb, (float *)C, ldc,+ ith, nth, task);if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI) && !(k % 32))- return llamafile_sgemm_eqs_avxvnni(m, n, k, (const block_q4_0 *)A, lda,- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,- nth, task);+ return llamafile_sgemm_e0q0s_avxvnni(m, n, k, (const block_q4_0 *)A, lda,+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,+ nth, task);if (X86_HAVE(FMA) && X86_HAVE(AVX2) && !(k % 32))- return llamafile_sgemm_eqs_fma(m, n, k, (const block_q4_0 *)A, lda,- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,- task);+ return llamafile_sgemm_e0q0s_fma(m, n, k, (const block_q4_0 *)A, lda,+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,+ task);+ return false;++ case GGML_TYPE_Q4_1:+ if (Btype != GGML_TYPE_Q8_1)+ return false;+ if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))+ return llamafile_sgemm_e1q1s_avx512vnni(m, n, k, (const block_q4_1 *)A, lda,+ (const block_q8_1 *)B, ldb, (float *)C, ldc,+ ith, nth, task);+ if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI) && !(k % 32))+ return llamafile_sgemm_e1q1s_avxvnni(m, n, k, (const block_q4_1 *)A, lda,+ (const block_q8_1 *)B, ldb, (float *)C, ldc, ith,+ nth, task);+ if (X86_HAVE(FMA) && X86_HAVE(AVX2) && !(k % 32))+ return llamafile_sgemm_e1q1s_fma(m, n, k, (const block_q4_1 *)A, lda,+ (const block_q8_1 *)B, ldb, (float *)C, ldc, ith, nth,+ task);return false;default:diff --git a/llamafile/sgemm.h b/llamafile/sgemm.hindex 14121d93..5c2c2db2 100644--- a/llamafile/sgemm.h+++ b/llamafile/sgemm.h@@ -17,19 +17,26 @@ bool llamafile_sgemm_hss_f16c(int, int, int, const unsigned short *, int, constbool llamafile_sgemm_hss_avx512f(int, int, int, const unsigned short *, int, const float *, int,float *, int, int, int, int);-bool llamafile_sgemm_qqs_avx512vnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,- float *, int, int, int, int);-bool llamafile_sgemm_qqs_avxvnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,- float *, int, int, int, int);-bool llamafile_sgemm_qqs_fma(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,- float *, int, int, int, int);+bool llamafile_sgemm_q0q0s_avx512vnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *,+ int, float *, int, int, int, int);+bool llamafile_sgemm_q0q0s_avxvnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,+ float *, int, int, int, int);+bool llamafile_sgemm_q0q0s_fma(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,+ float *, int, int, int, int);-bool llamafile_sgemm_eqs_avx512vnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,- float *, int, int, int, int);-bool llamafile_sgemm_eqs_avxvnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,- float *, int, int, int, int);-bool llamafile_sgemm_eqs_fma(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,- float *, int, int, int, int);+bool llamafile_sgemm_e0q0s_avx512vnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *,+ int, float *, int, int, int, int);+bool llamafile_sgemm_e0q0s_avxvnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,+ float *, int, int, int, int);+bool llamafile_sgemm_e0q0s_fma(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,+ float *, int, int, int, int);++bool llamafile_sgemm_e1q1s_avx512vnni(int, int, int, const block_q4_1 *, int, const block_q8_1 *,+ int, float *, int, int, int, int);+bool llamafile_sgemm_e1q1s_avxvnni(int, int, int, const block_q4_1 *, int, const block_q8_1 *, int,+ float *, int, int, int, int);+bool llamafile_sgemm_e1q1s_fma(int, int, int, const block_q4_1 *, int, const block_q8_1 *, int,+ float *, int, int, int, int);#ifdef __cplusplus}diff --git a/llamafile/sgemm_eqs_avxvnni.cpp b/llamafile/sgemm_e0q0s_avx512vnni.cppsimilarity index 78%rename from llamafile/sgemm_eqs_avxvnni.cpprename to llamafile/sgemm_e0q0s_avx512vnni.cppindex 22a572d4..18445ec0 100644--- a/llamafile/sgemm_eqs_avxvnni.cpp+++ b/llamafile/sgemm_e0q0s_avx512vnni.cpp@@ -22,13 +22,13 @@#define TA block_q4_0#define TB block_q8_0#define TC float-#include "llmmq0.inc"+#include "sgemmer0.inc"-bool llamafile_sgemm_eqs_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {+bool llamafile_sgemm_e0q0s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,+ int ldb, TC *C, int ldc, int ith, int nth, int task) {if (task != GGML_TASK_TYPE_COMPUTE)return true;- LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};tb.matmul(m, n);return true;}diff --git a/llamafile/sgemm_eqs_avx512vnni.cpp b/llamafile/sgemm_e0q0s_avxvnni.cppsimilarity index 78%rename from llamafile/sgemm_eqs_avx512vnni.cpprename to llamafile/sgemm_e0q0s_avxvnni.cppindex 78c00eb9..92598457 100644--- a/llamafile/sgemm_eqs_avx512vnni.cpp+++ b/llamafile/sgemm_e0q0s_avxvnni.cpp@@ -22,13 +22,13 @@#define TA block_q4_0#define TB block_q8_0#define TC float-#include "llmmq0.inc"+#include "sgemmer0.inc"-bool llamafile_sgemm_eqs_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {+bool llamafile_sgemm_e0q0s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {if (task != GGML_TASK_TYPE_COMPUTE)return true;- LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};tb.matmul(m, n);return true;}diff --git a/llamafile/sgemm_eqs_fma.cpp b/llamafile/sgemm_e0q0s_fma.cppsimilarity index 79%rename from llamafile/sgemm_eqs_fma.cpprename to llamafile/sgemm_e0q0s_fma.cppindex 9bb95430..bd69b661 100644--- a/llamafile/sgemm_eqs_fma.cpp+++ b/llamafile/sgemm_e0q0s_fma.cpp@@ -22,13 +22,13 @@#define TA block_q4_0#define TB block_q8_0#define TC float-#include "llmmq0.inc"+#include "sgemmer0.inc"-bool llamafile_sgemm_eqs_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,- int ldc, int ith, int nth, int task) {+bool llamafile_sgemm_e0q0s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {if (task != GGML_TASK_TYPE_COMPUTE)return true;- LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};tb.matmul(m, n);return true;}diff --git a/llamafile/sgemm_e1q1s_avx512vnni.cpp b/llamafile/sgemm_e1q1s_avx512vnni.cppnew file mode 100644index 00000000..e2a4836b--- /dev/null+++ b/llamafile/sgemm_e1q1s_avx512vnni.cpp@@ -0,0 +1,36 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"++#define TA block_q4_1+#define TB block_q8_1+#define TC float+#include "sgemmer1.inc"++bool llamafile_sgemm_e1q1s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,+ int ldb, TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ SGEMMER1 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_e1q1s_avxvnni.cpp b/llamafile/sgemm_e1q1s_avxvnni.cppnew file mode 100644index 00000000..9114427e--- /dev/null+++ b/llamafile/sgemm_e1q1s_avxvnni.cpp@@ -0,0 +1,36 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"++#define TA block_q4_1+#define TB block_q8_1+#define TC float+#include "sgemmer1.inc"++bool llamafile_sgemm_e1q1s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ SGEMMER1 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_e1q1s_fma.cpp b/llamafile/sgemm_e1q1s_fma.cppnew file mode 100644index 00000000..399232e2--- /dev/null+++ b/llamafile/sgemm_e1q1s_fma.cpp@@ -0,0 +1,36 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"++#define TA block_q4_1+#define TB block_q8_1+#define TC float+#include "sgemmer1.inc"++bool llamafile_sgemm_e1q1s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ SGEMMER1 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_hss_avx512f.cpp b/llamafile/sgemm_hss_avx512f.cppindex 17d88bcc..394f4124 100644--- a/llamafile/sgemm_hss_avx512f.cpp+++ b/llamafile/sgemm_hss_avx512f.cpp@@ -27,13 +27,13 @@#define zero() _mm512_setzero_ps()#define load_a(p) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(p)))#define load_b(p) _mm512_loadu_ps(p)-#include "llmm3x4.inc"+#include "sgemmer.inc"bool llamafile_sgemm_hss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,TC *C, int ldc, int ith, int nth, int task) {if (task != GGML_TASK_TYPE_COMPUTE)return true;- LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};tb.matmul(m, n);return true;}diff --git a/llamafile/sgemm_hss_f16c.cpp b/llamafile/sgemm_hss_f16c.cppindex ed16d736..f2f45473 100644--- a/llamafile/sgemm_hss_f16c.cpp+++ b/llamafile/sgemm_hss_f16c.cpp@@ -27,13 +27,13 @@#define zero() _mm256_setzero_ps()#define load_a(p) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(p)))#define load_b(p) _mm256_loadu_ps(p)-#include "llmm3x4.inc"+#include "sgemmer.inc"bool llamafile_sgemm_hss_f16c(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,TC *C, int ldc, int ith, int nth, int task) {if (task != GGML_TASK_TYPE_COMPUTE)return true;- LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};tb.matmul(m, n);return true;}diff --git a/llamafile/sgemm_qqs_avx512vnni.cpp b/llamafile/sgemm_q0q0s_avx512vnni.cppsimilarity index 78%rename from llamafile/sgemm_qqs_avx512vnni.cpprename to llamafile/sgemm_q0q0s_avx512vnni.cppindex bd9254e7..839b5c2d 100644--- a/llamafile/sgemm_qqs_avx512vnni.cpp+++ b/llamafile/sgemm_q0q0s_avx512vnni.cpp@@ -22,13 +22,13 @@#define TA block_q8_0#define TB block_q8_0#define TC float-#include "llmmq0.inc"+#include "sgemmer0.inc"-bool llamafile_sgemm_qqs_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {+bool llamafile_sgemm_q0q0s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,+ int ldb, TC *C, int ldc, int ith, int nth, int task) {if (task != GGML_TASK_TYPE_COMPUTE)return true;- LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};tb.matmul(m, n);return true;}diff --git a/llamafile/sgemm_qqs_avxvnni.cpp b/llamafile/sgemm_q0q0s_avxvnni.cppsimilarity index 78%rename from llamafile/sgemm_qqs_avxvnni.cpprename to llamafile/sgemm_q0q0s_avxvnni.cppindex cdde2d96..4c6eb23c 100644--- a/llamafile/sgemm_qqs_avxvnni.cpp+++ b/llamafile/sgemm_q0q0s_avxvnni.cpp@@ -22,13 +22,13 @@#define TA block_q8_0#define TB block_q8_0#define TC float-#include "llmmq0.inc"+#include "sgemmer0.inc"-bool llamafile_sgemm_qqs_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {+bool llamafile_sgemm_q0q0s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {if (task != GGML_TASK_TYPE_COMPUTE)return true;- LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};tb.matmul(m, n);return true;}diff --git a/llamafile/sgemm_qqs_fma.cpp b/llamafile/sgemm_q0q0s_fma.cppsimilarity index 79%rename from llamafile/sgemm_qqs_fma.cpprename to llamafile/sgemm_q0q0s_fma.cppindex 32b82c33..99f71dd5 100644--- a/llamafile/sgemm_qqs_fma.cpp+++ b/llamafile/sgemm_q0q0s_fma.cpp@@ -22,13 +22,13 @@#define TA block_q8_0#define TB block_q8_0#define TC float-#include "llmmq0.inc"+#include "sgemmer0.inc"-bool llamafile_sgemm_qqs_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,- int ldc, int ith, int nth, int task) {+bool llamafile_sgemm_q0q0s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {if (task != GGML_TASK_TYPE_COMPUTE)return true;- LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};tb.matmul(m, n);return true;}diff --git a/llamafile/sgemm_sss_avx.cpp b/llamafile/sgemm_sss_avx.cppindex 6c73fc2f..97bd377d 100644--- a/llamafile/sgemm_sss_avx.cpp+++ b/llamafile/sgemm_sss_avx.cpp@@ -27,13 +27,13 @@#define zero() _mm256_setzero_ps()#define load_a(p) _mm256_loadu_ps(p)#define load_b(p) _mm256_loadu_ps(p)-#include "llmm3x4.inc"+#include "sgemmer.inc"bool llamafile_sgemm_sss_avx(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,int ldc, int ith, int nth, int task) {if (task != GGML_TASK_TYPE_COMPUTE)return true;- LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};tb.matmul(m, n);return true;}diff --git a/llamafile/sgemm_sss_avx512f.cpp b/llamafile/sgemm_sss_avx512f.cppindex 1db08d29..7ccb188c 100644--- a/llamafile/sgemm_sss_avx512f.cpp+++ b/llamafile/sgemm_sss_avx512f.cpp@@ -27,13 +27,13 @@#define zero() _mm512_setzero_ps()#define load_a(p) _mm512_loadu_ps(p)#define load_b(p) _mm512_loadu_ps(p)-#include "llmm3x4.inc"+#include "sgemmer.inc"bool llamafile_sgemm_sss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,TC *C, int ldc, int ith, int nth, int task) {if (task != GGML_TASK_TYPE_COMPUTE)return true;- LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};tb.matmul(m, n);return true;}diff --git a/llamafile/sgemm_sss_fma.cpp b/llamafile/sgemm_sss_fma.cppindex 84813ee7..f018bfe6 100644--- a/llamafile/sgemm_sss_fma.cpp+++ b/llamafile/sgemm_sss_fma.cpp@@ -27,13 +27,13 @@#define zero() _mm256_setzero_ps()#define load_a(p) _mm256_loadu_ps(p)#define load_b(p) _mm256_loadu_ps(p)-#include "llmm3x4.inc"+#include "sgemmer.inc"bool llamafile_sgemm_sss_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,int ldc, int ith, int nth, int task) {if (task != GGML_TASK_TYPE_COMPUTE)return true;- LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};tb.matmul(m, n);return true;}diff --git a/llamafile/llmm3x4.inc b/llamafile/sgemmer.incsimilarity index 61%rename from llamafile/llmm3x4.increname to llamafile/sgemmer.incindex c0d6c66f..54b9e5e5 100644--- a/llamafile/llmm3x4.inc+++ b/llamafile/sgemmer.inc@@ -15,9 +15,6 @@// See the License for the specific language governing permissions and// limitations under the License.-#include <algorithm>-#include <cassert>-#include "llama.cpp/ggml.h"#include "hsum.h"@@ -26,9 +23,9 @@namespace {-class LLMM3X4 {+class SGEMMER {public:- LLMM3X4(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)+ SGEMMER(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth): A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {}@@ -41,18 +38,22 @@ class LLMM3X4 {if (m - m0 <= 0 || n - n0 <= 0)return;int mc, nc, mp, np;- if (m - m0 >= 3 && n - n0 >= 4) {+ if (VECTOR_REGISTERS >= 32 && m - m0 >= 5 && n - n0 >= 5) {+ mc = 5;+ nc = 5;+ gemm5x5(m0, m, n0, n);+ } else if (m - m0 >= 3 && n - n0 >= 4) {mc = 3;nc = 4;gemm3x4(m0, m, n0, n);- } else if (m - m0 >= 4 && n - n0 >= 1) {- mc = 4;- nc = 1;- gemm4x1(m0, m, n0, n);} else if (m - m0 >= 1 && n - n0 >= 4) {mc = 1;nc = 4;gemm1x4(m0, m, n0, n);+ } else if (m - m0 >= 4 && n - n0 >= 1) {+ mc = 4;+ nc = 1;+ gemm4x1(m0, m, n0, n);} else {mc = 1;nc = 1;@@ -65,6 +66,98 @@ class LLMM3X4 {mnpack(mp, m, np, n);}+ dontinline void gemm5x5(int m0, int m, int n0, int n) {+ BEGIN_KERNEL(5, 5)+ V c00 = zero();+ V c01 = zero();+ V c02 = zero();+ V c03 = zero();+ V c04 = zero();+ V c10 = zero();+ V c11 = zero();+ V c12 = zero();+ V c13 = zero();+ V c14 = zero();+ V c20 = zero();+ V c21 = zero();+ V c22 = zero();+ V c23 = zero();+ V c24 = zero();+ V c30 = zero();+ V c31 = zero();+ V c32 = zero();+ V c33 = zero();+ V c34 = zero();+ V c40 = zero();+ V c41 = zero();+ V c42 = zero();+ V c43 = zero();+ V c44 = zero();+ for (int l = 0; l < k; l += KN) {+ V k0 = load_b(B + ldb * (j + 0) + l);+ V k1 = load_b(B + ldb * (j + 1) + l);+ V k2 = load_b(B + ldb * (j + 2) + l);+ V k3 = load_b(B + ldb * (j + 3) + l);+ V k4 = load_b(B + ldb * (j + 4) + l);+ V a0 = load_a(A + lda * (i + 0) + l);+ c00 = madd(a0, k0, c00);+ c01 = madd(a0, k1, c01);+ c02 = madd(a0, k2, c02);+ c03 = madd(a0, k3, c03);+ c04 = madd(a0, k4, c04);+ V a1 = load_a(A + lda * (i + 1) + l);+ c10 = madd(a1, k0, c10);+ c11 = madd(a1, k1, c11);+ c12 = madd(a1, k2, c12);+ c13 = madd(a1, k3, c13);+ c14 = madd(a1, k4, c14);+ V a2 = load_a(A + lda * (i + 2) + l);+ c20 = madd(a2, k0, c20);+ c21 = madd(a2, k1, c21);+ c22 = madd(a2, k2, c22);+ c23 = madd(a2, k3, c23);+ c24 = madd(a2, k4, c24);+ V a3 = load_a(A + lda * (i + 3) + l);+ c30 = madd(a3, k0, c30);+ c31 = madd(a3, k1, c31);+ c32 = madd(a3, k2, c32);+ c33 = madd(a3, k3, c33);+ c34 = madd(a3, k4, c34);+ V a4 = load_a(A + lda * (i + 4) + l);+ c40 = madd(a4, k0, c40);+ c41 = madd(a4, k1, c41);+ c42 = madd(a4, k2, c42);+ c43 = madd(a4, k3, c43);+ c44 = madd(a4, k4, c44);+ }+ C[ldc * (j + 0) + (i + 0)] = hsum(c00);+ C[ldc * (j + 0) + (i + 1)] = hsum(c10);+ C[ldc * (j + 0) + (i + 2)] = hsum(c20);+ C[ldc * (j + 0) + (i + 3)] = hsum(c30);+ C[ldc * (j + 0) + (i + 4)] = hsum(c40);+ C[ldc * (j + 1) + (i + 0)] = hsum(c01);+ C[ldc * (j + 1) + (i + 1)] = hsum(c11);+ C[ldc * (j + 1) + (i + 2)] = hsum(c21);+ C[ldc * (j + 1) + (i + 3)] = hsum(c31);+ C[ldc * (j + 1) + (i + 4)] = hsum(c41);+ C[ldc * (j + 2) + (i + 0)] = hsum(c02);+ C[ldc * (j + 2) + (i + 1)] = hsum(c12);+ C[ldc * (j + 2) + (i + 2)] = hsum(c22);+ C[ldc * (j + 2) + (i + 3)] = hsum(c32);+ C[ldc * (j + 2) + (i + 4)] = hsum(c42);+ C[ldc * (j + 3) + (i + 0)] = hsum(c03);+ C[ldc * (j + 3) + (i + 1)] = hsum(c13);+ C[ldc * (j + 3) + (i + 2)] = hsum(c23);+ C[ldc * (j + 3) + (i + 3)] = hsum(c33);+ C[ldc * (j + 3) + (i + 4)] = hsum(c43);+ C[ldc * (j + 4) + (i + 0)] = hsum(c04);+ C[ldc * (j + 4) + (i + 1)] = hsum(c14);+ C[ldc * (j + 4) + (i + 2)] = hsum(c24);+ C[ldc * (j + 4) + (i + 3)] = hsum(c34);+ C[ldc * (j + 4) + (i + 4)] = hsum(c44);+ END_KERNEL()+ }+dontinline void gemm3x4(int m0, int m, int n0, int n) {BEGIN_KERNEL(3, 4)V c00 = zero();diff --git a/llamafile/llmmq0.inc b/llamafile/sgemmer0.incsimilarity index 60%rename from llamafile/llmmq0.increname to llamafile/sgemmer0.incindex 87fd1eed..310c8962 100644--- a/llamafile/llmmq0.inc+++ b/llamafile/sgemmer0.inc@@ -15,9 +15,6 @@// See the License for the specific language governing permissions and// limitations under the License.-#include <algorithm>-#include <cassert>-#include "llama.cpp/ggml-impl.h"#include "llama.cpp/ggml.h"@@ -25,13 +22,11 @@#include "kernel.h"#include "madd.h"-#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)-namespace {-class LLMMQ0 {+class SGEMMER0 {public:- LLMMQ0(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)+ SGEMMER0(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth): A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {}@@ -44,14 +39,18 @@ class LLMMQ0 {if (m - m0 <= 0 || n - n0 <= 0)return;int mc, nc, mp, np;- if (m - m0 >= 1 && n - n0 >= 4) {- mc = 1;- nc = 4;- gemm1x4(m0, m, n0, n);+ if (m - m0 >= 4 && n - n0 >= 3) {+ mc = 4;+ nc = 3;+ gemm4x3(m0, m, n0, n);} else if (m - m0 >= 4 && n - n0 >= 1) {mc = 4;nc = 1;gemm4x1(m0, m, n0, n);+ } else if (m - m0 >= 1 && n - n0 >= 4) {+ mc = 1;+ nc = 4;+ gemm1x4(m0, m, n0, n);} else {mc = 1;nc = 1;@@ -64,6 +63,98 @@ class LLMMQ0 {mnpack(mp, m, np, n);}+ dontinline void gemm4x3(int m0, int m, int n0, int n) {+ BEGIN_KERNEL(4, 3)+ int k2 = k / 32;+ __m256 c00 = _mm256_setzero_ps();+ __m256 c10 = _mm256_setzero_ps();+ __m256 c20 = _mm256_setzero_ps();+ __m256 c30 = _mm256_setzero_ps();+ __m256 c01 = _mm256_setzero_ps();+ __m256 c11 = _mm256_setzero_ps();+ __m256 c21 = _mm256_setzero_ps();+ __m256 c31 = _mm256_setzero_ps();+ __m256 c02 = _mm256_setzero_ps();+ __m256 c12 = _mm256_setzero_ps();+ __m256 c22 = _mm256_setzero_ps();+ __m256 c32 = _mm256_setzero_ps();+ const TA *Ap0 = A + lda * (i + 0);+ const TA *Ap1 = A + lda * (i + 1);+ const TA *Ap2 = A + lda * (i + 2);+ const TA *Ap3 = A + lda * (i + 3);+ const TB *Bp0 = B + ldb * (j + 0);+ const TB *Bp1 = B + ldb * (j + 1);+ const TB *Bp2 = B + ldb * (j + 2);+ for (int l = 0; l < k2; ++l) {+ float da0 = unhalf(Ap0[l].d);+ float da1 = unhalf(Ap1[l].d);+ float da2 = unhalf(Ap2[l].d);+ float da3 = unhalf(Ap3[l].d);+ __m256i e0 = load(Ap0 + l);+ __m256i e1 = load(Ap1 + l);+ __m256i e2 = load(Ap2 + l);+ __m256i e3 = load(Ap3 + l);+ float db0 = unhalf(Bp0[l].d);+ __m256 d00 = _mm256_set1_ps(da0 * db0);+ __m256 d10 = _mm256_set1_ps(da1 * db0);+ __m256 d20 = _mm256_set1_ps(da2 * db0);+ __m256 d30 = _mm256_set1_ps(da3 * db0);+ __m256i f0 = load(Bp0 + l);+ __m256i u0 = _mm256_sign_epi8(f0, f0);+ __m256i s00 = _mm256_sign_epi8(e0, f0);+ __m256i s10 = _mm256_sign_epi8(e1, f0);+ __m256i s20 = _mm256_sign_epi8(e2, f0);+ __m256i s30 = _mm256_sign_epi8(e3, f0);+ c00 = madd(d00, updot(u0, s00), c00);+ c10 = madd(d10, updot(u0, s10), c10);+ c20 = madd(d20, updot(u0, s20), c20);+ c30 = madd(d30, updot(u0, s30), c30);+ float db1 = unhalf(Bp1[l].d);+ __m256 d01 = _mm256_set1_ps(da0 * db1);+ __m256 d11 = _mm256_set1_ps(da1 * db1);+ __m256 d21 = _mm256_set1_ps(da2 * db1);+ __m256 d31 = _mm256_set1_ps(da3 * db1);+ __m256i f1 = load(Bp1 + l);+ __m256i u1 = _mm256_sign_epi8(f1, f1);+ __m256i s01 = _mm256_sign_epi8(e0, f1);+ __m256i s11 = _mm256_sign_epi8(e1, f1);+ __m256i s21 = _mm256_sign_epi8(e2, f1);+ __m256i s31 = _mm256_sign_epi8(e3, f1);+ c01 = madd(d01, updot(u1, s01), c01);+ c11 = madd(d11, updot(u1, s11), c11);+ c21 = madd(d21, updot(u1, s21), c21);+ c31 = madd(d31, updot(u1, s31), c31);+ float db2 = unhalf(Bp2[l].d);+ __m256 d02 = _mm256_set1_ps(da0 * db2);+ __m256 d12 = _mm256_set1_ps(da1 * db2);+ __m256 d22 = _mm256_set1_ps(da2 * db2);+ __m256 d32 = _mm256_set1_ps(da3 * db2);+ __m256i f2 = load(Bp2 + l);+ __m256i u2 = _mm256_sign_epi8(f2, f2);+ __m256i s02 = _mm256_sign_epi8(e0, f2);+ __m256i s12 = _mm256_sign_epi8(e1, f2);+ __m256i s22 = _mm256_sign_epi8(e2, f2);+ __m256i s32 = _mm256_sign_epi8(e3, f2);+ c02 = madd(d02, updot(u2, s02), c02);+ c12 = madd(d12, updot(u2, s12), c12);+ c22 = madd(d22, updot(u2, s22), c22);+ c32 = madd(d32, updot(u2, s32), c32);+ }+ C[ldc * (j + 0) + (i + 0)] = hsum(c00);+ C[ldc * (j + 0) + (i + 1)] = hsum(c10);+ C[ldc * (j + 0) + (i + 2)] = hsum(c20);+ C[ldc * (j + 0) + (i + 3)] = hsum(c30);+ C[ldc * (j + 1) + (i + 0)] = hsum(c01);+ C[ldc * (j + 1) + (i + 1)] = hsum(c11);+ C[ldc * (j + 1) + (i + 2)] = hsum(c21);+ C[ldc * (j + 1) + (i + 3)] = hsum(c31);+ C[ldc * (j + 2) + (i + 0)] = hsum(c02);+ C[ldc * (j + 2) + (i + 1)] = hsum(c12);+ C[ldc * (j + 2) + (i + 2)] = hsum(c22);+ C[ldc * (j + 2) + (i + 3)] = hsum(c32);+ END_KERNEL()+ }+dontinline void gemm4x1(int m0, int m, int n0, int n) {BEGIN_KERNEL(4, 1)int k2 = k / 32;diff --git a/llamafile/sgemmer1.inc b/llamafile/sgemmer1.incnew file mode 100644index 00000000..92b79fe4--- /dev/null+++ b/llamafile/sgemmer1.inc@@ -0,0 +1,219 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "llama.cpp/ggml-impl.h"+#include "llama.cpp/ggml.h"++#include "hsum.h"+#include "kernel.h"+#include "madd.h"++namespace {++class SGEMMER1 {+ public:+ SGEMMER1(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {+ }++ void matmul(int m, int n) {+ mnpack(0, m, 0, n);+ }++ private:+ dontinline void mnpack(int m0, int m, int n0, int n) {+ if (m - m0 <= 0 || n - n0 <= 0)+ return;+ int mc, nc, mp, np;+ if (VECTOR_REGISTERS >= 32 && m - m0 >= 4 && n - n0 >= 2) {+ mc = 4;+ nc = 2;+ gemm4x2(m0, m, n0, n);+ } else if (m - m0 >= 4 && n - n0 >= 1) {+ mc = 4;+ nc = 1;+ gemm4x1(m0, m, n0, n);+ } else {+ mc = 1;+ nc = 1;+ gemm1x1(m0, m, n0, n);+ }+ mp = m0 + (m - m0) / mc * mc;+ np = n0 + (n - n0) / nc * nc;+ mnpack(mp, m, n0, np);+ mnpack(m0, mp, np, n);+ mnpack(mp, m, np, n);+ }++ dontinline void gemm4x2(int m0, int m, int n0, int n) {+ BEGIN_KERNEL(4, 2)+ int k2 = k / 32;+ __m128 s0 = _mm_setzero_ps();+ __m128 s1 = _mm_setzero_ps();+ __m256 c00 = _mm256_setzero_ps();+ __m256 c01 = _mm256_setzero_ps();+ __m256 c02 = _mm256_setzero_ps();+ __m256 c03 = _mm256_setzero_ps();+ __m256 c10 = _mm256_setzero_ps();+ __m256 c11 = _mm256_setzero_ps();+ __m256 c12 = _mm256_setzero_ps();+ __m256 c13 = _mm256_setzero_ps();+ const TA *Ap0 = A + lda * (i + 0);+ const TA *Ap1 = A + lda * (i + 1);+ const TA *Ap2 = A + lda * (i + 2);+ const TA *Ap3 = A + lda * (i + 3);+ const TB *Bp0 = B + ldb * (j + 0);+ const TB *Bp1 = B + ldb * (j + 1);+ for (int l = 0; l < k2; ++l) {+ __m128 sa = _mm_set_ps(unhalf(Ap3[l].dm >> 16), //+ unhalf(Ap2[l].dm >> 16), //+ unhalf(Ap1[l].dm >> 16), //+ unhalf(Ap0[l].dm >> 16));+ s0 = madd(_mm_set1_ps(unhalf(Bp0[l].ds >> 16)), sa, s0);+ s1 = madd(_mm_set1_ps(unhalf(Bp1[l].ds >> 16)), sa, s1);+ __m256i f0 = load(Bp0 + l);+ __m256 d0 = _mm256_set1_ps(unhalf(Bp0[l].ds));+ c00 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap0[l].dm)), d0),+ updot(load(Ap0 + l), f0), c00);+ c01 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap1[l].dm)), d0),+ updot(load(Ap1 + l), f0), c01);+ c02 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap2[l].dm)), d0),+ updot(load(Ap2 + l), f0), c02);+ c03 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap3[l].dm)), d0),+ updot(load(Ap3 + l), f0), c03);+ __m256i f1 = load(Bp1 + l);+ __m256 d1 = _mm256_set1_ps(unhalf(Bp1[l].ds));+ c10 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap0[l].dm)), d1),+ updot(load(Ap0 + l), f1), c10);+ c11 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap1[l].dm)), d1),+ updot(load(Ap1 + l), f1), c11);+ c12 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap2[l].dm)), d1),+ updot(load(Ap2 + l), f1), c12);+ c13 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap3[l].dm)), d1),+ updot(load(Ap3 + l), f1), c13);+ }+ C[ldc * (j + 0) + (i + 0)] = hsum(c00) + s0[0];+ C[ldc * (j + 0) + (i + 1)] = hsum(c01) + s0[1];+ C[ldc * (j + 0) + (i + 2)] = hsum(c02) + s0[2];+ C[ldc * (j + 0) + (i + 3)] = hsum(c03) + s0[3];+ C[ldc * (j + 1) + (i + 0)] = hsum(c10) + s1[0];+ C[ldc * (j + 1) + (i + 1)] = hsum(c11) + s1[1];+ C[ldc * (j + 1) + (i + 2)] = hsum(c12) + s1[2];+ C[ldc * (j + 1) + (i + 3)] = hsum(c13) + s1[3];+ END_KERNEL()+ }++ dontinline void gemm4x1(int m0, int m, int n0, int n) {+ BEGIN_KERNEL(4, 1)+ int k2 = k / 32;+ __m128 s = _mm_setzero_ps();+ __m256 c0 = _mm256_setzero_ps();+ __m256 c1 = _mm256_setzero_ps();+ __m256 c2 = _mm256_setzero_ps();+ __m256 c3 = _mm256_setzero_ps();+ const TA *Ap0 = A + lda * (i + 0);+ const TA *Ap1 = A + lda * (i + 1);+ const TA *Ap2 = A + lda * (i + 2);+ const TA *Ap3 = A + lda * (i + 3);+ const TB *Bp = B + ldb * j;+ for (int l = 0; l < k2; ++l) {+ s = madd(_mm_set1_ps(unhalf(Bp[l].ds >> 16)),+ _mm_set_ps(unhalf(Ap3[l].dm >> 16), //+ unhalf(Ap2[l].dm >> 16), //+ unhalf(Ap1[l].dm >> 16), //+ unhalf(Ap0[l].dm >> 16)),+ s);+ __m256i f = load(Bp + l);+ __m256 d1 = _mm256_set1_ps(unhalf(Bp[l].ds));+ c0 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap0[l].dm)), d1), updot(load(Ap0 + l), f),+ c0);+ c1 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap1[l].dm)), d1), updot(load(Ap1 + l), f),+ c1);+ c2 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap2[l].dm)), d1), updot(load(Ap2 + l), f),+ c2);+ c3 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap3[l].dm)), d1), updot(load(Ap3 + l), f),+ c3);+ }+ C[ldc * j + (i + 0)] = hsum(c0) + s[0];+ C[ldc * j + (i + 1)] = hsum(c1) + s[1];+ C[ldc * j + (i + 2)] = hsum(c2) + s[2];+ C[ldc * j + (i + 3)] = hsum(c3) + s[3];+ END_KERNEL()+ }++ dontinline void gemm1x1(int m0, int m, int n0, int n) {+ BEGIN_KERNEL(1, 1)+ int k2 = k / 32;+ float s = 0;+ __m256 c = _mm256_setzero_ps();+ const TA *Ap = A + lda * i;+ const TB *Bp = B + ldb * j;+ for (int l = 0; l < k2; ++l) {+ unsigned dm = Ap[l].dm;+ unsigned ds = Bp[l].ds;+ __m256 d0 = _mm256_set1_ps(unhalf(dm));+ __m256 d1 = _mm256_set1_ps(unhalf(ds));+ s += unhalf(dm >> 16) * unhalf(ds >> 16);+ __m256i e = load(Ap + l);+ __m256i f = load(Bp + l);+ c = madd(_mm256_mul_ps(d0, d1), updot(e, f), c);+ }+ C[ldc * j + i] = hsum(c) + s;+ END_KERNEL()+ }++ static inline nosideeffect __m256i load(const block_q8_1 *b) {+ return _mm256_loadu_si256((const __m256i *)b->qs);+ }++ static inline nosideeffect __m256i load(const block_q4_1 *b) {+ return denibble(b->qs);+ }++ static inline pureconst __m256 updot(__m256i u, __m256i s) {+ __m256i res;+#if defined(__AVXVNNI__) || defined(__AVX512VNNI__)+ res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);+#else+ res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));+#endif+ return _mm256_cvtepi32_ps(res);+ }++ static inline nosideeffect __m256i denibble(const uint8_t *p) {+ const __m128i tmp = _mm_loadu_si128((const __m128i *)p);+ const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);+ const __m256i lowMask = _mm256_set1_epi8(15);+ return _mm256_and_si256(lowMask, bytes);+ }++ static inline pureconst float unhalf(unsigned short d) {+ return GGML_FP16_TO_FP32(d);+ }++ const TA *const A;+ const TB *const B;+ TC *const C;+ const int k;+ const int lda;+ const int ldb;+ const int ldc;+ const int ith;+ const int nth;+};++} // namespace