Skip to content

Distribute and run LLMs with a single file.

Add support for AVX512

Justine Tunneye629e752fd8b
diff --git a/README.md b/README.md
index f016aadf..a6765b0d 100644
--- a/README.md
+++ b/README.md
@@ -343,9 +343,8 @@ llamafile supports the following CPUs:
print an error and refuse to run. This means that if you have an Intel
CPU, it needs to be Intel Sandybridge or newer (circa 2011+), and if
you have an AMD CPU, then it needs to be Bulldozer or newer (circa
- 2011+). Support for AVX2, FMA, F16C, and VNNI are conditionally
- enabled at runtime if you have a newer CPU. There's no support for
- AVX512 runtime dispatching yet.
+ 2011+). Support for AVX512, AVX2, FMA, F16C, and VNNI are
+ conditionally enabled at runtime if you have a newer CPU.
- **ARM64** microprocessors must have ARMv8a+. This means everything
from Apple Silicon to 64-bit Raspberry Pis will work, provided your
diff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.h
index 324b7217..e68b7287 100644
--- a/llama.cpp/ggml-impl.h
+++ b/llama.cpp/ggml-impl.h
@@ -101,9 +101,6 @@ typedef uint16_t ggml_fp16_internal_t;
#include <riscv_vector.h>
#endif
-#define GGML_FP16_TO_FP32__F16C(x) _cvtsh_ss(x)
-#define GGML_FP32_TO_FP16__F16C(x) _cvtss_sh(x, 0)
-
#ifdef __F16C__
#ifdef _MSC_VER
diff --git a/llama.cpp/ggml-quants.c b/llama.cpp/ggml-quants.c
index d8ef1ec6..e9bec78b 100644
--- a/llama.cpp/ggml-quants.c
+++ b/llama.cpp/ggml-quants.c
@@ -155,18 +155,9 @@ static inline __m256 sum_i16_pairs_float_avx2(const __m256i x) {
__attribute__((__target__("avx2,fma,f16c")))
static inline __m256 mul_sum_us8_pairs_float_avx2(const __m256i ax, const __m256i sy) {
- if (X86_HAVE(AVXVNNI)) {
- // {vex} vpdpbusd %ymm1,%ymm2,%ymm0
- register __m256i ymm2 asm("ymm2") = ax;
- register __m256i ymm1 asm("ymm1") = sy;
- register __m256i ymm0 asm("ymm0") = _mm256_setzero_si256();
- asm(".byte\t0xc4,0xe2,0x6d,0x50,0xc1" : "+x"(ymm0) : "x"(ymm1), "x"(ymm2));
- return _mm256_cvtepi32_ps(ymm0);
- } else {
- // Perform multiplication and create 16-bit values
- const __m256i dot = _mm256_maddubs_epi16(ax, sy);
- return sum_i16_pairs_float_avx2(dot);
- }
+ // Perform multiplication and create 16-bit values
+ const __m256i dot = _mm256_maddubs_epi16(ax, sy);
+ return sum_i16_pairs_float_avx2(dot);
}
__attribute__((__target__("avx")))
@@ -967,7 +958,7 @@ void quantize_row_q8_1_reference(const float * restrict x, block_q8_1 * restrict
const float d = amax / ((1 << 7) - 1);
const float id = d ? 1.0f/d : 0.0f;
- y[i].d = d;
+ y[i].d = GGML_FP32_TO_FP16(d);
int sum = 0;
@@ -982,7 +973,7 @@ void quantize_row_q8_1_reference(const float * restrict x, block_q8_1 * restrict
sum += y[i].qs[QK8_1/2 + j];
}
- y[i].s = sum*d;
+ y[i].s = GGML_FP32_TO_FP16(sum*d);
}
}
@@ -1010,7 +1001,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {
const float d = amax / ((1 << 7) - 1);
const float id = d ? 1.0f/d : 0.0f;
- y[i].d = d;
+ y[i].d = GGML_FP32_TO_FP16(d);
int32x4_t accv = vdupq_n_s32(0);
@@ -1026,7 +1017,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {
accv = vaddq_s32(accv, vi);
}
- y[i].s = d * vaddvq_s32(accv);
+ y[i].s = GGML_FP32_TO_FP16(d * vaddvq_s32(accv));
}
#elif defined(__wasm_simd128__)
for (int i = 0; i < nb; i++) {
@@ -1049,7 +1040,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {
const float d = amax / ((1 << 7) - 1);
const float id = d ? 1.0f/d : 0.0f;
- y[i].d = d;
+ y[i].d = GGML_FP32_TO_FP16(d);
v128_t accv = wasm_i32x4_splat(0);
@@ -1065,10 +1056,11 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {
accv = wasm_i32x4_add(accv, vi);
}
- y[i].s = d * (wasm_i32x4_extract_lane(accv, 0) +
- wasm_i32x4_extract_lane(accv, 1) +
- wasm_i32x4_extract_lane(accv, 2) +
- wasm_i32x4_extract_lane(accv, 3));
+ y[i].s = GGML_FP32_TO_FP16(
+ d * (wasm_i32x4_extract_lane(accv, 0) +
+ wasm_i32x4_extract_lane(accv, 1) +
+ wasm_i32x4_extract_lane(accv, 2) +
+ wasm_i32x4_extract_lane(accv, 3)));
}
#elif defined(__AVX2__) || defined(__AVX__)
for (int i = 0; i < nb; i++) {
@@ -1093,7 +1085,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {
// Quantize these floats
const float d = maxScalar / 127.f;
- y[i].d = d;
+ y[i].d = GGML_FP32_TO_FP16(d);
const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f;
const __m256 mul = _mm256_set1_ps( id );
@@ -1117,7 +1109,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {
#if defined(__AVX2__)
// Compute the sum of the quants and set y[i].s
- y[i].s = d * hsum_i32_8(_mm256_add_epi32(_mm256_add_epi32(i0, i1), _mm256_add_epi32(i2, i3)));
+ y[i].s = GGML_FP32_TO_FP16(d * hsum_i32_8(_mm256_add_epi32(_mm256_add_epi32(i0, i1), _mm256_add_epi32(i2, i3))));
// Convert int32 to int16
i0 = _mm256_packs_epi32( i0, i1 ); // 0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15
@@ -1147,7 +1139,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {
// Compute the sum of the quants and set y[i].s
const __m128i s0 = _mm_add_epi32(_mm_add_epi32(ni0, ni1), _mm_add_epi32(ni2, ni3));
const __m128i s1 = _mm_add_epi32(_mm_add_epi32(ni4, ni5), _mm_add_epi32(ni6, ni7));
- y[i].s = d * hsum_i32_4(_mm_add_epi32(s0, s1));
+ y[i].s = GGML_FP32_TO_FP16(d * hsum_i32_4(_mm_add_epi32(s0, s1)));
// Convert int32 to int16
ni0 = _mm_packs_epi32( ni0, ni1 );
@@ -1178,7 +1170,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {
const float d = amax / ((1 << 7) - 1);
const float id = d ? 1.0f/d : 0.0f;
- y[i].d = d;
+ y[i].d = GGML_FP32_TO_FP16(d);
vfloat32m4_t x0 = __riscv_vfmul_vf_f32m4(v_x, id, vl);
@@ -1195,7 +1187,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int k) {
// set y[i].s
int sum = __riscv_vmv_x_s_i16m1_i16(vwrs);
- y[i].s = sum*d;
+ y[i].s = GGML_FP32_TO_FP16(sum*d);
}
#else
GGML_UNUSED(nb);
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 9a04d838..efea3060 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -305,7 +305,7 @@ static void ggml_fp32_to_fp16_row_f16c(const float * x, ggml_fp16_t * y, int n)
_mm_storel_epi64((__m128i *)(y + i), y_vec);
}
for (; i < n; i++) {
- y[i] = GGML_FP32_TO_FP16__F16C(x[i]);
+ y[i] = GGML_FP32_TO_FP16(x[i]);
}
}
#pragma GCC pop_options
@@ -10874,6 +10874,25 @@ static void ggml_compute_forward_mul_mat(
}
}
}
+
+#if 0
+ if (dst->type == GGML_TYPE_F32) {
+ for (int j = 0; j < ne1; ++j) {
+ for (int i = 0; i < ne0; ++i) {
+ if (isnan(*(float *)((char *)dst->data + nb1*j + nb0*i))) {
+ fprintf(stderr, "error: matmul(%s, %s/%s, %s, m=%d, n=%d, k=%d) produced nan at (%d,%d)\n",
+ ggml_type_name(src0->type),
+ ggml_type_name(src1->type),
+ ggml_type_name(vec_dot_type),
+ ggml_type_name(dst->type),
+ ne01, ne11, ne00,
+ j, i);
+ exit(1);
+ }
+ }
+ }
+ }
+#endif
}
// ggml_compute_forward_mul_mat_id
diff --git a/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc
index ebbaa970..ccb38cf9 100644
--- a/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc
+++ b/llama.cpp/ggml_vec_dot_q2_K_q8_K_256.inc
@@ -82,8 +82,8 @@ static void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, size_t bs, c
for (int i = 0; i < nb; ++i) {
- const float d = y[i].d * GGML_FP16_TO_FP32__F16C(x[i].d);
- const float dmin = -y[i].d * GGML_FP16_TO_FP32__F16C(x[i].dmin);
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
const uint8_t * restrict q2 = x[i].qs;
const int8_t * restrict q8 = y[i].qs;
diff --git a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc
index aaf501f2..a8fc9901 100644
--- a/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc
+++ b/llama.cpp/ggml_vec_dot_q3_K_q8_K_256.inc
@@ -115,7 +115,7 @@ static void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, size_t bs, c
for (int i = 0; i < nb; ++i) {
- const float d = y[i].d * GGML_FP16_TO_FP32__F16C(x[i].d);
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
const uint8_t * restrict q3 = x[i].qs;
const int8_t * restrict q8 = y[i].qs;
diff --git a/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc
index 822395d5..8b07eabd 100644
--- a/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc
+++ b/llama.cpp/ggml_vec_dot_q4_0_q8_0.inc
@@ -59,7 +59,7 @@ void ggml_vec_dot_q4_0_q8_0(const int n, float * restrict s, size_t bs, const vo
// Main loop
for (int i = 0; i < nb; ++i) {
/* Compute combined scale for the block */
- const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32__F16C(x[i].d) * GGML_FP16_TO_FP32__F16C(y[i].d) );
+ const __m256 d = _mm256_set1_ps( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );
__m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);
diff --git a/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc b/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc
index 07ab3038..ecab2852 100644
--- a/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc
+++ b/llama.cpp/ggml_vec_dot_q4_1_q8_1.inc
@@ -3,10 +3,88 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, c
const int nb = n / qk;
assert(n % qk == 0);
+#if defined(__ARM_FEATURE_MATMUL_INT8)
+ assert((nrc == 2) || (nrc == 1));
+#else
+ assert(nrc == 1);
+#endif
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
const block_q4_1 * restrict x = vx;
const block_q8_1 * restrict y = vy;
+#if defined(__ARM_FEATURE_MATMUL_INT8)
+ if (nrc == 2) {
+ const block_q4_1 * restrict vx0 = vx;
+ const block_q4_1 * restrict vx1 = vx + bx;
+ const block_q8_1 * restrict vy0 = vy;
+ const block_q8_1 * restrict vy1 = vy + by;
+
+ float32x4_t sumv0 = vdupq_n_f32(0.0f);
+ float32x4_t summs0 = vdupq_n_f32(0.0f);
+
+ for (int i = 0; i < nb; i++) {
+ const block_q4_1 * restrict b_x0 = &vx0[i];
+ const block_q4_1 * restrict b_x1 = &vx1[i];
+ const block_q8_1 * restrict b_y0 = &vy0[i];
+ const block_q8_1 * restrict b_y1 = &vy1[i];
+
+ float32x4_t summs_t = {GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y0->s),
+ GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y0->s),
+ GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y1->s),
+ GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y1->s)};
+ summs0 += summs_t;
+
+ const uint8x16_t m4b = vdupq_n_u8(0x0F);
+
+ const uint8x16_t v0_0 = vld1q_u8(b_x0->qs);
+ const uint8x16_t v0_1 = vld1q_u8(b_x1->qs);
+
+ // 4-bit -> 8-bit
+ const int8x16_t x0_l = vreinterpretq_s8_u8(vandq_u8 (v0_0, m4b));
+ const int8x16_t x0_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));
+ const int8x16_t x1_l = vreinterpretq_s8_u8(vandq_u8 (v0_1, m4b));
+ const int8x16_t x1_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));
+
+ // load y
+ const int8x16_t y0_l = vld1q_s8(b_y0->qs);
+ const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);
+ const int8x16_t y1_l = vld1q_s8(b_y1->qs);
+ const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);
+
+ // mmla into int32x4_t
+ float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*b_y0->d,
+ GGML_FP16_TO_FP32(b_x0->d)*b_y1->d,
+ GGML_FP16_TO_FP32(b_x1->d)*b_y0->d,
+ GGML_FP16_TO_FP32(b_x1->d)*b_y1->d};
+
+ int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
+ int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
+
+ int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
+ int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));
+
+ int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
+ int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));
+
+ int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
+ int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
+ sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),
+ l1, r1)), l2, r2)), l3, r3))), scale);
+ }
+
+ float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);
+ float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);
+ sumv2 = sumv2 + summs0;
+
+ vst1_f32(s, vget_low_f32(sumv2));
+ vst1_f32(s + bs, vget_high_f32(sumv2));
+ return;
+ }
+#endif
// TODO: add WASM SIMD
#if defined(__ARM_NEON)
float32x4_t sumv0 = vdupq_n_f32(0.0f);
@@ -22,7 +100,7 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, c
const block_q8_1 * restrict y0 = &y[i + 0];
const block_q8_1 * restrict y1 = &y[i + 1];
- summs += GGML_FP16_TO_FP32(x0->m) * y0->s + GGML_FP16_TO_FP32(x1->m) * y1->s;
+ summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s) + GGML_FP16_TO_FP32(x1->m) * GGML_FP16_TO_FP32(y1->s);
const uint8x16_t m4b = vdupq_n_u8(0x0F);
@@ -45,8 +123,8 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, c
const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0l, v1_0l), v0_0h, v1_0h);
const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1l, v1_1l), v0_1h, v1_1h);
- sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*y0->d);
- sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*y1->d);
+ sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d));
+ sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
}
*s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs;
@@ -58,18 +136,10 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, c
// Main loop
for (int i = 0; i < nb; ++i) {
-#ifdef __AVX2__
- const float d0 = GGML_FP16_TO_FP32__F16C(x[i].d);
-#else
const float d0 = GGML_FP16_TO_FP32(x[i].d);
-#endif
- const float d1 = y[i].d;
+ const float d1 = GGML_FP16_TO_FP32(y[i].d);
-#ifdef __AVX2__
- summs += GGML_FP16_TO_FP32__F16C(x[i].m) * y[i].s;
-#else
- summs += GGML_FP16_TO_FP32(x[i].m) * y[i].s;
-#endif
+ summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);
const __m256 d0v = _mm256_set1_ps( d0 );
const __m256 d1v = _mm256_set1_ps( d1 );
@@ -78,25 +148,22 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, c
const __m256 d0d1 = _mm256_mul_ps( d0v, d1v );
// Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes
+
#if defined(__AVX2__)
- const __m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);
+ const __m256i qx = bytes_from_nibbles_32_avx2(x[i].qs);
#else
- const __m256i bx = bytes_from_nibbles_32_avx(x[i].qs);
+ const __m256i qx = bytes_from_nibbles_32_avx(x[i].qs);
#endif
- const __m256i by = _mm256_loadu_si256( (const __m256i *)y[i].qs );
+ const __m256i qy = _mm256_loadu_si256( (const __m256i *)y[i].qs );
#if defined(__AVX2__)
- const __m256 xy = mul_sum_us8_pairs_float_avx2(bx, by);
+ const __m256 xy = mul_sum_us8_pairs_float_avx2(qx, qy);
#else
- const __m256 xy = mul_sum_us8_pairs_float_avx(bx, by);
+ const __m256 xy = mul_sum_us8_pairs_float_avx(qx, qy);
#endif
// Accumulate d0*d1*x*y
-#if defined(__AVX2__)
- acc = _mm256_fmadd_ps( d0d1, xy, acc );
-#else
acc = _mm256_add_ps( _mm256_mul_ps( d0d1, xy ), acc );
-#endif
}
*s = hsum_float_8(acc) + summs;
@@ -129,7 +196,7 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, c
int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);
- sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s;
+ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);
}
*s = sumf;
@@ -147,7 +214,7 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, size_t bs, c
sumi += (v0 * y[i].qs[j]) + (v1 * y[i].qs[j + qk/2]);
}
- sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s;
+ sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d))*sumi + GGML_FP16_TO_FP32(x[i].m)*GGML_FP16_TO_FP32(y[i].s);
}
*s = sumf;
diff --git a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc
index 589fe471..d8fa48e5 100644
--- a/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc
+++ b/llama.cpp/ggml_vec_dot_q4_K_q8_K_256.inc
@@ -84,8 +84,8 @@ static void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, size_t bs, c
for (int i = 0; i < nb; ++i) {
- const float d = y[i].d * GGML_FP16_TO_FP32__F16C(x[i].d);
- const float dmin = -y[i].d * GGML_FP16_TO_FP32__F16C(x[i].dmin);
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
memcpy(utmp, x[i].scales, 12);
utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
diff --git a/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc
index 8ba7bae9..527ca5c3 100644
--- a/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc
+++ b/llama.cpp/ggml_vec_dot_q5_0_q8_0.inc
@@ -145,7 +145,7 @@ static void ggml_vec_dot_q5_0_q8_0(const int n, float * restrict s, size_t bs, c
// Main loop
for (int i = 0; i < nb; i++) {
/* Compute combined scale for the block */
- const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32__F16C(x[i].d) * GGML_FP16_TO_FP32__F16C(y[i].d));
+ const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));
__m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);
__m256i bxhi = bytes_from_bits_32_avx2(x[i].qh);
diff --git a/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc b/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc
index bc77430a..2255fddb 100644
--- a/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc
+++ b/llama.cpp/ggml_vec_dot_q5_1_q8_1.inc
@@ -156,9 +156,9 @@ static void ggml_vec_dot_q5_1_q8_1(const int n, float * restrict s, size_t bs, c
// Main loop
for (int i = 0; i < nb; i++) {
- const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32__F16C(x[i].d));
+ const __m256 dx = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d));
- summs += GGML_FP16_TO_FP32__F16C(x[i].m) * y[i].s;
+ summs += GGML_FP16_TO_FP32(x[i].m) * y[i].s;
__m256i bx = bytes_from_nibbles_32_avx2(x[i].qs);
__m256i bxhi = bytes_from_bits_32_avx2(x[i].qh);
diff --git a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc b/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc
index eb8852fa..61d829bf 100644
--- a/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc
+++ b/llama.cpp/ggml_vec_dot_q5_K_q8_K_256.inc
@@ -96,8 +96,8 @@ static void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, size_t bs, c
const int8_t * restrict q8 = y[i].qs;
#if QK_K == 256
- const float d = y[i].d * GGML_FP16_TO_FP32__F16C(x[i].d);
- const float dmin = -y[i].d * GGML_FP16_TO_FP32__F16C(x[i].dmin);
+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);
+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);
memcpy(utmp, x[i].scales, 12);
utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
diff --git a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc b/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc
index c90ee156..29455a66 100644
--- a/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc
+++ b/llama.cpp/ggml_vec_dot_q8_0_q8_0.inc
@@ -47,11 +47,7 @@ static void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, size_t bs, c
// Main loop
for (int i = 0; i < nb; ++i) {
// Compute combined scale for the block
-#ifdef __AVX2__
- const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32__F16C(x[i].d) * GGML_FP16_TO_FP32__F16C(y[i].d));
-#else
const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));
-#endif
__m256i bx = _mm256_loadu_si256((const __m256i *)x[i].qs);
__m256i by = _mm256_loadu_si256((const __m256i *)y[i].qs);
@@ -62,11 +58,7 @@ static void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, size_t bs, c
#endif
// Multiply q with scale and accumulate
-#if defined(__AVX2__)
- acc = _mm256_fmadd_ps( d, q, acc );
-#else
acc = _mm256_add_ps( _mm256_mul_ps( d, q ), acc );
-#endif
}
*s = hsum_float_8(acc);
diff --git a/llama.cpp/quantize/quantize.cpp b/llama.cpp/quantize/quantize.cpp
index 4fc2d565..9b7b9cae 100644
--- a/llama.cpp/quantize/quantize.cpp
+++ b/llama.cpp/quantize/quantize.cpp
@@ -219,6 +219,7 @@ int main(int argc, char ** argv) {
}
llama_model_quantize_params params = llama_model_quantize_default_params();
+ params.nthread = 8;
int arg_idx = 1;
std::string imatrix_file;
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 81e26a78..92379c6e 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -17,18 +17,6 @@ LLAMAFILE_OBJS = \
$(LLAMAFILE_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \
$(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o)
-o/$(MODE)/llamafile/sgemm_sss_avx.o: private TARGET_ARCH += -Xx86_64-mavx
-o/$(MODE)/llamafile/sgemm_sss_fma.o: private TARGET_ARCH += -Xx86_64-mfma
-o/$(MODE)/llamafile/sgemm_sss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f
-o/$(MODE)/llamafile/sgemm_hss_f16c.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mf16c
-o/$(MODE)/llamafile/sgemm_hss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f
-o/$(MODE)/llamafile/sgemm_qqs_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni
-o/$(MODE)/llamafile/sgemm_qqs_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni
-o/$(MODE)/llamafile/sgemm_qqs_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma
-o/$(MODE)/llamafile/sgemm_eqs_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni
-o/$(MODE)/llamafile/sgemm_eqs_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni
-o/$(MODE)/llamafile/sgemm_eqs_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma
-
o/$(MODE)/llamafile/zipalign: \
o/$(MODE)/llamafile/zipalign.o \
o/$(MODE)/llamafile/help.o \
@@ -52,6 +40,24 @@ o/$(MODE)/llamafile: \
o/$(MODE)/llamafile/zipcheck \
o/$(MODE)/llamafile/addnl
+################################################################################
+# microarchitectures
+
+o/$(MODE)/llamafile/sgemm_sss_avx.o: private TARGET_ARCH += -Xx86_64-mavx
+o/$(MODE)/llamafile/sgemm_sss_fma.o: private TARGET_ARCH += -Xx86_64-mfma
+o/$(MODE)/llamafile/sgemm_sss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f
+o/$(MODE)/llamafile/sgemm_hss_f16c.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mf16c
+o/$(MODE)/llamafile/sgemm_hss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f
+o/$(MODE)/llamafile/sgemm_q0q0s_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni
+o/$(MODE)/llamafile/sgemm_q0q0s_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni
+o/$(MODE)/llamafile/sgemm_q0q0s_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma
+o/$(MODE)/llamafile/sgemm_e0q0s_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni
+o/$(MODE)/llamafile/sgemm_e0q0s_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni
+o/$(MODE)/llamafile/sgemm_e0q0s_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma
+o/$(MODE)/llamafile/sgemm_e1q1s_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni
+o/$(MODE)/llamafile/sgemm_e1q1s_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni
+o/$(MODE)/llamafile/sgemm_e1q1s_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma
+
################################################################################
# testing
diff --git a/llamafile/kernel.h b/llamafile/kernel.h
index 5bdca973..dced6c4d 100644
--- a/llamafile/kernel.h
+++ b/llamafile/kernel.h
@@ -1,5 +1,11 @@
#pragma once
+#ifdef __AVX512F__
+#define VECTOR_REGISTERS 32
+#else
+#define VECTOR_REGISTERS 16
+#endif
+
#define BEGIN_KERNEL(RM, RN) \
int ytiles = (m - m0) / RM; \
int xtiles = (n - n0) / RN; \
@@ -16,3 +22,5 @@
int j = n0 + job % xtiles * RN;
#define END_KERNEL() }
+
+#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)
diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cpp
index 3fde8dd0..54ba1b21 100644
--- a/llamafile/sgemm.cpp
+++ b/llamafile/sgemm.cpp
@@ -60,34 +60,51 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,
if (Btype != GGML_TYPE_Q8_0)
return false;
if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))
- return llamafile_sgemm_qqs_avx512vnni(m, n, k, (const block_q8_0 *)A, lda,
- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,
- nth, task);
+ return llamafile_sgemm_q0q0s_avx512vnni(m, n, k, (const block_q8_0 *)A, lda,
+ (const block_q8_0 *)B, ldb, (float *)C, ldc,
+ ith, nth, task);
if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI) && !(k % 32))
- return llamafile_sgemm_qqs_avxvnni(m, n, k, (const block_q8_0 *)A, lda,
- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,
- nth, task);
+ return llamafile_sgemm_q0q0s_avxvnni(m, n, k, (const block_q8_0 *)A, lda,
+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,
+ nth, task);
if (X86_HAVE(FMA) && X86_HAVE(AVX2) && !(k % 32))
- return llamafile_sgemm_qqs_fma(m, n, k, (const block_q8_0 *)A, lda,
- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,
- task);
+ return llamafile_sgemm_q0q0s_fma(m, n, k, (const block_q8_0 *)A, lda,
+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,
+ task);
return false;
case GGML_TYPE_Q4_0:
if (Btype != GGML_TYPE_Q8_0)
return false;
if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))
- return llamafile_sgemm_eqs_avx512vnni(m, n, k, (const block_q4_0 *)A, lda,
- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,
- nth, task);
+ return llamafile_sgemm_e0q0s_avx512vnni(m, n, k, (const block_q4_0 *)A, lda,
+ (const block_q8_0 *)B, ldb, (float *)C, ldc,
+ ith, nth, task);
if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI) && !(k % 32))
- return llamafile_sgemm_eqs_avxvnni(m, n, k, (const block_q4_0 *)A, lda,
- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,
- nth, task);
+ return llamafile_sgemm_e0q0s_avxvnni(m, n, k, (const block_q4_0 *)A, lda,
+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,
+ nth, task);
if (X86_HAVE(FMA) && X86_HAVE(AVX2) && !(k % 32))
- return llamafile_sgemm_eqs_fma(m, n, k, (const block_q4_0 *)A, lda,
- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,
- task);
+ return llamafile_sgemm_e0q0s_fma(m, n, k, (const block_q4_0 *)A, lda,
+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,
+ task);
+ return false;
+
+ case GGML_TYPE_Q4_1:
+ if (Btype != GGML_TYPE_Q8_1)
+ return false;
+ if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))
+ return llamafile_sgemm_e1q1s_avx512vnni(m, n, k, (const block_q4_1 *)A, lda,
+ (const block_q8_1 *)B, ldb, (float *)C, ldc,
+ ith, nth, task);
+ if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI) && !(k % 32))
+ return llamafile_sgemm_e1q1s_avxvnni(m, n, k, (const block_q4_1 *)A, lda,
+ (const block_q8_1 *)B, ldb, (float *)C, ldc, ith,
+ nth, task);
+ if (X86_HAVE(FMA) && X86_HAVE(AVX2) && !(k % 32))
+ return llamafile_sgemm_e1q1s_fma(m, n, k, (const block_q4_1 *)A, lda,
+ (const block_q8_1 *)B, ldb, (float *)C, ldc, ith, nth,
+ task);
return false;
default:
diff --git a/llamafile/sgemm.h b/llamafile/sgemm.h
index 14121d93..5c2c2db2 100644
--- a/llamafile/sgemm.h
+++ b/llamafile/sgemm.h
@@ -17,19 +17,26 @@ bool llamafile_sgemm_hss_f16c(int, int, int, const unsigned short *, int, const
bool llamafile_sgemm_hss_avx512f(int, int, int, const unsigned short *, int, const float *, int,
float *, int, int, int, int);
-bool llamafile_sgemm_qqs_avx512vnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,
- float *, int, int, int, int);
-bool llamafile_sgemm_qqs_avxvnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,
- float *, int, int, int, int);
-bool llamafile_sgemm_qqs_fma(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,
- float *, int, int, int, int);
+bool llamafile_sgemm_q0q0s_avx512vnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *,
+ int, float *, int, int, int, int);
+bool llamafile_sgemm_q0q0s_avxvnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,
+ float *, int, int, int, int);
+bool llamafile_sgemm_q0q0s_fma(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,
+ float *, int, int, int, int);
-bool llamafile_sgemm_eqs_avx512vnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,
- float *, int, int, int, int);
-bool llamafile_sgemm_eqs_avxvnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,
- float *, int, int, int, int);
-bool llamafile_sgemm_eqs_fma(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,
- float *, int, int, int, int);
+bool llamafile_sgemm_e0q0s_avx512vnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *,
+ int, float *, int, int, int, int);
+bool llamafile_sgemm_e0q0s_avxvnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,
+ float *, int, int, int, int);
+bool llamafile_sgemm_e0q0s_fma(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,
+ float *, int, int, int, int);
+
+bool llamafile_sgemm_e1q1s_avx512vnni(int, int, int, const block_q4_1 *, int, const block_q8_1 *,
+ int, float *, int, int, int, int);
+bool llamafile_sgemm_e1q1s_avxvnni(int, int, int, const block_q4_1 *, int, const block_q8_1 *, int,
+ float *, int, int, int, int);
+bool llamafile_sgemm_e1q1s_fma(int, int, int, const block_q4_1 *, int, const block_q8_1 *, int,
+ float *, int, int, int, int);
#ifdef __cplusplus
}
diff --git a/llamafile/sgemm_eqs_avxvnni.cpp b/llamafile/sgemm_e0q0s_avx512vnni.cpp
similarity index 78%
rename from llamafile/sgemm_eqs_avxvnni.cpp
rename to llamafile/sgemm_e0q0s_avx512vnni.cpp
index 22a572d4..18445ec0 100644
--- a/llamafile/sgemm_eqs_avxvnni.cpp
+++ b/llamafile/sgemm_e0q0s_avx512vnni.cpp
@@ -22,13 +22,13 @@
#define TA block_q4_0
#define TB block_q8_0
#define TC float
-#include "llmmq0.inc"
+#include "sgemmer0.inc"
-bool llamafile_sgemm_eqs_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
+bool llamafile_sgemm_e0q0s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,
+ int ldb, TC *C, int ldc, int ith, int nth, int task) {
if (task != GGML_TASK_TYPE_COMPUTE)
return true;
- LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
tb.matmul(m, n);
return true;
}
diff --git a/llamafile/sgemm_eqs_avx512vnni.cpp b/llamafile/sgemm_e0q0s_avxvnni.cpp
similarity index 78%
rename from llamafile/sgemm_eqs_avx512vnni.cpp
rename to llamafile/sgemm_e0q0s_avxvnni.cpp
index 78c00eb9..92598457 100644
--- a/llamafile/sgemm_eqs_avx512vnni.cpp
+++ b/llamafile/sgemm_e0q0s_avxvnni.cpp
@@ -22,13 +22,13 @@
#define TA block_q4_0
#define TB block_q8_0
#define TC float
-#include "llmmq0.inc"
+#include "sgemmer0.inc"
-bool llamafile_sgemm_eqs_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
+bool llamafile_sgemm_e0q0s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
if (task != GGML_TASK_TYPE_COMPUTE)
return true;
- LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
tb.matmul(m, n);
return true;
}
diff --git a/llamafile/sgemm_eqs_fma.cpp b/llamafile/sgemm_e0q0s_fma.cpp
similarity index 79%
rename from llamafile/sgemm_eqs_fma.cpp
rename to llamafile/sgemm_e0q0s_fma.cpp
index 9bb95430..bd69b661 100644
--- a/llamafile/sgemm_eqs_fma.cpp
+++ b/llamafile/sgemm_e0q0s_fma.cpp
@@ -22,13 +22,13 @@
#define TA block_q4_0
#define TB block_q8_0
#define TC float
-#include "llmmq0.inc"
+#include "sgemmer0.inc"
-bool llamafile_sgemm_eqs_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,
- int ldc, int ith, int nth, int task) {
+bool llamafile_sgemm_e0q0s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
if (task != GGML_TASK_TYPE_COMPUTE)
return true;
- LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
tb.matmul(m, n);
return true;
}
diff --git a/llamafile/sgemm_e1q1s_avx512vnni.cpp b/llamafile/sgemm_e1q1s_avx512vnni.cpp
new file mode 100644
index 00000000..e2a4836b
--- /dev/null
+++ b/llamafile/sgemm_e1q1s_avx512vnni.cpp
@@ -0,0 +1,36 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+
+#define TA block_q4_1
+#define TB block_q8_1
+#define TC float
+#include "sgemmer1.inc"
+
+bool llamafile_sgemm_e1q1s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,
+ int ldb, TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ SGEMMER1 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_e1q1s_avxvnni.cpp b/llamafile/sgemm_e1q1s_avxvnni.cpp
new file mode 100644
index 00000000..9114427e
--- /dev/null
+++ b/llamafile/sgemm_e1q1s_avxvnni.cpp
@@ -0,0 +1,36 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+
+#define TA block_q4_1
+#define TB block_q8_1
+#define TC float
+#include "sgemmer1.inc"
+
+bool llamafile_sgemm_e1q1s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ SGEMMER1 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_e1q1s_fma.cpp b/llamafile/sgemm_e1q1s_fma.cpp
new file mode 100644
index 00000000..399232e2
--- /dev/null
+++ b/llamafile/sgemm_e1q1s_fma.cpp
@@ -0,0 +1,36 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+
+#define TA block_q4_1
+#define TB block_q8_1
+#define TC float
+#include "sgemmer1.inc"
+
+bool llamafile_sgemm_e1q1s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ SGEMMER1 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_hss_avx512f.cpp b/llamafile/sgemm_hss_avx512f.cpp
index 17d88bcc..394f4124 100644
--- a/llamafile/sgemm_hss_avx512f.cpp
+++ b/llamafile/sgemm_hss_avx512f.cpp
@@ -27,13 +27,13 @@
#define zero() _mm512_setzero_ps()
#define load_a(p) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(p)))
#define load_b(p) _mm512_loadu_ps(p)
-#include "llmm3x4.inc"
+#include "sgemmer.inc"
bool llamafile_sgemm_hss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
TC *C, int ldc, int ith, int nth, int task) {
if (task != GGML_TASK_TYPE_COMPUTE)
return true;
- LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
tb.matmul(m, n);
return true;
}
diff --git a/llamafile/sgemm_hss_f16c.cpp b/llamafile/sgemm_hss_f16c.cpp
index ed16d736..f2f45473 100644
--- a/llamafile/sgemm_hss_f16c.cpp
+++ b/llamafile/sgemm_hss_f16c.cpp
@@ -27,13 +27,13 @@
#define zero() _mm256_setzero_ps()
#define load_a(p) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(p)))
#define load_b(p) _mm256_loadu_ps(p)
-#include "llmm3x4.inc"
+#include "sgemmer.inc"
bool llamafile_sgemm_hss_f16c(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
TC *C, int ldc, int ith, int nth, int task) {
if (task != GGML_TASK_TYPE_COMPUTE)
return true;
- LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
tb.matmul(m, n);
return true;
}
diff --git a/llamafile/sgemm_qqs_avx512vnni.cpp b/llamafile/sgemm_q0q0s_avx512vnni.cpp
similarity index 78%
rename from llamafile/sgemm_qqs_avx512vnni.cpp
rename to llamafile/sgemm_q0q0s_avx512vnni.cpp
index bd9254e7..839b5c2d 100644
--- a/llamafile/sgemm_qqs_avx512vnni.cpp
+++ b/llamafile/sgemm_q0q0s_avx512vnni.cpp
@@ -22,13 +22,13 @@
#define TA block_q8_0
#define TB block_q8_0
#define TC float
-#include "llmmq0.inc"
+#include "sgemmer0.inc"
-bool llamafile_sgemm_qqs_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
+bool llamafile_sgemm_q0q0s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,
+ int ldb, TC *C, int ldc, int ith, int nth, int task) {
if (task != GGML_TASK_TYPE_COMPUTE)
return true;
- LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
tb.matmul(m, n);
return true;
}
diff --git a/llamafile/sgemm_qqs_avxvnni.cpp b/llamafile/sgemm_q0q0s_avxvnni.cpp
similarity index 78%
rename from llamafile/sgemm_qqs_avxvnni.cpp
rename to llamafile/sgemm_q0q0s_avxvnni.cpp
index cdde2d96..4c6eb23c 100644
--- a/llamafile/sgemm_qqs_avxvnni.cpp
+++ b/llamafile/sgemm_q0q0s_avxvnni.cpp
@@ -22,13 +22,13 @@
#define TA block_q8_0
#define TB block_q8_0
#define TC float
-#include "llmmq0.inc"
+#include "sgemmer0.inc"
-bool llamafile_sgemm_qqs_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
+bool llamafile_sgemm_q0q0s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
if (task != GGML_TASK_TYPE_COMPUTE)
return true;
- LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
tb.matmul(m, n);
return true;
}
diff --git a/llamafile/sgemm_qqs_fma.cpp b/llamafile/sgemm_q0q0s_fma.cpp
similarity index 79%
rename from llamafile/sgemm_qqs_fma.cpp
rename to llamafile/sgemm_q0q0s_fma.cpp
index 32b82c33..99f71dd5 100644
--- a/llamafile/sgemm_qqs_fma.cpp
+++ b/llamafile/sgemm_q0q0s_fma.cpp
@@ -22,13 +22,13 @@
#define TA block_q8_0
#define TB block_q8_0
#define TC float
-#include "llmmq0.inc"
+#include "sgemmer0.inc"
-bool llamafile_sgemm_qqs_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,
- int ldc, int ith, int nth, int task) {
+bool llamafile_sgemm_q0q0s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
if (task != GGML_TASK_TYPE_COMPUTE)
return true;
- LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
tb.matmul(m, n);
return true;
}
diff --git a/llamafile/sgemm_sss_avx.cpp b/llamafile/sgemm_sss_avx.cpp
index 6c73fc2f..97bd377d 100644
--- a/llamafile/sgemm_sss_avx.cpp
+++ b/llamafile/sgemm_sss_avx.cpp
@@ -27,13 +27,13 @@
#define zero() _mm256_setzero_ps()
#define load_a(p) _mm256_loadu_ps(p)
#define load_b(p) _mm256_loadu_ps(p)
-#include "llmm3x4.inc"
+#include "sgemmer.inc"
bool llamafile_sgemm_sss_avx(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,
int ldc, int ith, int nth, int task) {
if (task != GGML_TASK_TYPE_COMPUTE)
return true;
- LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
tb.matmul(m, n);
return true;
}
diff --git a/llamafile/sgemm_sss_avx512f.cpp b/llamafile/sgemm_sss_avx512f.cpp
index 1db08d29..7ccb188c 100644
--- a/llamafile/sgemm_sss_avx512f.cpp
+++ b/llamafile/sgemm_sss_avx512f.cpp
@@ -27,13 +27,13 @@
#define zero() _mm512_setzero_ps()
#define load_a(p) _mm512_loadu_ps(p)
#define load_b(p) _mm512_loadu_ps(p)
-#include "llmm3x4.inc"
+#include "sgemmer.inc"
bool llamafile_sgemm_sss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
TC *C, int ldc, int ith, int nth, int task) {
if (task != GGML_TASK_TYPE_COMPUTE)
return true;
- LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
tb.matmul(m, n);
return true;
}
diff --git a/llamafile/sgemm_sss_fma.cpp b/llamafile/sgemm_sss_fma.cpp
index 84813ee7..f018bfe6 100644
--- a/llamafile/sgemm_sss_fma.cpp
+++ b/llamafile/sgemm_sss_fma.cpp
@@ -27,13 +27,13 @@
#define zero() _mm256_setzero_ps()
#define load_a(p) _mm256_loadu_ps(p)
#define load_b(p) _mm256_loadu_ps(p)
-#include "llmm3x4.inc"
+#include "sgemmer.inc"
bool llamafile_sgemm_sss_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,
int ldc, int ith, int nth, int task) {
if (task != GGML_TASK_TYPE_COMPUTE)
return true;
- LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
tb.matmul(m, n);
return true;
}
diff --git a/llamafile/llmm3x4.inc b/llamafile/sgemmer.inc
similarity index 61%
rename from llamafile/llmm3x4.inc
rename to llamafile/sgemmer.inc
index c0d6c66f..54b9e5e5 100644
--- a/llamafile/llmm3x4.inc
+++ b/llamafile/sgemmer.inc
@@ -15,9 +15,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-#include <algorithm>
-#include <cassert>
-
#include "llama.cpp/ggml.h"
#include "hsum.h"
@@ -26,9 +23,9 @@
namespace {
-class LLMM3X4 {
+class SGEMMER {
public:
- LLMM3X4(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)
+ SGEMMER(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)
: A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
}
@@ -41,18 +38,22 @@ class LLMM3X4 {
if (m - m0 <= 0 || n - n0 <= 0)
return;
int mc, nc, mp, np;
- if (m - m0 >= 3 && n - n0 >= 4) {
+ if (VECTOR_REGISTERS >= 32 && m - m0 >= 5 && n - n0 >= 5) {
+ mc = 5;
+ nc = 5;
+ gemm5x5(m0, m, n0, n);
+ } else if (m - m0 >= 3 && n - n0 >= 4) {
mc = 3;
nc = 4;
gemm3x4(m0, m, n0, n);
- } else if (m - m0 >= 4 && n - n0 >= 1) {
- mc = 4;
- nc = 1;
- gemm4x1(m0, m, n0, n);
} else if (m - m0 >= 1 && n - n0 >= 4) {
mc = 1;
nc = 4;
gemm1x4(m0, m, n0, n);
+ } else if (m - m0 >= 4 && n - n0 >= 1) {
+ mc = 4;
+ nc = 1;
+ gemm4x1(m0, m, n0, n);
} else {
mc = 1;
nc = 1;
@@ -65,6 +66,98 @@ class LLMM3X4 {
mnpack(mp, m, np, n);
}
+ dontinline void gemm5x5(int m0, int m, int n0, int n) {
+ BEGIN_KERNEL(5, 5)
+ V c00 = zero();
+ V c01 = zero();
+ V c02 = zero();
+ V c03 = zero();
+ V c04 = zero();
+ V c10 = zero();
+ V c11 = zero();
+ V c12 = zero();
+ V c13 = zero();
+ V c14 = zero();
+ V c20 = zero();
+ V c21 = zero();
+ V c22 = zero();
+ V c23 = zero();
+ V c24 = zero();
+ V c30 = zero();
+ V c31 = zero();
+ V c32 = zero();
+ V c33 = zero();
+ V c34 = zero();
+ V c40 = zero();
+ V c41 = zero();
+ V c42 = zero();
+ V c43 = zero();
+ V c44 = zero();
+ for (int l = 0; l < k; l += KN) {
+ V k0 = load_b(B + ldb * (j + 0) + l);
+ V k1 = load_b(B + ldb * (j + 1) + l);
+ V k2 = load_b(B + ldb * (j + 2) + l);
+ V k3 = load_b(B + ldb * (j + 3) + l);
+ V k4 = load_b(B + ldb * (j + 4) + l);
+ V a0 = load_a(A + lda * (i + 0) + l);
+ c00 = madd(a0, k0, c00);
+ c01 = madd(a0, k1, c01);
+ c02 = madd(a0, k2, c02);
+ c03 = madd(a0, k3, c03);
+ c04 = madd(a0, k4, c04);
+ V a1 = load_a(A + lda * (i + 1) + l);
+ c10 = madd(a1, k0, c10);
+ c11 = madd(a1, k1, c11);
+ c12 = madd(a1, k2, c12);
+ c13 = madd(a1, k3, c13);
+ c14 = madd(a1, k4, c14);
+ V a2 = load_a(A + lda * (i + 2) + l);
+ c20 = madd(a2, k0, c20);
+ c21 = madd(a2, k1, c21);
+ c22 = madd(a2, k2, c22);
+ c23 = madd(a2, k3, c23);
+ c24 = madd(a2, k4, c24);
+ V a3 = load_a(A + lda * (i + 3) + l);
+ c30 = madd(a3, k0, c30);
+ c31 = madd(a3, k1, c31);
+ c32 = madd(a3, k2, c32);
+ c33 = madd(a3, k3, c33);
+ c34 = madd(a3, k4, c34);
+ V a4 = load_a(A + lda * (i + 4) + l);
+ c40 = madd(a4, k0, c40);
+ c41 = madd(a4, k1, c41);
+ c42 = madd(a4, k2, c42);
+ c43 = madd(a4, k3, c43);
+ c44 = madd(a4, k4, c44);
+ }
+ C[ldc * (j + 0) + (i + 0)] = hsum(c00);
+ C[ldc * (j + 0) + (i + 1)] = hsum(c10);
+ C[ldc * (j + 0) + (i + 2)] = hsum(c20);
+ C[ldc * (j + 0) + (i + 3)] = hsum(c30);
+ C[ldc * (j + 0) + (i + 4)] = hsum(c40);
+ C[ldc * (j + 1) + (i + 0)] = hsum(c01);
+ C[ldc * (j + 1) + (i + 1)] = hsum(c11);
+ C[ldc * (j + 1) + (i + 2)] = hsum(c21);
+ C[ldc * (j + 1) + (i + 3)] = hsum(c31);
+ C[ldc * (j + 1) + (i + 4)] = hsum(c41);
+ C[ldc * (j + 2) + (i + 0)] = hsum(c02);
+ C[ldc * (j + 2) + (i + 1)] = hsum(c12);
+ C[ldc * (j + 2) + (i + 2)] = hsum(c22);
+ C[ldc * (j + 2) + (i + 3)] = hsum(c32);
+ C[ldc * (j + 2) + (i + 4)] = hsum(c42);
+ C[ldc * (j + 3) + (i + 0)] = hsum(c03);
+ C[ldc * (j + 3) + (i + 1)] = hsum(c13);
+ C[ldc * (j + 3) + (i + 2)] = hsum(c23);
+ C[ldc * (j + 3) + (i + 3)] = hsum(c33);
+ C[ldc * (j + 3) + (i + 4)] = hsum(c43);
+ C[ldc * (j + 4) + (i + 0)] = hsum(c04);
+ C[ldc * (j + 4) + (i + 1)] = hsum(c14);
+ C[ldc * (j + 4) + (i + 2)] = hsum(c24);
+ C[ldc * (j + 4) + (i + 3)] = hsum(c34);
+ C[ldc * (j + 4) + (i + 4)] = hsum(c44);
+ END_KERNEL()
+ }
+
dontinline void gemm3x4(int m0, int m, int n0, int n) {
BEGIN_KERNEL(3, 4)
V c00 = zero();
diff --git a/llamafile/llmmq0.inc b/llamafile/sgemmer0.inc
similarity index 60%
rename from llamafile/llmmq0.inc
rename to llamafile/sgemmer0.inc
index 87fd1eed..310c8962 100644
--- a/llamafile/llmmq0.inc
+++ b/llamafile/sgemmer0.inc
@@ -15,9 +15,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-#include <algorithm>
-#include <cassert>
-
#include "llama.cpp/ggml-impl.h"
#include "llama.cpp/ggml.h"
@@ -25,13 +22,11 @@
#include "kernel.h"
#include "madd.h"
-#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)
-
namespace {
-class LLMMQ0 {
+class SGEMMER0 {
public:
- LLMMQ0(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)
+ SGEMMER0(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)
: A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
}
@@ -44,14 +39,18 @@ class LLMMQ0 {
if (m - m0 <= 0 || n - n0 <= 0)
return;
int mc, nc, mp, np;
- if (m - m0 >= 1 && n - n0 >= 4) {
- mc = 1;
- nc = 4;
- gemm1x4(m0, m, n0, n);
+ if (m - m0 >= 4 && n - n0 >= 3) {
+ mc = 4;
+ nc = 3;
+ gemm4x3(m0, m, n0, n);
} else if (m - m0 >= 4 && n - n0 >= 1) {
mc = 4;
nc = 1;
gemm4x1(m0, m, n0, n);
+ } else if (m - m0 >= 1 && n - n0 >= 4) {
+ mc = 1;
+ nc = 4;
+ gemm1x4(m0, m, n0, n);
} else {
mc = 1;
nc = 1;
@@ -64,6 +63,98 @@ class LLMMQ0 {
mnpack(mp, m, np, n);
}
+ dontinline void gemm4x3(int m0, int m, int n0, int n) {
+ BEGIN_KERNEL(4, 3)
+ int k2 = k / 32;
+ __m256 c00 = _mm256_setzero_ps();
+ __m256 c10 = _mm256_setzero_ps();
+ __m256 c20 = _mm256_setzero_ps();
+ __m256 c30 = _mm256_setzero_ps();
+ __m256 c01 = _mm256_setzero_ps();
+ __m256 c11 = _mm256_setzero_ps();
+ __m256 c21 = _mm256_setzero_ps();
+ __m256 c31 = _mm256_setzero_ps();
+ __m256 c02 = _mm256_setzero_ps();
+ __m256 c12 = _mm256_setzero_ps();
+ __m256 c22 = _mm256_setzero_ps();
+ __m256 c32 = _mm256_setzero_ps();
+ const TA *Ap0 = A + lda * (i + 0);
+ const TA *Ap1 = A + lda * (i + 1);
+ const TA *Ap2 = A + lda * (i + 2);
+ const TA *Ap3 = A + lda * (i + 3);
+ const TB *Bp0 = B + ldb * (j + 0);
+ const TB *Bp1 = B + ldb * (j + 1);
+ const TB *Bp2 = B + ldb * (j + 2);
+ for (int l = 0; l < k2; ++l) {
+ float da0 = unhalf(Ap0[l].d);
+ float da1 = unhalf(Ap1[l].d);
+ float da2 = unhalf(Ap2[l].d);
+ float da3 = unhalf(Ap3[l].d);
+ __m256i e0 = load(Ap0 + l);
+ __m256i e1 = load(Ap1 + l);
+ __m256i e2 = load(Ap2 + l);
+ __m256i e3 = load(Ap3 + l);
+ float db0 = unhalf(Bp0[l].d);
+ __m256 d00 = _mm256_set1_ps(da0 * db0);
+ __m256 d10 = _mm256_set1_ps(da1 * db0);
+ __m256 d20 = _mm256_set1_ps(da2 * db0);
+ __m256 d30 = _mm256_set1_ps(da3 * db0);
+ __m256i f0 = load(Bp0 + l);
+ __m256i u0 = _mm256_sign_epi8(f0, f0);
+ __m256i s00 = _mm256_sign_epi8(e0, f0);
+ __m256i s10 = _mm256_sign_epi8(e1, f0);
+ __m256i s20 = _mm256_sign_epi8(e2, f0);
+ __m256i s30 = _mm256_sign_epi8(e3, f0);
+ c00 = madd(d00, updot(u0, s00), c00);
+ c10 = madd(d10, updot(u0, s10), c10);
+ c20 = madd(d20, updot(u0, s20), c20);
+ c30 = madd(d30, updot(u0, s30), c30);
+ float db1 = unhalf(Bp1[l].d);
+ __m256 d01 = _mm256_set1_ps(da0 * db1);
+ __m256 d11 = _mm256_set1_ps(da1 * db1);
+ __m256 d21 = _mm256_set1_ps(da2 * db1);
+ __m256 d31 = _mm256_set1_ps(da3 * db1);
+ __m256i f1 = load(Bp1 + l);
+ __m256i u1 = _mm256_sign_epi8(f1, f1);
+ __m256i s01 = _mm256_sign_epi8(e0, f1);
+ __m256i s11 = _mm256_sign_epi8(e1, f1);
+ __m256i s21 = _mm256_sign_epi8(e2, f1);
+ __m256i s31 = _mm256_sign_epi8(e3, f1);
+ c01 = madd(d01, updot(u1, s01), c01);
+ c11 = madd(d11, updot(u1, s11), c11);
+ c21 = madd(d21, updot(u1, s21), c21);
+ c31 = madd(d31, updot(u1, s31), c31);
+ float db2 = unhalf(Bp2[l].d);
+ __m256 d02 = _mm256_set1_ps(da0 * db2);
+ __m256 d12 = _mm256_set1_ps(da1 * db2);
+ __m256 d22 = _mm256_set1_ps(da2 * db2);
+ __m256 d32 = _mm256_set1_ps(da3 * db2);
+ __m256i f2 = load(Bp2 + l);
+ __m256i u2 = _mm256_sign_epi8(f2, f2);
+ __m256i s02 = _mm256_sign_epi8(e0, f2);
+ __m256i s12 = _mm256_sign_epi8(e1, f2);
+ __m256i s22 = _mm256_sign_epi8(e2, f2);
+ __m256i s32 = _mm256_sign_epi8(e3, f2);
+ c02 = madd(d02, updot(u2, s02), c02);
+ c12 = madd(d12, updot(u2, s12), c12);
+ c22 = madd(d22, updot(u2, s22), c22);
+ c32 = madd(d32, updot(u2, s32), c32);
+ }
+ C[ldc * (j + 0) + (i + 0)] = hsum(c00);
+ C[ldc * (j + 0) + (i + 1)] = hsum(c10);
+ C[ldc * (j + 0) + (i + 2)] = hsum(c20);
+ C[ldc * (j + 0) + (i + 3)] = hsum(c30);
+ C[ldc * (j + 1) + (i + 0)] = hsum(c01);
+ C[ldc * (j + 1) + (i + 1)] = hsum(c11);
+ C[ldc * (j + 1) + (i + 2)] = hsum(c21);
+ C[ldc * (j + 1) + (i + 3)] = hsum(c31);
+ C[ldc * (j + 2) + (i + 0)] = hsum(c02);
+ C[ldc * (j + 2) + (i + 1)] = hsum(c12);
+ C[ldc * (j + 2) + (i + 2)] = hsum(c22);
+ C[ldc * (j + 2) + (i + 3)] = hsum(c32);
+ END_KERNEL()
+ }
+
dontinline void gemm4x1(int m0, int m, int n0, int n) {
BEGIN_KERNEL(4, 1)
int k2 = k / 32;
diff --git a/llamafile/sgemmer1.inc b/llamafile/sgemmer1.inc
new file mode 100644
index 00000000..92b79fe4
--- /dev/null
+++ b/llamafile/sgemmer1.inc
@@ -0,0 +1,219 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "llama.cpp/ggml-impl.h"
+#include "llama.cpp/ggml.h"
+
+#include "hsum.h"
+#include "kernel.h"
+#include "madd.h"
+
+namespace {
+
+class SGEMMER1 {
+ public:
+ SGEMMER1(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
+ }
+
+ void matmul(int m, int n) {
+ mnpack(0, m, 0, n);
+ }
+
+ private:
+ dontinline void mnpack(int m0, int m, int n0, int n) {
+ if (m - m0 <= 0 || n - n0 <= 0)
+ return;
+ int mc, nc, mp, np;
+ if (VECTOR_REGISTERS >= 32 && m - m0 >= 4 && n - n0 >= 2) {
+ mc = 4;
+ nc = 2;
+ gemm4x2(m0, m, n0, n);
+ } else if (m - m0 >= 4 && n - n0 >= 1) {
+ mc = 4;
+ nc = 1;
+ gemm4x1(m0, m, n0, n);
+ } else {
+ mc = 1;
+ nc = 1;
+ gemm1x1(m0, m, n0, n);
+ }
+ mp = m0 + (m - m0) / mc * mc;
+ np = n0 + (n - n0) / nc * nc;
+ mnpack(mp, m, n0, np);
+ mnpack(m0, mp, np, n);
+ mnpack(mp, m, np, n);
+ }
+
+ dontinline void gemm4x2(int m0, int m, int n0, int n) {
+ BEGIN_KERNEL(4, 2)
+ int k2 = k / 32;
+ __m128 s0 = _mm_setzero_ps();
+ __m128 s1 = _mm_setzero_ps();
+ __m256 c00 = _mm256_setzero_ps();
+ __m256 c01 = _mm256_setzero_ps();
+ __m256 c02 = _mm256_setzero_ps();
+ __m256 c03 = _mm256_setzero_ps();
+ __m256 c10 = _mm256_setzero_ps();
+ __m256 c11 = _mm256_setzero_ps();
+ __m256 c12 = _mm256_setzero_ps();
+ __m256 c13 = _mm256_setzero_ps();
+ const TA *Ap0 = A + lda * (i + 0);
+ const TA *Ap1 = A + lda * (i + 1);
+ const TA *Ap2 = A + lda * (i + 2);
+ const TA *Ap3 = A + lda * (i + 3);
+ const TB *Bp0 = B + ldb * (j + 0);
+ const TB *Bp1 = B + ldb * (j + 1);
+ for (int l = 0; l < k2; ++l) {
+ __m128 sa = _mm_set_ps(unhalf(Ap3[l].dm >> 16), //
+ unhalf(Ap2[l].dm >> 16), //
+ unhalf(Ap1[l].dm >> 16), //
+ unhalf(Ap0[l].dm >> 16));
+ s0 = madd(_mm_set1_ps(unhalf(Bp0[l].ds >> 16)), sa, s0);
+ s1 = madd(_mm_set1_ps(unhalf(Bp1[l].ds >> 16)), sa, s1);
+ __m256i f0 = load(Bp0 + l);
+ __m256 d0 = _mm256_set1_ps(unhalf(Bp0[l].ds));
+ c00 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap0[l].dm)), d0),
+ updot(load(Ap0 + l), f0), c00);
+ c01 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap1[l].dm)), d0),
+ updot(load(Ap1 + l), f0), c01);
+ c02 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap2[l].dm)), d0),
+ updot(load(Ap2 + l), f0), c02);
+ c03 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap3[l].dm)), d0),
+ updot(load(Ap3 + l), f0), c03);
+ __m256i f1 = load(Bp1 + l);
+ __m256 d1 = _mm256_set1_ps(unhalf(Bp1[l].ds));
+ c10 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap0[l].dm)), d1),
+ updot(load(Ap0 + l), f1), c10);
+ c11 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap1[l].dm)), d1),
+ updot(load(Ap1 + l), f1), c11);
+ c12 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap2[l].dm)), d1),
+ updot(load(Ap2 + l), f1), c12);
+ c13 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap3[l].dm)), d1),
+ updot(load(Ap3 + l), f1), c13);
+ }
+ C[ldc * (j + 0) + (i + 0)] = hsum(c00) + s0[0];
+ C[ldc * (j + 0) + (i + 1)] = hsum(c01) + s0[1];
+ C[ldc * (j + 0) + (i + 2)] = hsum(c02) + s0[2];
+ C[ldc * (j + 0) + (i + 3)] = hsum(c03) + s0[3];
+ C[ldc * (j + 1) + (i + 0)] = hsum(c10) + s1[0];
+ C[ldc * (j + 1) + (i + 1)] = hsum(c11) + s1[1];
+ C[ldc * (j + 1) + (i + 2)] = hsum(c12) + s1[2];
+ C[ldc * (j + 1) + (i + 3)] = hsum(c13) + s1[3];
+ END_KERNEL()
+ }
+
+ dontinline void gemm4x1(int m0, int m, int n0, int n) {
+ BEGIN_KERNEL(4, 1)
+ int k2 = k / 32;
+ __m128 s = _mm_setzero_ps();
+ __m256 c0 = _mm256_setzero_ps();
+ __m256 c1 = _mm256_setzero_ps();
+ __m256 c2 = _mm256_setzero_ps();
+ __m256 c3 = _mm256_setzero_ps();
+ const TA *Ap0 = A + lda * (i + 0);
+ const TA *Ap1 = A + lda * (i + 1);
+ const TA *Ap2 = A + lda * (i + 2);
+ const TA *Ap3 = A + lda * (i + 3);
+ const TB *Bp = B + ldb * j;
+ for (int l = 0; l < k2; ++l) {
+ s = madd(_mm_set1_ps(unhalf(Bp[l].ds >> 16)),
+ _mm_set_ps(unhalf(Ap3[l].dm >> 16), //
+ unhalf(Ap2[l].dm >> 16), //
+ unhalf(Ap1[l].dm >> 16), //
+ unhalf(Ap0[l].dm >> 16)),
+ s);
+ __m256i f = load(Bp + l);
+ __m256 d1 = _mm256_set1_ps(unhalf(Bp[l].ds));
+ c0 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap0[l].dm)), d1), updot(load(Ap0 + l), f),
+ c0);
+ c1 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap1[l].dm)), d1), updot(load(Ap1 + l), f),
+ c1);
+ c2 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap2[l].dm)), d1), updot(load(Ap2 + l), f),
+ c2);
+ c3 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap3[l].dm)), d1), updot(load(Ap3 + l), f),
+ c3);
+ }
+ C[ldc * j + (i + 0)] = hsum(c0) + s[0];
+ C[ldc * j + (i + 1)] = hsum(c1) + s[1];
+ C[ldc * j + (i + 2)] = hsum(c2) + s[2];
+ C[ldc * j + (i + 3)] = hsum(c3) + s[3];
+ END_KERNEL()
+ }
+
+ dontinline void gemm1x1(int m0, int m, int n0, int n) {
+ BEGIN_KERNEL(1, 1)
+ int k2 = k / 32;
+ float s = 0;
+ __m256 c = _mm256_setzero_ps();
+ const TA *Ap = A + lda * i;
+ const TB *Bp = B + ldb * j;
+ for (int l = 0; l < k2; ++l) {
+ unsigned dm = Ap[l].dm;
+ unsigned ds = Bp[l].ds;
+ __m256 d0 = _mm256_set1_ps(unhalf(dm));
+ __m256 d1 = _mm256_set1_ps(unhalf(ds));
+ s += unhalf(dm >> 16) * unhalf(ds >> 16);
+ __m256i e = load(Ap + l);
+ __m256i f = load(Bp + l);
+ c = madd(_mm256_mul_ps(d0, d1), updot(e, f), c);
+ }
+ C[ldc * j + i] = hsum(c) + s;
+ END_KERNEL()
+ }
+
+ static inline nosideeffect __m256i load(const block_q8_1 *b) {
+ return _mm256_loadu_si256((const __m256i *)b->qs);
+ }
+
+ static inline nosideeffect __m256i load(const block_q4_1 *b) {
+ return denibble(b->qs);
+ }
+
+ static inline pureconst __m256 updot(__m256i u, __m256i s) {
+ __m256i res;
+#if defined(__AVXVNNI__) || defined(__AVX512VNNI__)
+ res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);
+#else
+ res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));
+#endif
+ return _mm256_cvtepi32_ps(res);
+ }
+
+ static inline nosideeffect __m256i denibble(const uint8_t *p) {
+ const __m128i tmp = _mm_loadu_si128((const __m128i *)p);
+ const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);
+ const __m256i lowMask = _mm256_set1_epi8(15);
+ return _mm256_and_si256(lowMask, bytes);
+ }
+
+ static inline pureconst float unhalf(unsigned short d) {
+ return GGML_FP16_TO_FP32(d);
+ }
+
+ const TA *const A;
+ const TB *const B;
+ TC *const C;
+ const int k;
+ const int lda;
+ const int ldb;
+ const int ldc;
+ const int ith;
+ const int nth;
+};
+
+} // namespace