Skip to content

Distribute and run LLMs with a single file.

Clean up kernel code

Justine Tunneyaba495ea873e
diff --git a/llama.cpp/quantize/quantize.1 b/llama.cpp/quantize/quantize.1
index e92f10da..44c075f1 100644
--- a/llama.cpp/quantize/quantize.1
+++ b/llama.cpp/quantize/quantize.1
@@ -76,7 +76,7 @@ The following quantization types are available:
.It
   7 Q8_0 6.70G +0.0004 ppl @ LLaMA-v1-7B
.It
-  31 BF16 Google Brain Floating Point
+  32 BF16 Google Brain Floating Point
.It
   1 F16 13.00G @ 7B
.It
diff --git a/llamafile/madd.h b/llamafile/madd.h
deleted file mode 100644
index 3994b306..00000000
--- a/llamafile/madd.h
+++ /dev/null
@@ -1,38 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-#pragma once
-#ifdef __ARM_NEON
-#include <arm_neon.h>
-
-inline float32x4_t madd(float32x4_t x, float32x4_t y, float32x4_t z) {
- return vaddq_f32(vmulq_f32(x, y), z);
-}
-
-#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
-inline float16x8_t madd(float16x8_t x, float16x8_t y, float16x8_t z) {
- return vaddq_f16(vmulq_f16(x, y), z);
-}
-#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
-
-#elif defined(__SSE__)
-#include <immintrin.h>
-
-inline __m128 madd(__m128 x, __m128 y, __m128 z) {
- return _mm_add_ps(_mm_mul_ps(x, y), z);
-}
-
-#ifdef __AVX__
-
-inline __m256 madd(__m256 x, __m256 y, __m256 z) {
- return _mm256_add_ps(_mm256_mul_ps(x, y), z);
-}
-
-#ifdef __AVX512F__
-
-inline __m512 madd(__m512 x, __m512 y, __m512 z) {
- return _mm512_add_ps(_mm512_mul_ps(x, y), z);
-}
-
-#endif // __AVX512F__
-#endif // __AVX__
-#endif // __SSE__
diff --git a/llamafile/sgemm_bss_avx2.cpp b/llamafile/sgemm_bss_avx2.cpp
index 9d22305d..5630e34c 100644
--- a/llamafile/sgemm_bss_avx2.cpp
+++ b/llamafile/sgemm_bss_avx2.cpp
@@ -22,15 +22,11 @@
#define KN 8
-typedef __m256 V;
-typedef __m256 D;
-typedef ggml_bf16_t TA;
-typedef float TB;
-typedef float TC;
-
-static inline __m256 zero() {
- return _mm256_setzero_ps();
-}
+#define V __m256
+#define D __m256
+#define TA ggml_bf16_t
+#define TB float
+#define TC float
static inline __m256 load(const float *p) {
return _mm256_loadu_ps(p);
diff --git a/llamafile/sgemm_bss_avx512.cpp b/llamafile/sgemm_bss_avx512.cpp
index 9fcebf40..96c563a9 100644
--- a/llamafile/sgemm_bss_avx512.cpp
+++ b/llamafile/sgemm_bss_avx512.cpp
@@ -22,15 +22,11 @@
#define KN 16
-typedef __m512 V;
-typedef __m512 D;
-typedef ggml_bf16_t TA;
-typedef float TB;
-typedef float TC;
-
-static inline __m512 zero() {
- return _mm512_setzero_ps();
-}
+#define V __m512
+#define D __m512
+#define TA ggml_bf16_t
+#define TB float
+#define TC float
static inline __m512 load(const float *p) {
return _mm512_loadu_ps(p);
diff --git a/llamafile/sgemm_bss_avx512bf16.cpp b/llamafile/sgemm_bss_avx512bf16.cpp
index 4afe9704..a130e3a8 100644
--- a/llamafile/sgemm_bss_avx512bf16.cpp
+++ b/llamafile/sgemm_bss_avx512bf16.cpp
@@ -18,19 +18,16 @@
#ifdef __x86_64__
#include "sgemm.h"
+#include "varith.h"
#include <immintrin.h>
#define KN 32
-typedef __m512 D;
-typedef __m512bh V;
-typedef ggml_bf16_t TA;
-typedef float TB;
-typedef float TC;
-
-static inline __m512 zero() {
- return _mm512_setzero_ps();
-}
+#define D __m512
+#define V __m512bh
+#define TA ggml_bf16_t
+#define TB float
+#define TC float
static inline __m512bh load(const ggml_bf16_t *p) {
return (__m512bh)_mm512_loadu_ps((const float *)p);
@@ -40,7 +37,11 @@ static inline __m512bh load(const float *p) {
return _mm512_cvtne2ps_pbh(_mm512_loadu_ps(p + 16), _mm512_loadu_ps(p));
}
-static inline __m512 madd(__m512bh x, __m512bh y, __m512 z) {
+template <> inline __m512 madd(__m512bh x, __m512bh y, __m512 z) {
+ return _mm512_dpbf16_ps(z, x, y);
+}
+
+template <> inline __m512 madder(__m512bh x, __m512bh y, __m512 z, __m512 *e) {
return _mm512_dpbf16_ps(z, x, y);
}
diff --git a/llamafile/sgemm_e0q0s_avx512vnni.cpp b/llamafile/sgemm_e0q0s_avx512vnni.cpp
index 1e2a1fd9..5107717a 100644
--- a/llamafile/sgemm_e0q0s_avx512vnni.cpp
+++ b/llamafile/sgemm_e0q0s_avx512vnni.cpp
@@ -19,9 +19,9 @@
#include "sgemm.h"
-typedef block_q4_0 TA;
-typedef block_q8_0 TB;
-typedef float TC;
+#define TA block_q4_0
+#define TB block_q8_0
+#define TC float
#include "sgemmer0.inc"
diff --git a/llamafile/sgemm_e0q0s_avxvnni.cpp b/llamafile/sgemm_e0q0s_avxvnni.cpp
index 6682938c..b5713bb7 100644
--- a/llamafile/sgemm_e0q0s_avxvnni.cpp
+++ b/llamafile/sgemm_e0q0s_avxvnni.cpp
@@ -19,9 +19,9 @@
#include "sgemm.h"
-typedef block_q4_0 TA;
-typedef block_q8_0 TB;
-typedef float TC;
+#define TA block_q4_0
+#define TB block_q8_0
+#define TC float
#include "sgemmer0.inc"
diff --git a/llamafile/sgemm_e0q0s_fma.cpp b/llamafile/sgemm_e0q0s_fma.cpp
index fad68bf0..8b0ccee3 100644
--- a/llamafile/sgemm_e0q0s_fma.cpp
+++ b/llamafile/sgemm_e0q0s_fma.cpp
@@ -19,9 +19,9 @@
#include "sgemm.h"
-typedef block_q4_0 TA;
-typedef block_q8_0 TB;
-typedef float TC;
+#define TA block_q4_0
+#define TB block_q8_0
+#define TC float
#include "sgemmer0.inc"
diff --git a/llamafile/sgemm_e1q1s_avx512vnni.cpp b/llamafile/sgemm_e1q1s_avx512vnni.cpp
index 83f50c81..1aa2264c 100644
--- a/llamafile/sgemm_e1q1s_avx512vnni.cpp
+++ b/llamafile/sgemm_e1q1s_avx512vnni.cpp
@@ -19,9 +19,9 @@
#include "sgemm.h"
-typedef block_q4_1 TA;
-typedef block_q8_1 TB;
-typedef float TC;
+#define TA block_q4_1
+#define TB block_q8_1
+#define TC float
#include "sgemmer1.inc"
diff --git a/llamafile/sgemm_e1q1s_avxvnni.cpp b/llamafile/sgemm_e1q1s_avxvnni.cpp
index fe41b49b..49ced14d 100644
--- a/llamafile/sgemm_e1q1s_avxvnni.cpp
+++ b/llamafile/sgemm_e1q1s_avxvnni.cpp
@@ -19,9 +19,9 @@
#include "sgemm.h"
-typedef block_q4_1 TA;
-typedef block_q8_1 TB;
-typedef float TC;
+#define TA block_q4_1
+#define TB block_q8_1
+#define TC float
#include "sgemmer1.inc"
diff --git a/llamafile/sgemm_e1q1s_fma.cpp b/llamafile/sgemm_e1q1s_fma.cpp
index 585e7587..178e23ac 100644
--- a/llamafile/sgemm_e1q1s_fma.cpp
+++ b/llamafile/sgemm_e1q1s_fma.cpp
@@ -19,9 +19,9 @@
#include "sgemm.h"
-typedef block_q4_1 TA;
-typedef block_q8_1 TB;
-typedef float TC;
+#define TA block_q4_1
+#define TB block_q8_1
+#define TC float
#include "sgemmer1.inc"
diff --git a/llamafile/sgemm_hhs_neon.cpp b/llamafile/sgemm_hhs_neon.cpp
index f93e1e89..5507a045 100644
--- a/llamafile/sgemm_hhs_neon.cpp
+++ b/llamafile/sgemm_hhs_neon.cpp
@@ -22,15 +22,11 @@
#define KN 8
-typedef float16x8_t V;
-typedef float16x8_t D;
-typedef unsigned short TA;
-typedef unsigned short TB;
-typedef float TC;
-
-static inline V zero() {
- return vdupq_n_f16(0.f);
-}
+#define V float16x8_t
+#define D float16x8_t
+#define TA unsigned short
+#define TB unsigned short
+#define TC float
static inline V load(const unsigned short *p) {
return vld1q_f16((const __fp16 *)p);
diff --git a/llamafile/sgemm_hss_avx512f.cpp b/llamafile/sgemm_hss_avx512f.cpp
index 7abd5f52..f0a8bf33 100644
--- a/llamafile/sgemm_hss_avx512f.cpp
+++ b/llamafile/sgemm_hss_avx512f.cpp
@@ -22,15 +22,11 @@
#define KN 16
-typedef __m512 V;
-typedef __m512 D;
-typedef unsigned short TA;
-typedef float TB;
-typedef float TC;
-
-static inline V zero() {
- return _mm512_setzero_ps();
-}
+#define V __m512
+#define D __m512
+#define TA unsigned short
+#define TB float
+#define TC float
static inline V load(const float *p) {
return _mm512_loadu_ps(p);
diff --git a/llamafile/sgemm_hss_f16c.cpp b/llamafile/sgemm_hss_f16c.cpp
index 14859edc..e9ba461b 100644
--- a/llamafile/sgemm_hss_f16c.cpp
+++ b/llamafile/sgemm_hss_f16c.cpp
@@ -22,15 +22,11 @@
#define KN 8
-typedef __m256 V;
-typedef __m256 D;
-typedef unsigned short TA;
-typedef float TB;
-typedef float TC;
-
-static inline V zero() {
- return _mm256_setzero_ps();
-}
+#define V __m256
+#define D __m256
+#define TA unsigned short
+#define TB float
+#define TC float
static inline V load(const float *p) {
return _mm256_loadu_ps(p);
diff --git a/llamafile/sgemm_hss_neon.cpp b/llamafile/sgemm_hss_neon.cpp
index 9ae6e68e..e10e37e9 100644
--- a/llamafile/sgemm_hss_neon.cpp
+++ b/llamafile/sgemm_hss_neon.cpp
@@ -22,15 +22,11 @@
#define KN 4
-typedef float32x4_t V;
-typedef float32x4_t D;
-typedef unsigned short TA;
-typedef float TB;
-typedef float TC;
-
-static inline V zero() {
- return vdupq_n_f32(0.f);
-}
+#define V float32x4_t
+#define D float32x4_t
+#define TA unsigned short
+#define TB float
+#define TC float
static inline V load(const float *p) {
return vld1q_f32(p);
diff --git a/llamafile/sgemm_q0q0s_avx512vnni.cpp b/llamafile/sgemm_q0q0s_avx512vnni.cpp
index aac59018..e5b3ed24 100644
--- a/llamafile/sgemm_q0q0s_avx512vnni.cpp
+++ b/llamafile/sgemm_q0q0s_avx512vnni.cpp
@@ -19,9 +19,9 @@
#include "sgemm.h"
-typedef block_q8_0 TA;
-typedef block_q8_0 TB;
-typedef float TC;
+#define TA block_q8_0
+#define TB block_q8_0
+#define TC float
#include "sgemmer0.inc"
diff --git a/llamafile/sgemm_q0q0s_avxvnni.cpp b/llamafile/sgemm_q0q0s_avxvnni.cpp
index 34c5307a..d53b2979 100644
--- a/llamafile/sgemm_q0q0s_avxvnni.cpp
+++ b/llamafile/sgemm_q0q0s_avxvnni.cpp
@@ -19,9 +19,9 @@
#include "sgemm.h"
-typedef block_q8_0 TA;
-typedef block_q8_0 TB;
-typedef float TC;
+#define TA block_q8_0
+#define TB block_q8_0
+#define TC float
#include "sgemmer0.inc"
diff --git a/llamafile/sgemm_q0q0s_fma.cpp b/llamafile/sgemm_q0q0s_fma.cpp
index ff6eeda6..437ec389 100644
--- a/llamafile/sgemm_q0q0s_fma.cpp
+++ b/llamafile/sgemm_q0q0s_fma.cpp
@@ -19,9 +19,9 @@
#include "sgemm.h"
-typedef block_q8_0 TA;
-typedef block_q8_0 TB;
-typedef float TC;
+#define TA block_q8_0
+#define TB block_q8_0
+#define TC float
#include "sgemmer0.inc"
diff --git a/llamafile/sgemm_sss_avx.cpp b/llamafile/sgemm_sss_avx.cpp
index a7ea9055..a49427c8 100644
--- a/llamafile/sgemm_sss_avx.cpp
+++ b/llamafile/sgemm_sss_avx.cpp
@@ -22,15 +22,11 @@
#define KN 8
-typedef __m256 V;
-typedef __m256 D;
-typedef float TA;
-typedef float TB;
-typedef float TC;
-
-static inline V zero() {
- return _mm256_setzero_ps();
-}
+#define V __m256
+#define D __m256
+#define TA float
+#define TB float
+#define TC float
static inline V load(const float *p) {
return _mm256_loadu_ps(p);
diff --git a/llamafile/sgemm_sss_avx512f.cpp b/llamafile/sgemm_sss_avx512f.cpp
index 09eb708f..a9dbc1a0 100644
--- a/llamafile/sgemm_sss_avx512f.cpp
+++ b/llamafile/sgemm_sss_avx512f.cpp
@@ -22,15 +22,11 @@
#define KN 16
-typedef __m512 V;
-typedef __m512 D;
-typedef float TA;
-typedef float TB;
-typedef float TC;
-
-static inline V zero() {
- return _mm512_setzero_ps();
-}
+#define V __m512
+#define D __m512
+#define TA float
+#define TB float
+#define TC float
static inline V load(const float *p) {
return _mm512_loadu_ps(p);
diff --git a/llamafile/sgemm_sss_fma.cpp b/llamafile/sgemm_sss_fma.cpp
index b32e6567..f12ad7d6 100644
--- a/llamafile/sgemm_sss_fma.cpp
+++ b/llamafile/sgemm_sss_fma.cpp
@@ -22,15 +22,11 @@
#define KN 8
-typedef __m256 V;
-typedef __m256 D;
-typedef float TA;
-typedef float TB;
-typedef float TC;
-
-static inline V zero() {
- return _mm256_setzero_ps();
-}
+#define V __m256
+#define D __m256
+#define TA float
+#define TB float
+#define TC float
static inline V load(const float *p) {
return _mm256_loadu_ps(p);
diff --git a/llamafile/sgemm_sss_neon.cpp b/llamafile/sgemm_sss_neon.cpp
index f744654f..88644221 100644
--- a/llamafile/sgemm_sss_neon.cpp
+++ b/llamafile/sgemm_sss_neon.cpp
@@ -22,15 +22,11 @@
#define KN 4
-typedef float32x4_t V;
-typedef float32x4_t D;
-typedef float TA;
-typedef float TB;
-typedef float TC;
-
-static inline V zero() {
- return vdupq_n_f32(0.f);
-}
+#define V float32x4_t
+#define D float32x4_t
+#define TA float
+#define TB float
+#define TC float
static inline V load(const float *p) {
return vld1q_f32(p);
diff --git a/llamafile/sgemm_vecdot_test.cpp b/llamafile/sgemm_vecdot_test.cpp
index 908a8b11..79507d41 100644
--- a/llamafile/sgemm_vecdot_test.cpp
+++ b/llamafile/sgemm_vecdot_test.cpp
@@ -85,6 +85,10 @@ int main(int argc, char *argv[]) {
// 22.291 average ulp
// 1566 worst ulp
+ // using kahan summation
+ // 2.14244 average ulp
+ // 134 worst ulp
+
if (err_avg > 30)
return 5;
if (err_worst > 3000)
diff --git a/llamafile/sgemmer.inc b/llamafile/sgemmer.inc
index c44d3852..7bc8cf3b 100644
--- a/llamafile/sgemmer.inc
+++ b/llamafile/sgemmer.inc
@@ -19,7 +19,7 @@
#include "hsum.h"
#include "kernel.h"
-#include "madd.h"
+#include "varith.h"
namespace {
@@ -68,31 +68,31 @@ class SGEMMER {
dontinline void gemm5x5(int m0, int m, int n0, int n) {
BEGIN_KERNEL(5, 5)
- D c00 = zero();
- D c01 = zero();
- D c02 = zero();
- D c03 = zero();
- D c04 = zero();
- D c10 = zero();
- D c11 = zero();
- D c12 = zero();
- D c13 = zero();
- D c14 = zero();
- D c20 = zero();
- D c21 = zero();
- D c22 = zero();
- D c23 = zero();
- D c24 = zero();
- D c30 = zero();
- D c31 = zero();
- D c32 = zero();
- D c33 = zero();
- D c34 = zero();
- D c40 = zero();
- D c41 = zero();
- D c42 = zero();
- D c43 = zero();
- D c44 = zero();
+ D c00 = {0};
+ D c01 = {0};
+ D c02 = {0};
+ D c03 = {0};
+ D c04 = {0};
+ D c10 = {0};
+ D c11 = {0};
+ D c12 = {0};
+ D c13 = {0};
+ D c14 = {0};
+ D c20 = {0};
+ D c21 = {0};
+ D c22 = {0};
+ D c23 = {0};
+ D c24 = {0};
+ D c30 = {0};
+ D c31 = {0};
+ D c32 = {0};
+ D c33 = {0};
+ D c34 = {0};
+ D c40 = {0};
+ D c41 = {0};
+ D c42 = {0};
+ D c43 = {0};
+ D c44 = {0};
for (int l = 0; l < k; l += KN) {
V k0 = load(B + ldb * (j + 0) + l);
V k1 = load(B + ldb * (j + 1) + l);
@@ -160,18 +160,18 @@ class SGEMMER {
dontinline void gemm3x4(int m0, int m, int n0, int n) {
BEGIN_KERNEL(3, 4)
- D c00 = zero();
- D c01 = zero();
- D c02 = zero();
- D c03 = zero();
- D c10 = zero();
- D c11 = zero();
- D c12 = zero();
- D c13 = zero();
- D c20 = zero();
- D c21 = zero();
- D c22 = zero();
- D c23 = zero();
+ D c00 = {0};
+ D c01 = {0};
+ D c02 = {0};
+ D c03 = {0};
+ D c10 = {0};
+ D c11 = {0};
+ D c12 = {0};
+ D c13 = {0};
+ D c20 = {0};
+ D c21 = {0};
+ D c22 = {0};
+ D c23 = {0};
for (int l = 0; l < k; l += KN) {
V k0 = load(B + ldb * (j + 0) + l);
V k1 = load(B + ldb * (j + 1) + l);
@@ -210,20 +210,16 @@ class SGEMMER {
dontinline void gemm1x4(int m0, int m, int n0, int n) {
BEGIN_KERNEL(1, 4)
- D c00 = zero();
- D c01 = zero();
- D c02 = zero();
- D c03 = zero();
+ D c00 = {0}, e00 = {0};
+ D c01 = {0}, e01 = {0};
+ D c02 = {0}, e02 = {0};
+ D c03 = {0}, e03 = {0};
for (int l = 0; l < k; l += KN) {
- V a0 = load(A + lda * (i + 0) + l);
- V k0 = load(B + ldb * (j + 0) + l);
- V k1 = load(B + ldb * (j + 1) + l);
- V k2 = load(B + ldb * (j + 2) + l);
- V k3 = load(B + ldb * (j + 3) + l);
- c00 = madd(a0, k0, c00);
- c01 = madd(a0, k1, c01);
- c02 = madd(a0, k2, c02);
- c03 = madd(a0, k3, c03);
+ V a = load(A + lda * (i + 0) + l);
+ c00 = madder(a, load(B + ldb * (j + 0) + l), c00, &e00);
+ c01 = madder(a, load(B + ldb * (j + 1) + l), c01, &e01);
+ c02 = madder(a, load(B + ldb * (j + 2) + l), c02, &e02);
+ c03 = madder(a, load(B + ldb * (j + 3) + l), c03, &e03);
}
C[ldc * (j + 0) + (i + 0)] = hsum(c00);
C[ldc * (j + 1) + (i + 0)] = hsum(c01);
@@ -234,79 +230,30 @@ class SGEMMER {
dontinline void gemm4x1(int m0, int m, int n0, int n) {
BEGIN_KERNEL(4, 1)
- D c00 = zero();
- D c01 = zero();
- D c02 = zero();
- D c10 = zero();
- D c11 = zero();
- D c12 = zero();
- D c20 = zero();
- D c21 = zero();
- D c22 = zero();
- D c30 = zero();
- D c31 = zero();
- D c32 = zero();
- int l = 0;
- while (l + KN * 3 <= k) {
- {
- V k0 = load(B + ldb * (j + 0) + l);
- c00 = madd(load(A + lda * (i + 0) + l), k0, c00);
- c10 = madd(load(A + lda * (i + 1) + l), k0, c10);
- c20 = madd(load(A + lda * (i + 2) + l), k0, c20);
- c30 = madd(load(A + lda * (i + 3) + l), k0, c30);
- }
- l += KN;
- {
- V k0 = load(B + ldb * (j + 0) + l);
- c01 = madd(load(A + lda * (i + 0) + l), k0, c01);
- c11 = madd(load(A + lda * (i + 1) + l), k0, c11);
- c21 = madd(load(A + lda * (i + 2) + l), k0, c21);
- c31 = madd(load(A + lda * (i + 3) + l), k0, c31);
- }
- l += KN;
- {
- V k0 = load(B + ldb * (j + 0) + l);
- c02 = madd(load(A + lda * (i + 0) + l), k0, c02);
- c12 = madd(load(A + lda * (i + 1) + l), k0, c12);
- c22 = madd(load(A + lda * (i + 2) + l), k0, c22);
- c32 = madd(load(A + lda * (i + 3) + l), k0, c32);
- }
- l += KN;
- }
- for (; l < k; l += KN) {
- V k0 = load(B + ldb * (j + 0) + l);
- c00 = madd(load(A + lda * (i + 0) + l), k0, c00);
- c10 = madd(load(A + lda * (i + 1) + l), k0, c10);
- c20 = madd(load(A + lda * (i + 2) + l), k0, c20);
- c30 = madd(load(A + lda * (i + 3) + l), k0, c30);
+ D c00 = {0}, e00 = {0};
+ D c10 = {0}, e10 = {0};
+ D c20 = {0}, e20 = {0};
+ D c30 = {0}, e30 = {0};
+ for (int l = 0; l < k; l += KN) {
+ V b = load(B + ldb * (j + 0) + l);
+ c00 = madder(load(A + lda * (i + 0) + l), b, c00, &e00);
+ c10 = madder(load(A + lda * (i + 1) + l), b, c10, &e10);
+ c20 = madder(load(A + lda * (i + 2) + l), b, c20, &e20);
+ c30 = madder(load(A + lda * (i + 3) + l), b, c30, &e30);
}
- C[ldc * (j + 0) + (i + 0)] = hsum(c00) + hsum(c01) + hsum(c02);
- C[ldc * (j + 0) + (i + 1)] = hsum(c10) + hsum(c11) + hsum(c12);
- C[ldc * (j + 0) + (i + 2)] = hsum(c20) + hsum(c21) + hsum(c22);
- C[ldc * (j + 0) + (i + 3)] = hsum(c30) + hsum(c31) + hsum(c32);
+ C[ldc * (j + 0) + (i + 0)] = hsum(c00);
+ C[ldc * (j + 0) + (i + 1)] = hsum(c10);
+ C[ldc * (j + 0) + (i + 2)] = hsum(c20);
+ C[ldc * (j + 0) + (i + 3)] = hsum(c30);
END_KERNEL()
}
dontinline void gemm1x1(int m0, int m, int n0, int n) {
BEGIN_KERNEL(1, 1)
- D c0 = zero();
- D c1 = zero();
- D c2 = zero();
- D c3 = zero();
- int l = 0;
- while (l + KN * 4 <= k) {
- c0 = madd(load(A + lda * i + l), load(B + ldb * j + l), c0);
- l += KN;
- c1 = madd(load(A + lda * i + l), load(B + ldb * j + l), c1);
- l += KN;
- c2 = madd(load(A + lda * i + l), load(B + ldb * j + l), c2);
- l += KN;
- c3 = madd(load(A + lda * i + l), load(B + ldb * j + l), c3);
- l += KN;
- }
- for (; l < k; l += KN)
- c0 = madd(load(A + lda * i + l), load(B + ldb * j + l), c0);
- C[ldc * j + i] = hsum(c0) + hsum(c1) + hsum(c2) + hsum(c3);
+ D c = {0}, e = {0};
+ for (int l = 0; l < k; l += KN)
+ c = madder(load(A + lda * i + l), load(B + ldb * j + l), c, &e);
+ C[ldc * j + i] = hsum(c);
END_KERNEL()
}
diff --git a/llamafile/sgemmer0.inc b/llamafile/sgemmer0.inc
index 0ff72d7b..68fecf7a 100644
--- a/llamafile/sgemmer0.inc
+++ b/llamafile/sgemmer0.inc
@@ -20,7 +20,7 @@
#include "hsum.h"
#include "kernel.h"
-#include "madd.h"
+#include "varith.h"
namespace {
diff --git a/llamafile/sgemmer1.inc b/llamafile/sgemmer1.inc
index 9c487dbc..7eb37d40 100644
--- a/llamafile/sgemmer1.inc
+++ b/llamafile/sgemmer1.inc
@@ -20,7 +20,7 @@
#include "hsum.h"
#include "kernel.h"
-#include "madd.h"
+#include "varith.h"
namespace {
diff --git a/llamafile/varith.h b/llamafile/varith.h
new file mode 100644
index 00000000..1d5a1abf
--- /dev/null
+++ b/llamafile/varith.h
@@ -0,0 +1,64 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+// clang-format off
+#pragma once
+
+#include <arm_neon.h>
+#include <immintrin.h>
+
+#ifdef __SSE__
+inline __m128 add(__m128 x, __m128 y) { return _mm_add_ps(x, y); }
+inline __m128 sub(__m128 x, __m128 y) { return _mm_sub_ps(x, y); }
+inline __m128 mul(__m128 x, __m128 y) { return _mm_mul_ps(x, y); }
+#endif // __SSE__
+
+#ifdef __AVX__
+inline __m256 add(__m256 x, __m256 y) { return _mm256_add_ps(x, y); }
+inline __m256 sub(__m256 x, __m256 y) { return _mm256_sub_ps(x, y); }
+inline __m256 mul(__m256 x, __m256 y) { return _mm256_mul_ps(x, y); }
+#endif // __AVX__
+
+#ifdef __AVX512F__
+inline __m512 add(__m512 x, __m512 y) { return _mm512_add_ps(x, y); }
+inline __m512 sub(__m512 x, __m512 y) { return _mm512_sub_ps(x, y); }
+inline __m512 mul(__m512 x, __m512 y) { return _mm512_mul_ps(x, y); }
+#endif // __AVX512F__
+
+#ifdef __ARM_NEON
+inline float32x4_t add(float32x4_t x, float32x4_t y) { return vaddq_f32(x, y); }
+inline float32x4_t sub(float32x4_t x, float32x4_t y) { return vsubq_f32(x, y); }
+inline float32x4_t mul(float32x4_t x, float32x4_t y) { return vmulq_f32(x, y); }
+#endif // __ARM_NEON
+
+#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+inline float16x8_t add(float16x8_t x, float16x8_t y) { return vaddq_f16(x, y); }
+inline float16x8_t sub(float16x8_t x, float16x8_t y) { return vsubq_f16(x, y); }
+inline float16x8_t mul(float16x8_t x, float16x8_t y) { return vmulq_f16(x, y); }
+#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+
+/**
+ * Computes a * b + c.
+ *
+ * This operation will become fused into a single arithmetic instruction
+ * if the hardware has support for this feature, e.g. Intel Haswell+ (c.
+ * 2013), AMD Bulldozer+ (c. 2011), etc.
+ */
+template <typename T, typename U>
+inline U madd(T a, T b, U c) {
+ return add(mul(a, b), c);
+}
+
+/**
+ * Computes a * b + c with error correction.
+ *
+ * @see W. Kahan, "Further remarks on reducing truncation errors,"
+ * Communications of the ACM, vol. 8, no. 1, p. 40, Jan. 1965,
+ * doi: 10.1145/363707.363723.
+ */
+template <typename T, typename U>
+inline U madder(T a, T b, U c, U *e) {
+ U y = sub(mul(a, b), *e);
+ U t = add(c, y);
+ *e = sub(sub(t, c), y);
+ return t;
+}