Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Clean up kernel code
Justine Tunneyaba495ea873e
diff --git a/llama.cpp/quantize/quantize.1 b/llama.cpp/quantize/quantize.1index e92f10da..44c075f1 100644--- a/llama.cpp/quantize/quantize.1+++ b/llama.cpp/quantize/quantize.1@@ -76,7 +76,7 @@ The following quantization types are available:.It7 Q8_0 6.70G +0.0004 ppl @ LLaMA-v1-7B.It- 31 BF16 Google Brain Floating Point+ 32 BF16 Google Brain Floating Point.It1 F16 13.00G @ 7B.Itdiff --git a/llamafile/madd.h b/llamafile/madd.hdeleted file mode 100644index 3994b306..00000000--- a/llamafile/madd.h+++ /dev/null@@ -1,38 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-#pragma once-#ifdef __ARM_NEON-#include <arm_neon.h>--inline float32x4_t madd(float32x4_t x, float32x4_t y, float32x4_t z) {- return vaddq_f32(vmulq_f32(x, y), z);-}--#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC-inline float16x8_t madd(float16x8_t x, float16x8_t y, float16x8_t z) {- return vaddq_f16(vmulq_f16(x, y), z);-}-#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC--#elif defined(__SSE__)-#include <immintrin.h>--inline __m128 madd(__m128 x, __m128 y, __m128 z) {- return _mm_add_ps(_mm_mul_ps(x, y), z);-}--#ifdef __AVX__--inline __m256 madd(__m256 x, __m256 y, __m256 z) {- return _mm256_add_ps(_mm256_mul_ps(x, y), z);-}--#ifdef __AVX512F__--inline __m512 madd(__m512 x, __m512 y, __m512 z) {- return _mm512_add_ps(_mm512_mul_ps(x, y), z);-}--#endif // __AVX512F__-#endif // __AVX__-#endif // __SSE__diff --git a/llamafile/sgemm_bss_avx2.cpp b/llamafile/sgemm_bss_avx2.cppindex 9d22305d..5630e34c 100644--- a/llamafile/sgemm_bss_avx2.cpp+++ b/llamafile/sgemm_bss_avx2.cpp@@ -22,15 +22,11 @@#define KN 8-typedef __m256 V;-typedef __m256 D;-typedef ggml_bf16_t TA;-typedef float TB;-typedef float TC;--static inline __m256 zero() {- return _mm256_setzero_ps();-}+#define V __m256+#define D __m256+#define TA ggml_bf16_t+#define TB float+#define TC floatstatic inline __m256 load(const float *p) {return _mm256_loadu_ps(p);diff --git a/llamafile/sgemm_bss_avx512.cpp b/llamafile/sgemm_bss_avx512.cppindex 9fcebf40..96c563a9 100644--- a/llamafile/sgemm_bss_avx512.cpp+++ b/llamafile/sgemm_bss_avx512.cpp@@ -22,15 +22,11 @@#define KN 16-typedef __m512 V;-typedef __m512 D;-typedef ggml_bf16_t TA;-typedef float TB;-typedef float TC;--static inline __m512 zero() {- return _mm512_setzero_ps();-}+#define V __m512+#define D __m512+#define TA ggml_bf16_t+#define TB float+#define TC floatstatic inline __m512 load(const float *p) {return _mm512_loadu_ps(p);diff --git a/llamafile/sgemm_bss_avx512bf16.cpp b/llamafile/sgemm_bss_avx512bf16.cppindex 4afe9704..a130e3a8 100644--- a/llamafile/sgemm_bss_avx512bf16.cpp+++ b/llamafile/sgemm_bss_avx512bf16.cpp@@ -18,19 +18,16 @@#ifdef __x86_64__#include "sgemm.h"+#include "varith.h"#include <immintrin.h>#define KN 32-typedef __m512 D;-typedef __m512bh V;-typedef ggml_bf16_t TA;-typedef float TB;-typedef float TC;--static inline __m512 zero() {- return _mm512_setzero_ps();-}+#define D __m512+#define V __m512bh+#define TA ggml_bf16_t+#define TB float+#define TC floatstatic inline __m512bh load(const ggml_bf16_t *p) {return (__m512bh)_mm512_loadu_ps((const float *)p);@@ -40,7 +37,11 @@ static inline __m512bh load(const float *p) {return _mm512_cvtne2ps_pbh(_mm512_loadu_ps(p + 16), _mm512_loadu_ps(p));}-static inline __m512 madd(__m512bh x, __m512bh y, __m512 z) {+template <> inline __m512 madd(__m512bh x, __m512bh y, __m512 z) {+ return _mm512_dpbf16_ps(z, x, y);+}++template <> inline __m512 madder(__m512bh x, __m512bh y, __m512 z, __m512 *e) {return _mm512_dpbf16_ps(z, x, y);}diff --git a/llamafile/sgemm_e0q0s_avx512vnni.cpp b/llamafile/sgemm_e0q0s_avx512vnni.cppindex 1e2a1fd9..5107717a 100644--- a/llamafile/sgemm_e0q0s_avx512vnni.cpp+++ b/llamafile/sgemm_e0q0s_avx512vnni.cpp@@ -19,9 +19,9 @@#include "sgemm.h"-typedef block_q4_0 TA;-typedef block_q8_0 TB;-typedef float TC;+#define TA block_q4_0+#define TB block_q8_0+#define TC float#include "sgemmer0.inc"diff --git a/llamafile/sgemm_e0q0s_avxvnni.cpp b/llamafile/sgemm_e0q0s_avxvnni.cppindex 6682938c..b5713bb7 100644--- a/llamafile/sgemm_e0q0s_avxvnni.cpp+++ b/llamafile/sgemm_e0q0s_avxvnni.cpp@@ -19,9 +19,9 @@#include "sgemm.h"-typedef block_q4_0 TA;-typedef block_q8_0 TB;-typedef float TC;+#define TA block_q4_0+#define TB block_q8_0+#define TC float#include "sgemmer0.inc"diff --git a/llamafile/sgemm_e0q0s_fma.cpp b/llamafile/sgemm_e0q0s_fma.cppindex fad68bf0..8b0ccee3 100644--- a/llamafile/sgemm_e0q0s_fma.cpp+++ b/llamafile/sgemm_e0q0s_fma.cpp@@ -19,9 +19,9 @@#include "sgemm.h"-typedef block_q4_0 TA;-typedef block_q8_0 TB;-typedef float TC;+#define TA block_q4_0+#define TB block_q8_0+#define TC float#include "sgemmer0.inc"diff --git a/llamafile/sgemm_e1q1s_avx512vnni.cpp b/llamafile/sgemm_e1q1s_avx512vnni.cppindex 83f50c81..1aa2264c 100644--- a/llamafile/sgemm_e1q1s_avx512vnni.cpp+++ b/llamafile/sgemm_e1q1s_avx512vnni.cpp@@ -19,9 +19,9 @@#include "sgemm.h"-typedef block_q4_1 TA;-typedef block_q8_1 TB;-typedef float TC;+#define TA block_q4_1+#define TB block_q8_1+#define TC float#include "sgemmer1.inc"diff --git a/llamafile/sgemm_e1q1s_avxvnni.cpp b/llamafile/sgemm_e1q1s_avxvnni.cppindex fe41b49b..49ced14d 100644--- a/llamafile/sgemm_e1q1s_avxvnni.cpp+++ b/llamafile/sgemm_e1q1s_avxvnni.cpp@@ -19,9 +19,9 @@#include "sgemm.h"-typedef block_q4_1 TA;-typedef block_q8_1 TB;-typedef float TC;+#define TA block_q4_1+#define TB block_q8_1+#define TC float#include "sgemmer1.inc"diff --git a/llamafile/sgemm_e1q1s_fma.cpp b/llamafile/sgemm_e1q1s_fma.cppindex 585e7587..178e23ac 100644--- a/llamafile/sgemm_e1q1s_fma.cpp+++ b/llamafile/sgemm_e1q1s_fma.cpp@@ -19,9 +19,9 @@#include "sgemm.h"-typedef block_q4_1 TA;-typedef block_q8_1 TB;-typedef float TC;+#define TA block_q4_1+#define TB block_q8_1+#define TC float#include "sgemmer1.inc"diff --git a/llamafile/sgemm_hhs_neon.cpp b/llamafile/sgemm_hhs_neon.cppindex f93e1e89..5507a045 100644--- a/llamafile/sgemm_hhs_neon.cpp+++ b/llamafile/sgemm_hhs_neon.cpp@@ -22,15 +22,11 @@#define KN 8-typedef float16x8_t V;-typedef float16x8_t D;-typedef unsigned short TA;-typedef unsigned short TB;-typedef float TC;--static inline V zero() {- return vdupq_n_f16(0.f);-}+#define V float16x8_t+#define D float16x8_t+#define TA unsigned short+#define TB unsigned short+#define TC floatstatic inline V load(const unsigned short *p) {return vld1q_f16((const __fp16 *)p);diff --git a/llamafile/sgemm_hss_avx512f.cpp b/llamafile/sgemm_hss_avx512f.cppindex 7abd5f52..f0a8bf33 100644--- a/llamafile/sgemm_hss_avx512f.cpp+++ b/llamafile/sgemm_hss_avx512f.cpp@@ -22,15 +22,11 @@#define KN 16-typedef __m512 V;-typedef __m512 D;-typedef unsigned short TA;-typedef float TB;-typedef float TC;--static inline V zero() {- return _mm512_setzero_ps();-}+#define V __m512+#define D __m512+#define TA unsigned short+#define TB float+#define TC floatstatic inline V load(const float *p) {return _mm512_loadu_ps(p);diff --git a/llamafile/sgemm_hss_f16c.cpp b/llamafile/sgemm_hss_f16c.cppindex 14859edc..e9ba461b 100644--- a/llamafile/sgemm_hss_f16c.cpp+++ b/llamafile/sgemm_hss_f16c.cpp@@ -22,15 +22,11 @@#define KN 8-typedef __m256 V;-typedef __m256 D;-typedef unsigned short TA;-typedef float TB;-typedef float TC;--static inline V zero() {- return _mm256_setzero_ps();-}+#define V __m256+#define D __m256+#define TA unsigned short+#define TB float+#define TC floatstatic inline V load(const float *p) {return _mm256_loadu_ps(p);diff --git a/llamafile/sgemm_hss_neon.cpp b/llamafile/sgemm_hss_neon.cppindex 9ae6e68e..e10e37e9 100644--- a/llamafile/sgemm_hss_neon.cpp+++ b/llamafile/sgemm_hss_neon.cpp@@ -22,15 +22,11 @@#define KN 4-typedef float32x4_t V;-typedef float32x4_t D;-typedef unsigned short TA;-typedef float TB;-typedef float TC;--static inline V zero() {- return vdupq_n_f32(0.f);-}+#define V float32x4_t+#define D float32x4_t+#define TA unsigned short+#define TB float+#define TC floatstatic inline V load(const float *p) {return vld1q_f32(p);diff --git a/llamafile/sgemm_q0q0s_avx512vnni.cpp b/llamafile/sgemm_q0q0s_avx512vnni.cppindex aac59018..e5b3ed24 100644--- a/llamafile/sgemm_q0q0s_avx512vnni.cpp+++ b/llamafile/sgemm_q0q0s_avx512vnni.cpp@@ -19,9 +19,9 @@#include "sgemm.h"-typedef block_q8_0 TA;-typedef block_q8_0 TB;-typedef float TC;+#define TA block_q8_0+#define TB block_q8_0+#define TC float#include "sgemmer0.inc"diff --git a/llamafile/sgemm_q0q0s_avxvnni.cpp b/llamafile/sgemm_q0q0s_avxvnni.cppindex 34c5307a..d53b2979 100644--- a/llamafile/sgemm_q0q0s_avxvnni.cpp+++ b/llamafile/sgemm_q0q0s_avxvnni.cpp@@ -19,9 +19,9 @@#include "sgemm.h"-typedef block_q8_0 TA;-typedef block_q8_0 TB;-typedef float TC;+#define TA block_q8_0+#define TB block_q8_0+#define TC float#include "sgemmer0.inc"diff --git a/llamafile/sgemm_q0q0s_fma.cpp b/llamafile/sgemm_q0q0s_fma.cppindex ff6eeda6..437ec389 100644--- a/llamafile/sgemm_q0q0s_fma.cpp+++ b/llamafile/sgemm_q0q0s_fma.cpp@@ -19,9 +19,9 @@#include "sgemm.h"-typedef block_q8_0 TA;-typedef block_q8_0 TB;-typedef float TC;+#define TA block_q8_0+#define TB block_q8_0+#define TC float#include "sgemmer0.inc"diff --git a/llamafile/sgemm_sss_avx.cpp b/llamafile/sgemm_sss_avx.cppindex a7ea9055..a49427c8 100644--- a/llamafile/sgemm_sss_avx.cpp+++ b/llamafile/sgemm_sss_avx.cpp@@ -22,15 +22,11 @@#define KN 8-typedef __m256 V;-typedef __m256 D;-typedef float TA;-typedef float TB;-typedef float TC;--static inline V zero() {- return _mm256_setzero_ps();-}+#define V __m256+#define D __m256+#define TA float+#define TB float+#define TC floatstatic inline V load(const float *p) {return _mm256_loadu_ps(p);diff --git a/llamafile/sgemm_sss_avx512f.cpp b/llamafile/sgemm_sss_avx512f.cppindex 09eb708f..a9dbc1a0 100644--- a/llamafile/sgemm_sss_avx512f.cpp+++ b/llamafile/sgemm_sss_avx512f.cpp@@ -22,15 +22,11 @@#define KN 16-typedef __m512 V;-typedef __m512 D;-typedef float TA;-typedef float TB;-typedef float TC;--static inline V zero() {- return _mm512_setzero_ps();-}+#define V __m512+#define D __m512+#define TA float+#define TB float+#define TC floatstatic inline V load(const float *p) {return _mm512_loadu_ps(p);diff --git a/llamafile/sgemm_sss_fma.cpp b/llamafile/sgemm_sss_fma.cppindex b32e6567..f12ad7d6 100644--- a/llamafile/sgemm_sss_fma.cpp+++ b/llamafile/sgemm_sss_fma.cpp@@ -22,15 +22,11 @@#define KN 8-typedef __m256 V;-typedef __m256 D;-typedef float TA;-typedef float TB;-typedef float TC;--static inline V zero() {- return _mm256_setzero_ps();-}+#define V __m256+#define D __m256+#define TA float+#define TB float+#define TC floatstatic inline V load(const float *p) {return _mm256_loadu_ps(p);diff --git a/llamafile/sgemm_sss_neon.cpp b/llamafile/sgemm_sss_neon.cppindex f744654f..88644221 100644--- a/llamafile/sgemm_sss_neon.cpp+++ b/llamafile/sgemm_sss_neon.cpp@@ -22,15 +22,11 @@#define KN 4-typedef float32x4_t V;-typedef float32x4_t D;-typedef float TA;-typedef float TB;-typedef float TC;--static inline V zero() {- return vdupq_n_f32(0.f);-}+#define V float32x4_t+#define D float32x4_t+#define TA float+#define TB float+#define TC floatstatic inline V load(const float *p) {return vld1q_f32(p);diff --git a/llamafile/sgemm_vecdot_test.cpp b/llamafile/sgemm_vecdot_test.cppindex 908a8b11..79507d41 100644--- a/llamafile/sgemm_vecdot_test.cpp+++ b/llamafile/sgemm_vecdot_test.cpp@@ -85,6 +85,10 @@ int main(int argc, char *argv[]) {// 22.291 average ulp// 1566 worst ulp+ // using kahan summation+ // 2.14244 average ulp+ // 134 worst ulp+if (err_avg > 30)return 5;if (err_worst > 3000)diff --git a/llamafile/sgemmer.inc b/llamafile/sgemmer.incindex c44d3852..7bc8cf3b 100644--- a/llamafile/sgemmer.inc+++ b/llamafile/sgemmer.inc@@ -19,7 +19,7 @@#include "hsum.h"#include "kernel.h"-#include "madd.h"+#include "varith.h"namespace {@@ -68,31 +68,31 @@ class SGEMMER {dontinline void gemm5x5(int m0, int m, int n0, int n) {BEGIN_KERNEL(5, 5)- D c00 = zero();- D c01 = zero();- D c02 = zero();- D c03 = zero();- D c04 = zero();- D c10 = zero();- D c11 = zero();- D c12 = zero();- D c13 = zero();- D c14 = zero();- D c20 = zero();- D c21 = zero();- D c22 = zero();- D c23 = zero();- D c24 = zero();- D c30 = zero();- D c31 = zero();- D c32 = zero();- D c33 = zero();- D c34 = zero();- D c40 = zero();- D c41 = zero();- D c42 = zero();- D c43 = zero();- D c44 = zero();+ D c00 = {0};+ D c01 = {0};+ D c02 = {0};+ D c03 = {0};+ D c04 = {0};+ D c10 = {0};+ D c11 = {0};+ D c12 = {0};+ D c13 = {0};+ D c14 = {0};+ D c20 = {0};+ D c21 = {0};+ D c22 = {0};+ D c23 = {0};+ D c24 = {0};+ D c30 = {0};+ D c31 = {0};+ D c32 = {0};+ D c33 = {0};+ D c34 = {0};+ D c40 = {0};+ D c41 = {0};+ D c42 = {0};+ D c43 = {0};+ D c44 = {0};for (int l = 0; l < k; l += KN) {V k0 = load(B + ldb * (j + 0) + l);V k1 = load(B + ldb * (j + 1) + l);@@ -160,18 +160,18 @@ class SGEMMER {dontinline void gemm3x4(int m0, int m, int n0, int n) {BEGIN_KERNEL(3, 4)- D c00 = zero();- D c01 = zero();- D c02 = zero();- D c03 = zero();- D c10 = zero();- D c11 = zero();- D c12 = zero();- D c13 = zero();- D c20 = zero();- D c21 = zero();- D c22 = zero();- D c23 = zero();+ D c00 = {0};+ D c01 = {0};+ D c02 = {0};+ D c03 = {0};+ D c10 = {0};+ D c11 = {0};+ D c12 = {0};+ D c13 = {0};+ D c20 = {0};+ D c21 = {0};+ D c22 = {0};+ D c23 = {0};for (int l = 0; l < k; l += KN) {V k0 = load(B + ldb * (j + 0) + l);V k1 = load(B + ldb * (j + 1) + l);@@ -210,20 +210,16 @@ class SGEMMER {dontinline void gemm1x4(int m0, int m, int n0, int n) {BEGIN_KERNEL(1, 4)- D c00 = zero();- D c01 = zero();- D c02 = zero();- D c03 = zero();+ D c00 = {0}, e00 = {0};+ D c01 = {0}, e01 = {0};+ D c02 = {0}, e02 = {0};+ D c03 = {0}, e03 = {0};for (int l = 0; l < k; l += KN) {- V a0 = load(A + lda * (i + 0) + l);- V k0 = load(B + ldb * (j + 0) + l);- V k1 = load(B + ldb * (j + 1) + l);- V k2 = load(B + ldb * (j + 2) + l);- V k3 = load(B + ldb * (j + 3) + l);- c00 = madd(a0, k0, c00);- c01 = madd(a0, k1, c01);- c02 = madd(a0, k2, c02);- c03 = madd(a0, k3, c03);+ V a = load(A + lda * (i + 0) + l);+ c00 = madder(a, load(B + ldb * (j + 0) + l), c00, &e00);+ c01 = madder(a, load(B + ldb * (j + 1) + l), c01, &e01);+ c02 = madder(a, load(B + ldb * (j + 2) + l), c02, &e02);+ c03 = madder(a, load(B + ldb * (j + 3) + l), c03, &e03);}C[ldc * (j + 0) + (i + 0)] = hsum(c00);C[ldc * (j + 1) + (i + 0)] = hsum(c01);@@ -234,79 +230,30 @@ class SGEMMER {dontinline void gemm4x1(int m0, int m, int n0, int n) {BEGIN_KERNEL(4, 1)- D c00 = zero();- D c01 = zero();- D c02 = zero();- D c10 = zero();- D c11 = zero();- D c12 = zero();- D c20 = zero();- D c21 = zero();- D c22 = zero();- D c30 = zero();- D c31 = zero();- D c32 = zero();- int l = 0;- while (l + KN * 3 <= k) {- {- V k0 = load(B + ldb * (j + 0) + l);- c00 = madd(load(A + lda * (i + 0) + l), k0, c00);- c10 = madd(load(A + lda * (i + 1) + l), k0, c10);- c20 = madd(load(A + lda * (i + 2) + l), k0, c20);- c30 = madd(load(A + lda * (i + 3) + l), k0, c30);- }- l += KN;- {- V k0 = load(B + ldb * (j + 0) + l);- c01 = madd(load(A + lda * (i + 0) + l), k0, c01);- c11 = madd(load(A + lda * (i + 1) + l), k0, c11);- c21 = madd(load(A + lda * (i + 2) + l), k0, c21);- c31 = madd(load(A + lda * (i + 3) + l), k0, c31);- }- l += KN;- {- V k0 = load(B + ldb * (j + 0) + l);- c02 = madd(load(A + lda * (i + 0) + l), k0, c02);- c12 = madd(load(A + lda * (i + 1) + l), k0, c12);- c22 = madd(load(A + lda * (i + 2) + l), k0, c22);- c32 = madd(load(A + lda * (i + 3) + l), k0, c32);- }- l += KN;- }- for (; l < k; l += KN) {- V k0 = load(B + ldb * (j + 0) + l);- c00 = madd(load(A + lda * (i + 0) + l), k0, c00);- c10 = madd(load(A + lda * (i + 1) + l), k0, c10);- c20 = madd(load(A + lda * (i + 2) + l), k0, c20);- c30 = madd(load(A + lda * (i + 3) + l), k0, c30);+ D c00 = {0}, e00 = {0};+ D c10 = {0}, e10 = {0};+ D c20 = {0}, e20 = {0};+ D c30 = {0}, e30 = {0};+ for (int l = 0; l < k; l += KN) {+ V b = load(B + ldb * (j + 0) + l);+ c00 = madder(load(A + lda * (i + 0) + l), b, c00, &e00);+ c10 = madder(load(A + lda * (i + 1) + l), b, c10, &e10);+ c20 = madder(load(A + lda * (i + 2) + l), b, c20, &e20);+ c30 = madder(load(A + lda * (i + 3) + l), b, c30, &e30);}- C[ldc * (j + 0) + (i + 0)] = hsum(c00) + hsum(c01) + hsum(c02);- C[ldc * (j + 0) + (i + 1)] = hsum(c10) + hsum(c11) + hsum(c12);- C[ldc * (j + 0) + (i + 2)] = hsum(c20) + hsum(c21) + hsum(c22);- C[ldc * (j + 0) + (i + 3)] = hsum(c30) + hsum(c31) + hsum(c32);+ C[ldc * (j + 0) + (i + 0)] = hsum(c00);+ C[ldc * (j + 0) + (i + 1)] = hsum(c10);+ C[ldc * (j + 0) + (i + 2)] = hsum(c20);+ C[ldc * (j + 0) + (i + 3)] = hsum(c30);END_KERNEL()}dontinline void gemm1x1(int m0, int m, int n0, int n) {BEGIN_KERNEL(1, 1)- D c0 = zero();- D c1 = zero();- D c2 = zero();- D c3 = zero();- int l = 0;- while (l + KN * 4 <= k) {- c0 = madd(load(A + lda * i + l), load(B + ldb * j + l), c0);- l += KN;- c1 = madd(load(A + lda * i + l), load(B + ldb * j + l), c1);- l += KN;- c2 = madd(load(A + lda * i + l), load(B + ldb * j + l), c2);- l += KN;- c3 = madd(load(A + lda * i + l), load(B + ldb * j + l), c3);- l += KN;- }- for (; l < k; l += KN)- c0 = madd(load(A + lda * i + l), load(B + ldb * j + l), c0);- C[ldc * j + i] = hsum(c0) + hsum(c1) + hsum(c2) + hsum(c3);+ D c = {0}, e = {0};+ for (int l = 0; l < k; l += KN)+ c = madder(load(A + lda * i + l), load(B + ldb * j + l), c, &e);+ C[ldc * j + i] = hsum(c);END_KERNEL()}diff --git a/llamafile/sgemmer0.inc b/llamafile/sgemmer0.incindex 0ff72d7b..68fecf7a 100644--- a/llamafile/sgemmer0.inc+++ b/llamafile/sgemmer0.inc@@ -20,7 +20,7 @@#include "hsum.h"#include "kernel.h"-#include "madd.h"+#include "varith.h"namespace {diff --git a/llamafile/sgemmer1.inc b/llamafile/sgemmer1.incindex 9c487dbc..7eb37d40 100644--- a/llamafile/sgemmer1.inc+++ b/llamafile/sgemmer1.inc@@ -20,7 +20,7 @@#include "hsum.h"#include "kernel.h"-#include "madd.h"+#include "varith.h"namespace {diff --git a/llamafile/varith.h b/llamafile/varith.hnew file mode 100644index 00000000..1d5a1abf--- /dev/null+++ b/llamafile/varith.h@@ -0,0 +1,64 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+// clang-format off+#pragma once++#include <arm_neon.h>+#include <immintrin.h>++#ifdef __SSE__+inline __m128 add(__m128 x, __m128 y) { return _mm_add_ps(x, y); }+inline __m128 sub(__m128 x, __m128 y) { return _mm_sub_ps(x, y); }+inline __m128 mul(__m128 x, __m128 y) { return _mm_mul_ps(x, y); }+#endif // __SSE__++#ifdef __AVX__+inline __m256 add(__m256 x, __m256 y) { return _mm256_add_ps(x, y); }+inline __m256 sub(__m256 x, __m256 y) { return _mm256_sub_ps(x, y); }+inline __m256 mul(__m256 x, __m256 y) { return _mm256_mul_ps(x, y); }+#endif // __AVX__++#ifdef __AVX512F__+inline __m512 add(__m512 x, __m512 y) { return _mm512_add_ps(x, y); }+inline __m512 sub(__m512 x, __m512 y) { return _mm512_sub_ps(x, y); }+inline __m512 mul(__m512 x, __m512 y) { return _mm512_mul_ps(x, y); }+#endif // __AVX512F__++#ifdef __ARM_NEON+inline float32x4_t add(float32x4_t x, float32x4_t y) { return vaddq_f32(x, y); }+inline float32x4_t sub(float32x4_t x, float32x4_t y) { return vsubq_f32(x, y); }+inline float32x4_t mul(float32x4_t x, float32x4_t y) { return vmulq_f32(x, y); }+#endif // __ARM_NEON++#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC+inline float16x8_t add(float16x8_t x, float16x8_t y) { return vaddq_f16(x, y); }+inline float16x8_t sub(float16x8_t x, float16x8_t y) { return vsubq_f16(x, y); }+inline float16x8_t mul(float16x8_t x, float16x8_t y) { return vmulq_f16(x, y); }+#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC++/**+ * Computes a * b + c.+ *+ * This operation will become fused into a single arithmetic instruction+ * if the hardware has support for this feature, e.g. Intel Haswell+ (c.+ * 2013), AMD Bulldozer+ (c. 2011), etc.+ */+template <typename T, typename U>+inline U madd(T a, T b, U c) {+ return add(mul(a, b), c);+}++/**+ * Computes a * b + c with error correction.+ *+ * @see W. Kahan, "Further remarks on reducing truncation errors,"+ * Communications of the ACM, vol. 8, no. 1, p. 40, Jan. 1965,+ * doi: 10.1145/363707.363723.+ */+template <typename T, typename U>+inline U madder(T a, T b, U c, U *e) {+ U y = sub(mul(a, b), *e);+ U t = add(c, y);+ *e = sub(sub(t, c), y);+ return t;+}