Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Clean up more code
Justine Tunneye583f63dc074
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 3522ee27..fac7ec17 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -353,7 +353,7 @@ static void ggml_fp32_to_bf16_row_avx512bf16(const float * x, ggml_bf16_t * y, ivoid ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int n) {#ifdef __x86_64__- if (have_avx512bf16)+ if (X86_HAVE(AVX512_BF16))return ggml_fp32_to_bf16_row_avx512bf16(x, y, n);#endiffor (int i = 0; i < n; i++) {diff --git a/llamafile/hsum.h b/llamafile/hsum.hindex 7890ca17..347e32c1 100644--- a/llamafile/hsum.h+++ b/llamafile/hsum.h@@ -2,12 +2,22 @@// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi#pragma once-#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)+#ifdef __ARM_NEON#include <arm_neon.h>+#endif+#ifdef __x86_64__+#include <immintrin.h>+#endif++inline float hsum(float x) {+ return x;+}+#ifdef __ARM_NEONinline float hsum(float32x4_t x) {return vaddvq_f32(x);}+#endif#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETICinline float hsum(float16x8_t x) {@@ -15,29 +25,32 @@ inline float hsum(float16x8_t x) {float32x4_t u = vcvt_f32_f16(vget_high_f16(x));return vaddvq_f32(vaddq_f32(t, u));}-#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC--#elif defined(__SSE__)-#include <immintrin.h>+#endif+#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)inline float hsum(__m128 x) {+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)x = _mm_add_ps(x, _mm_movehl_ps(x, x));x = _mm_add_ss(x, _mm_movehdup_ps(x));return _mm_cvtss_f32(x);+#else+ __m128 shuf = _mm_shuffle_ps(x, x, _MM_SHUFFLE(2, 3, 0, 1));+ __m128 sums = _mm_add_ps(x, shuf);+ shuf = _mm_movehl_ps(shuf, sums);+ sums = _mm_add_ss(sums, shuf);+ return _mm_cvtss_f32(sums);+#endif}+#endif-#ifdef __AVX__-+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)inline float hsum(__m256 x) {return hsum(_mm_add_ps(_mm256_extractf128_ps(x, 1), _mm256_castps256_ps128(x)));}+#endif-#ifdef __AVX512F__-+#if defined(__AVX512F__)inline float hsum(__m512 x) {return _mm512_reduce_add_ps(x);}--#endif // __AVX512F__-#endif // __AVX__-#endif // __SSE__+#endifdiff --git a/llamafile/kernel.h b/llamafile/kernel.hindex 9318eb71..66d5ff4f 100644--- a/llamafile/kernel.h+++ b/llamafile/kernel.h@@ -6,13 +6,19 @@#define VECTOR_REGISTERS 16#endif+#if defined(__AVX512F__)+#define VECTOR_WIDTH 64+#elif defined(__AVX__) || defined(__AVX2__)+#define VECTOR_WIDTH 32+#else+#define VECTOR_WIDTH 16+#endif+#define BEGIN_KERNEL(RM, RN) \int ytiles = (m - m0) / RM; \int xtiles = (n - n0) / RN; \int tiles = ytiles * xtiles; \int duty = (tiles + nth - 1) / nth; \- if (duty < 1) \- duty = 1; \int start = duty * ith; \int end = start + duty; \if (end > tiles) \diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex bb6046e9..1f5694a4 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -31,7 +31,6 @@ void llamafile_launch_browser(const char *);int llamafile_get_math_cpu_count(void);extern bool FLAG_unsecure;-extern const bool have_avx512bf16;#define LLAMAFILE_GPU_ERROR -2#define LLAMAFILE_GPU_DISABLE -1diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppindex f109148b..1ae42ab1 100644--- a/llamafile/sgemm.cpp+++ b/llamafile/sgemm.cpp@@ -23,22 +23,7 @@#include <libc/sysv/consts/hwcap.h>#include <sys/auxv.h>-// TODO(jart): Delete when cosmocc 3.3.3 goes live.-static bool check_avx512bf16(void) {-#ifdef __x86_64__- // available on cooperlake, zen4, etc.- int abcd[4];- __cpuidex(abcd, 1, 0);- if (abcd[0] >= 7) {- __cpuidex(abcd, 7, 1);- return !!(abcd[0] & (1u << 5));- }-#endif- return false;-}-static const long hwcap = getauxval(AT_HWCAP);-const bool have_avx512bf16 = check_avx512bf16();/*** Performs optimized matrix multiplication on CPU.@@ -132,7 +117,7 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,#ifdef __x86_64__if (Btype != GGML_TYPE_F32)return false;- if (have_avx512bf16 && !(k % 32))+ if (X86_HAVE(AVX512_BF16) && !(k % 32))return llamafile_sgemm_bss_avx512bf16(m, n, k, (const ggml_bf16_t *)A, lda,(const float *)B, ldb, (float *)C, ldc, ith, nth,task);diff --git a/llamafile/sgemmer.inc b/llamafile/sgemmer.incindex 7bc8cf3b..aa9fad16 100644--- a/llamafile/sgemmer.inc+++ b/llamafile/sgemmer.inc@@ -35,29 +35,29 @@ class SGEMMER {private:dontinline void mnpack(int m0, int m, int n0, int n) {+ int mc, nc, mp, np;if (m - m0 <= 0 || n - n0 <= 0)return;- int mc, nc, mp, np;- if (VECTOR_REGISTERS >= 32 && n - n0 >= 5 && m - m0 >= 5) {- mc = 5;- nc = 5;- gemm5x5(m0, m, n0, n);- } else if (n - n0 >= 4 && m - m0 >= 3) {- mc = 3;- nc = 4;- gemm3x4(m0, m, n0, n);+ if (VECTOR_REGISTERS >= 32 && m - m0 >= 8 && n - n0 >= 3) {+ mc = 8;+ nc = 3;+ gemm<8, 3>(m0, m, n0, n);+ } else if (m - m0 >= 4 && n - n0 >= 3) {+ mc = 4;+ nc = 3;+ gemm<4, 3>(m0, m, n0, n);} else if (n - n0 >= 4) {mc = 1;nc = 4;- gemm1x4(m0, m, n0, n);+ gemm<1, 4>(m0, m, n0, n);} else if (m - m0 >= 4) {mc = 4;nc = 1;- gemm4x1(m0, m, n0, n);+ gemm<4, 1>(m0, m, n0, n);} else {mc = 1;nc = 1;- gemm1x1(m0, m, n0, n);+ gemm<1, 1>(m0, m, n0, n);}mp = m0 + (m - m0) / mc * mc;np = n0 + (n - n0) / nc * nc;@@ -66,195 +66,33 @@ class SGEMMER {mnpack(mp, m, np, n);}- dontinline void gemm5x5(int m0, int m, int n0, int n) {- BEGIN_KERNEL(5, 5)- D c00 = {0};- D c01 = {0};- D c02 = {0};- D c03 = {0};- D c04 = {0};- D c10 = {0};- D c11 = {0};- D c12 = {0};- D c13 = {0};- D c14 = {0};- D c20 = {0};- D c21 = {0};- D c22 = {0};- D c23 = {0};- D c24 = {0};- D c30 = {0};- D c31 = {0};- D c32 = {0};- D c33 = {0};- D c34 = {0};- D c40 = {0};- D c41 = {0};- D c42 = {0};- D c43 = {0};- D c44 = {0};- for (int l = 0; l < k; l += KN) {- V k0 = load(B + ldb * (j + 0) + l);- V k1 = load(B + ldb * (j + 1) + l);- V k2 = load(B + ldb * (j + 2) + l);- V k3 = load(B + ldb * (j + 3) + l);- V k4 = load(B + ldb * (j + 4) + l);- V a0 = load(A + lda * (i + 0) + l);- c00 = madd(a0, k0, c00);- c01 = madd(a0, k1, c01);- c02 = madd(a0, k2, c02);- c03 = madd(a0, k3, c03);- c04 = madd(a0, k4, c04);- V a1 = load(A + lda * (i + 1) + l);- c10 = madd(a1, k0, c10);- c11 = madd(a1, k1, c11);- c12 = madd(a1, k2, c12);- c13 = madd(a1, k3, c13);- c14 = madd(a1, k4, c14);- V a2 = load(A + lda * (i + 2) + l);- c20 = madd(a2, k0, c20);- c21 = madd(a2, k1, c21);- c22 = madd(a2, k2, c22);- c23 = madd(a2, k3, c23);- c24 = madd(a2, k4, c24);- V a3 = load(A + lda * (i + 3) + l);- c30 = madd(a3, k0, c30);- c31 = madd(a3, k1, c31);- c32 = madd(a3, k2, c32);- c33 = madd(a3, k3, c33);- c34 = madd(a3, k4, c34);- V a4 = load(A + lda * (i + 4) + l);- c40 = madd(a4, k0, c40);- c41 = madd(a4, k1, c41);- c42 = madd(a4, k2, c42);- c43 = madd(a4, k3, c43);- c44 = madd(a4, k4, c44);+ template <int RM, int RN> dontinline void gemm(int m0, int m, int n0, int n) {+ int ytiles = (m - m0) / RM;+ int xtiles = (n - n0) / RN;+ int tiles = xtiles * ytiles;+ int duty = (tiles + nth - 1) / nth;+ int start = duty * ith;+ int end = start + duty;+ if (end > tiles)+ end = tiles;+ for (int job = start; job < end; ++job) {+ int ii = m0 + job / xtiles * RM;+ int jj = n0 + job % xtiles * RN;+ D Cv[RN][RM] = {0};+ for (int l = 0; l < k; l += KN)+ for (int j = 0; j < RN; ++j)+ for (int i = 0; i < RM; ++i)+ Cv[j][i] = madd(load(A + lda * (ii + i) + l), //+ load(B + ldb * (jj + j) + l), //+ Cv[j][i]);+ TC Cd[RN][RM];+ for (int j = 0; j < RN; ++j)+ for (int i = 0; i < RM; ++i)+ Cd[j][i] = hsum(Cv[j][i]);+ for (int j = 0; j < RN; ++j)+ for (int i = 0; i < RM; ++i)+ C[ldc * (jj + j) + (ii + i)] = Cd[j][i];}- C[ldc * (j + 0) + (i + 0)] = hsum(c00);- C[ldc * (j + 0) + (i + 1)] = hsum(c10);- C[ldc * (j + 0) + (i + 2)] = hsum(c20);- C[ldc * (j + 0) + (i + 3)] = hsum(c30);- C[ldc * (j + 0) + (i + 4)] = hsum(c40);- C[ldc * (j + 1) + (i + 0)] = hsum(c01);- C[ldc * (j + 1) + (i + 1)] = hsum(c11);- C[ldc * (j + 1) + (i + 2)] = hsum(c21);- C[ldc * (j + 1) + (i + 3)] = hsum(c31);- C[ldc * (j + 1) + (i + 4)] = hsum(c41);- C[ldc * (j + 2) + (i + 0)] = hsum(c02);- C[ldc * (j + 2) + (i + 1)] = hsum(c12);- C[ldc * (j + 2) + (i + 2)] = hsum(c22);- C[ldc * (j + 2) + (i + 3)] = hsum(c32);- C[ldc * (j + 2) + (i + 4)] = hsum(c42);- C[ldc * (j + 3) + (i + 0)] = hsum(c03);- C[ldc * (j + 3) + (i + 1)] = hsum(c13);- C[ldc * (j + 3) + (i + 2)] = hsum(c23);- C[ldc * (j + 3) + (i + 3)] = hsum(c33);- C[ldc * (j + 3) + (i + 4)] = hsum(c43);- C[ldc * (j + 4) + (i + 0)] = hsum(c04);- C[ldc * (j + 4) + (i + 1)] = hsum(c14);- C[ldc * (j + 4) + (i + 2)] = hsum(c24);- C[ldc * (j + 4) + (i + 3)] = hsum(c34);- C[ldc * (j + 4) + (i + 4)] = hsum(c44);- END_KERNEL()- }-- dontinline void gemm3x4(int m0, int m, int n0, int n) {- BEGIN_KERNEL(3, 4)- D c00 = {0};- D c01 = {0};- D c02 = {0};- D c03 = {0};- D c10 = {0};- D c11 = {0};- D c12 = {0};- D c13 = {0};- D c20 = {0};- D c21 = {0};- D c22 = {0};- D c23 = {0};- for (int l = 0; l < k; l += KN) {- V k0 = load(B + ldb * (j + 0) + l);- V k1 = load(B + ldb * (j + 1) + l);- V k2 = load(B + ldb * (j + 2) + l);- V k3 = load(B + ldb * (j + 3) + l);- V a0 = load(A + lda * (i + 0) + l);- c00 = madd(a0, k0, c00);- c01 = madd(a0, k1, c01);- c02 = madd(a0, k2, c02);- c03 = madd(a0, k3, c03);- V a1 = load(A + lda * (i + 1) + l);- c10 = madd(a1, k0, c10);- c11 = madd(a1, k1, c11);- c12 = madd(a1, k2, c12);- c13 = madd(a1, k3, c13);- V a2 = load(A + lda * (i + 2) + l);- c20 = madd(a2, k0, c20);- c21 = madd(a2, k1, c21);- c22 = madd(a2, k2, c22);- c23 = madd(a2, k3, c23);- }- C[ldc * (j + 0) + (i + 0)] = hsum(c00);- C[ldc * (j + 0) + (i + 1)] = hsum(c10);- C[ldc * (j + 0) + (i + 2)] = hsum(c20);- C[ldc * (j + 1) + (i + 0)] = hsum(c01);- C[ldc * (j + 1) + (i + 1)] = hsum(c11);- C[ldc * (j + 1) + (i + 2)] = hsum(c21);- C[ldc * (j + 2) + (i + 0)] = hsum(c02);- C[ldc * (j + 2) + (i + 1)] = hsum(c12);- C[ldc * (j + 2) + (i + 2)] = hsum(c22);- C[ldc * (j + 3) + (i + 0)] = hsum(c03);- C[ldc * (j + 3) + (i + 1)] = hsum(c13);- C[ldc * (j + 3) + (i + 2)] = hsum(c23);- END_KERNEL()- }-- dontinline void gemm1x4(int m0, int m, int n0, int n) {- BEGIN_KERNEL(1, 4)- D c00 = {0}, e00 = {0};- D c01 = {0}, e01 = {0};- D c02 = {0}, e02 = {0};- D c03 = {0}, e03 = {0};- for (int l = 0; l < k; l += KN) {- V a = load(A + lda * (i + 0) + l);- c00 = madder(a, load(B + ldb * (j + 0) + l), c00, &e00);- c01 = madder(a, load(B + ldb * (j + 1) + l), c01, &e01);- c02 = madder(a, load(B + ldb * (j + 2) + l), c02, &e02);- c03 = madder(a, load(B + ldb * (j + 3) + l), c03, &e03);- }- C[ldc * (j + 0) + (i + 0)] = hsum(c00);- C[ldc * (j + 1) + (i + 0)] = hsum(c01);- C[ldc * (j + 2) + (i + 0)] = hsum(c02);- C[ldc * (j + 3) + (i + 0)] = hsum(c03);- END_KERNEL()- }-- dontinline void gemm4x1(int m0, int m, int n0, int n) {- BEGIN_KERNEL(4, 1)- D c00 = {0}, e00 = {0};- D c10 = {0}, e10 = {0};- D c20 = {0}, e20 = {0};- D c30 = {0}, e30 = {0};- for (int l = 0; l < k; l += KN) {- V b = load(B + ldb * (j + 0) + l);- c00 = madder(load(A + lda * (i + 0) + l), b, c00, &e00);- c10 = madder(load(A + lda * (i + 1) + l), b, c10, &e10);- c20 = madder(load(A + lda * (i + 2) + l), b, c20, &e20);- c30 = madder(load(A + lda * (i + 3) + l), b, c30, &e30);- }- C[ldc * (j + 0) + (i + 0)] = hsum(c00);- C[ldc * (j + 0) + (i + 1)] = hsum(c10);- C[ldc * (j + 0) + (i + 2)] = hsum(c20);- C[ldc * (j + 0) + (i + 3)] = hsum(c30);- END_KERNEL()- }-- dontinline void gemm1x1(int m0, int m, int n0, int n) {- BEGIN_KERNEL(1, 1)- D c = {0}, e = {0};- for (int l = 0; l < k; l += KN)- c = madder(load(A + lda * i + l), load(B + ldb * j + l), c, &e);- C[ldc * j + i] = hsum(c);- END_KERNEL()}const TA *const A;diff --git a/llamafile/varith.h b/llamafile/varith.hindex 1d5a1abf..8fb4d414 100644--- a/llamafile/varith.h+++ b/llamafile/varith.h@@ -3,8 +3,16 @@// clang-format off#pragma once+#ifdef __ARM_NEON#include <arm_neon.h>+#endif+#ifdef __x86_64__#include <immintrin.h>+#endif++inline float add(float x, float y) { return x + y; }+inline float sub(float x, float y) { return x - y; }+inline float mul(float x, float y) { return x * y; }#ifdef __SSE__inline __m128 add(__m128 x, __m128 y) { return _mm_add_ps(x, y); }