Skip to content

Distribute and run LLMs with a single file.

Clean up more code

Justine Tunneye583f63dc074
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 3522ee27..fac7ec17 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -353,7 +353,7 @@ static void ggml_fp32_to_bf16_row_avx512bf16(const float * x, ggml_bf16_t * y, i
void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int n) {
#ifdef __x86_64__
- if (have_avx512bf16)
+ if (X86_HAVE(AVX512_BF16))
return ggml_fp32_to_bf16_row_avx512bf16(x, y, n);
#endif
for (int i = 0; i < n; i++) {
diff --git a/llamafile/hsum.h b/llamafile/hsum.h
index 7890ca17..347e32c1 100644
--- a/llamafile/hsum.h
+++ b/llamafile/hsum.h
@@ -2,12 +2,22 @@
// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
#pragma once
-#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)
+#ifdef __ARM_NEON
#include <arm_neon.h>
+#endif
+#ifdef __x86_64__
+#include <immintrin.h>
+#endif
+
+inline float hsum(float x) {
+ return x;
+}
+#ifdef __ARM_NEON
inline float hsum(float32x4_t x) {
return vaddvq_f32(x);
}
+#endif
#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
inline float hsum(float16x8_t x) {
@@ -15,29 +25,32 @@ inline float hsum(float16x8_t x) {
float32x4_t u = vcvt_f32_f16(vget_high_f16(x));
return vaddvq_f32(vaddq_f32(t, u));
}
-#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
-
-#elif defined(__SSE__)
-#include <immintrin.h>
+#endif
+#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
inline float hsum(__m128 x) {
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
x = _mm_add_ps(x, _mm_movehl_ps(x, x));
x = _mm_add_ss(x, _mm_movehdup_ps(x));
return _mm_cvtss_f32(x);
+#else
+ __m128 shuf = _mm_shuffle_ps(x, x, _MM_SHUFFLE(2, 3, 0, 1));
+ __m128 sums = _mm_add_ps(x, shuf);
+ shuf = _mm_movehl_ps(shuf, sums);
+ sums = _mm_add_ss(sums, shuf);
+ return _mm_cvtss_f32(sums);
+#endif
}
+#endif
-#ifdef __AVX__
-
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
inline float hsum(__m256 x) {
return hsum(_mm_add_ps(_mm256_extractf128_ps(x, 1), _mm256_castps256_ps128(x)));
}
+#endif
-#ifdef __AVX512F__
-
+#if defined(__AVX512F__)
inline float hsum(__m512 x) {
return _mm512_reduce_add_ps(x);
}
-
-#endif // __AVX512F__
-#endif // __AVX__
-#endif // __SSE__
+#endif
diff --git a/llamafile/kernel.h b/llamafile/kernel.h
index 9318eb71..66d5ff4f 100644
--- a/llamafile/kernel.h
+++ b/llamafile/kernel.h
@@ -6,13 +6,19 @@
#define VECTOR_REGISTERS 16
#endif
+#if defined(__AVX512F__)
+#define VECTOR_WIDTH 64
+#elif defined(__AVX__) || defined(__AVX2__)
+#define VECTOR_WIDTH 32
+#else
+#define VECTOR_WIDTH 16
+#endif
+
#define BEGIN_KERNEL(RM, RN) \
int ytiles = (m - m0) / RM; \
int xtiles = (n - n0) / RN; \
int tiles = ytiles * xtiles; \
int duty = (tiles + nth - 1) / nth; \
- if (duty < 1) \
- duty = 1; \
int start = duty * ith; \
int end = start + duty; \
if (end > tiles) \
diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
index bb6046e9..1f5694a4 100644
--- a/llamafile/llamafile.h
+++ b/llamafile/llamafile.h
@@ -31,7 +31,6 @@ void llamafile_launch_browser(const char *);
int llamafile_get_math_cpu_count(void);
extern bool FLAG_unsecure;
-extern const bool have_avx512bf16;
#define LLAMAFILE_GPU_ERROR -2
#define LLAMAFILE_GPU_DISABLE -1
diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cpp
index f109148b..1ae42ab1 100644
--- a/llamafile/sgemm.cpp
+++ b/llamafile/sgemm.cpp
@@ -23,22 +23,7 @@
#include <libc/sysv/consts/hwcap.h>
#include <sys/auxv.h>
-// TODO(jart): Delete when cosmocc 3.3.3 goes live.
-static bool check_avx512bf16(void) {
-#ifdef __x86_64__
- // available on cooperlake, zen4, etc.
- int abcd[4];
- __cpuidex(abcd, 1, 0);
- if (abcd[0] >= 7) {
- __cpuidex(abcd, 7, 1);
- return !!(abcd[0] & (1u << 5));
- }
-#endif
- return false;
-}
-
static const long hwcap = getauxval(AT_HWCAP);
-const bool have_avx512bf16 = check_avx512bf16();
/**
* Performs optimized matrix multiplication on CPU.
@@ -132,7 +117,7 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,
#ifdef __x86_64__
if (Btype != GGML_TYPE_F32)
return false;
- if (have_avx512bf16 && !(k % 32))
+ if (X86_HAVE(AVX512_BF16) && !(k % 32))
return llamafile_sgemm_bss_avx512bf16(m, n, k, (const ggml_bf16_t *)A, lda,
(const float *)B, ldb, (float *)C, ldc, ith, nth,
task);
diff --git a/llamafile/sgemmer.inc b/llamafile/sgemmer.inc
index 7bc8cf3b..aa9fad16 100644
--- a/llamafile/sgemmer.inc
+++ b/llamafile/sgemmer.inc
@@ -35,29 +35,29 @@ class SGEMMER {
private:
dontinline void mnpack(int m0, int m, int n0, int n) {
+ int mc, nc, mp, np;
if (m - m0 <= 0 || n - n0 <= 0)
return;
- int mc, nc, mp, np;
- if (VECTOR_REGISTERS >= 32 && n - n0 >= 5 && m - m0 >= 5) {
- mc = 5;
- nc = 5;
- gemm5x5(m0, m, n0, n);
- } else if (n - n0 >= 4 && m - m0 >= 3) {
- mc = 3;
- nc = 4;
- gemm3x4(m0, m, n0, n);
+ if (VECTOR_REGISTERS >= 32 && m - m0 >= 8 && n - n0 >= 3) {
+ mc = 8;
+ nc = 3;
+ gemm<8, 3>(m0, m, n0, n);
+ } else if (m - m0 >= 4 && n - n0 >= 3) {
+ mc = 4;
+ nc = 3;
+ gemm<4, 3>(m0, m, n0, n);
} else if (n - n0 >= 4) {
mc = 1;
nc = 4;
- gemm1x4(m0, m, n0, n);
+ gemm<1, 4>(m0, m, n0, n);
} else if (m - m0 >= 4) {
mc = 4;
nc = 1;
- gemm4x1(m0, m, n0, n);
+ gemm<4, 1>(m0, m, n0, n);
} else {
mc = 1;
nc = 1;
- gemm1x1(m0, m, n0, n);
+ gemm<1, 1>(m0, m, n0, n);
}
mp = m0 + (m - m0) / mc * mc;
np = n0 + (n - n0) / nc * nc;
@@ -66,195 +66,33 @@ class SGEMMER {
mnpack(mp, m, np, n);
}
- dontinline void gemm5x5(int m0, int m, int n0, int n) {
- BEGIN_KERNEL(5, 5)
- D c00 = {0};
- D c01 = {0};
- D c02 = {0};
- D c03 = {0};
- D c04 = {0};
- D c10 = {0};
- D c11 = {0};
- D c12 = {0};
- D c13 = {0};
- D c14 = {0};
- D c20 = {0};
- D c21 = {0};
- D c22 = {0};
- D c23 = {0};
- D c24 = {0};
- D c30 = {0};
- D c31 = {0};
- D c32 = {0};
- D c33 = {0};
- D c34 = {0};
- D c40 = {0};
- D c41 = {0};
- D c42 = {0};
- D c43 = {0};
- D c44 = {0};
- for (int l = 0; l < k; l += KN) {
- V k0 = load(B + ldb * (j + 0) + l);
- V k1 = load(B + ldb * (j + 1) + l);
- V k2 = load(B + ldb * (j + 2) + l);
- V k3 = load(B + ldb * (j + 3) + l);
- V k4 = load(B + ldb * (j + 4) + l);
- V a0 = load(A + lda * (i + 0) + l);
- c00 = madd(a0, k0, c00);
- c01 = madd(a0, k1, c01);
- c02 = madd(a0, k2, c02);
- c03 = madd(a0, k3, c03);
- c04 = madd(a0, k4, c04);
- V a1 = load(A + lda * (i + 1) + l);
- c10 = madd(a1, k0, c10);
- c11 = madd(a1, k1, c11);
- c12 = madd(a1, k2, c12);
- c13 = madd(a1, k3, c13);
- c14 = madd(a1, k4, c14);
- V a2 = load(A + lda * (i + 2) + l);
- c20 = madd(a2, k0, c20);
- c21 = madd(a2, k1, c21);
- c22 = madd(a2, k2, c22);
- c23 = madd(a2, k3, c23);
- c24 = madd(a2, k4, c24);
- V a3 = load(A + lda * (i + 3) + l);
- c30 = madd(a3, k0, c30);
- c31 = madd(a3, k1, c31);
- c32 = madd(a3, k2, c32);
- c33 = madd(a3, k3, c33);
- c34 = madd(a3, k4, c34);
- V a4 = load(A + lda * (i + 4) + l);
- c40 = madd(a4, k0, c40);
- c41 = madd(a4, k1, c41);
- c42 = madd(a4, k2, c42);
- c43 = madd(a4, k3, c43);
- c44 = madd(a4, k4, c44);
+ template <int RM, int RN> dontinline void gemm(int m0, int m, int n0, int n) {
+ int ytiles = (m - m0) / RM;
+ int xtiles = (n - n0) / RN;
+ int tiles = xtiles * ytiles;
+ int duty = (tiles + nth - 1) / nth;
+ int start = duty * ith;
+ int end = start + duty;
+ if (end > tiles)
+ end = tiles;
+ for (int job = start; job < end; ++job) {
+ int ii = m0 + job / xtiles * RM;
+ int jj = n0 + job % xtiles * RN;
+ D Cv[RN][RM] = {0};
+ for (int l = 0; l < k; l += KN)
+ for (int j = 0; j < RN; ++j)
+ for (int i = 0; i < RM; ++i)
+ Cv[j][i] = madd(load(A + lda * (ii + i) + l), //
+ load(B + ldb * (jj + j) + l), //
+ Cv[j][i]);
+ TC Cd[RN][RM];
+ for (int j = 0; j < RN; ++j)
+ for (int i = 0; i < RM; ++i)
+ Cd[j][i] = hsum(Cv[j][i]);
+ for (int j = 0; j < RN; ++j)
+ for (int i = 0; i < RM; ++i)
+ C[ldc * (jj + j) + (ii + i)] = Cd[j][i];
}
- C[ldc * (j + 0) + (i + 0)] = hsum(c00);
- C[ldc * (j + 0) + (i + 1)] = hsum(c10);
- C[ldc * (j + 0) + (i + 2)] = hsum(c20);
- C[ldc * (j + 0) + (i + 3)] = hsum(c30);
- C[ldc * (j + 0) + (i + 4)] = hsum(c40);
- C[ldc * (j + 1) + (i + 0)] = hsum(c01);
- C[ldc * (j + 1) + (i + 1)] = hsum(c11);
- C[ldc * (j + 1) + (i + 2)] = hsum(c21);
- C[ldc * (j + 1) + (i + 3)] = hsum(c31);
- C[ldc * (j + 1) + (i + 4)] = hsum(c41);
- C[ldc * (j + 2) + (i + 0)] = hsum(c02);
- C[ldc * (j + 2) + (i + 1)] = hsum(c12);
- C[ldc * (j + 2) + (i + 2)] = hsum(c22);
- C[ldc * (j + 2) + (i + 3)] = hsum(c32);
- C[ldc * (j + 2) + (i + 4)] = hsum(c42);
- C[ldc * (j + 3) + (i + 0)] = hsum(c03);
- C[ldc * (j + 3) + (i + 1)] = hsum(c13);
- C[ldc * (j + 3) + (i + 2)] = hsum(c23);
- C[ldc * (j + 3) + (i + 3)] = hsum(c33);
- C[ldc * (j + 3) + (i + 4)] = hsum(c43);
- C[ldc * (j + 4) + (i + 0)] = hsum(c04);
- C[ldc * (j + 4) + (i + 1)] = hsum(c14);
- C[ldc * (j + 4) + (i + 2)] = hsum(c24);
- C[ldc * (j + 4) + (i + 3)] = hsum(c34);
- C[ldc * (j + 4) + (i + 4)] = hsum(c44);
- END_KERNEL()
- }
-
- dontinline void gemm3x4(int m0, int m, int n0, int n) {
- BEGIN_KERNEL(3, 4)
- D c00 = {0};
- D c01 = {0};
- D c02 = {0};
- D c03 = {0};
- D c10 = {0};
- D c11 = {0};
- D c12 = {0};
- D c13 = {0};
- D c20 = {0};
- D c21 = {0};
- D c22 = {0};
- D c23 = {0};
- for (int l = 0; l < k; l += KN) {
- V k0 = load(B + ldb * (j + 0) + l);
- V k1 = load(B + ldb * (j + 1) + l);
- V k2 = load(B + ldb * (j + 2) + l);
- V k3 = load(B + ldb * (j + 3) + l);
- V a0 = load(A + lda * (i + 0) + l);
- c00 = madd(a0, k0, c00);
- c01 = madd(a0, k1, c01);
- c02 = madd(a0, k2, c02);
- c03 = madd(a0, k3, c03);
- V a1 = load(A + lda * (i + 1) + l);
- c10 = madd(a1, k0, c10);
- c11 = madd(a1, k1, c11);
- c12 = madd(a1, k2, c12);
- c13 = madd(a1, k3, c13);
- V a2 = load(A + lda * (i + 2) + l);
- c20 = madd(a2, k0, c20);
- c21 = madd(a2, k1, c21);
- c22 = madd(a2, k2, c22);
- c23 = madd(a2, k3, c23);
- }
- C[ldc * (j + 0) + (i + 0)] = hsum(c00);
- C[ldc * (j + 0) + (i + 1)] = hsum(c10);
- C[ldc * (j + 0) + (i + 2)] = hsum(c20);
- C[ldc * (j + 1) + (i + 0)] = hsum(c01);
- C[ldc * (j + 1) + (i + 1)] = hsum(c11);
- C[ldc * (j + 1) + (i + 2)] = hsum(c21);
- C[ldc * (j + 2) + (i + 0)] = hsum(c02);
- C[ldc * (j + 2) + (i + 1)] = hsum(c12);
- C[ldc * (j + 2) + (i + 2)] = hsum(c22);
- C[ldc * (j + 3) + (i + 0)] = hsum(c03);
- C[ldc * (j + 3) + (i + 1)] = hsum(c13);
- C[ldc * (j + 3) + (i + 2)] = hsum(c23);
- END_KERNEL()
- }
-
- dontinline void gemm1x4(int m0, int m, int n0, int n) {
- BEGIN_KERNEL(1, 4)
- D c00 = {0}, e00 = {0};
- D c01 = {0}, e01 = {0};
- D c02 = {0}, e02 = {0};
- D c03 = {0}, e03 = {0};
- for (int l = 0; l < k; l += KN) {
- V a = load(A + lda * (i + 0) + l);
- c00 = madder(a, load(B + ldb * (j + 0) + l), c00, &e00);
- c01 = madder(a, load(B + ldb * (j + 1) + l), c01, &e01);
- c02 = madder(a, load(B + ldb * (j + 2) + l), c02, &e02);
- c03 = madder(a, load(B + ldb * (j + 3) + l), c03, &e03);
- }
- C[ldc * (j + 0) + (i + 0)] = hsum(c00);
- C[ldc * (j + 1) + (i + 0)] = hsum(c01);
- C[ldc * (j + 2) + (i + 0)] = hsum(c02);
- C[ldc * (j + 3) + (i + 0)] = hsum(c03);
- END_KERNEL()
- }
-
- dontinline void gemm4x1(int m0, int m, int n0, int n) {
- BEGIN_KERNEL(4, 1)
- D c00 = {0}, e00 = {0};
- D c10 = {0}, e10 = {0};
- D c20 = {0}, e20 = {0};
- D c30 = {0}, e30 = {0};
- for (int l = 0; l < k; l += KN) {
- V b = load(B + ldb * (j + 0) + l);
- c00 = madder(load(A + lda * (i + 0) + l), b, c00, &e00);
- c10 = madder(load(A + lda * (i + 1) + l), b, c10, &e10);
- c20 = madder(load(A + lda * (i + 2) + l), b, c20, &e20);
- c30 = madder(load(A + lda * (i + 3) + l), b, c30, &e30);
- }
- C[ldc * (j + 0) + (i + 0)] = hsum(c00);
- C[ldc * (j + 0) + (i + 1)] = hsum(c10);
- C[ldc * (j + 0) + (i + 2)] = hsum(c20);
- C[ldc * (j + 0) + (i + 3)] = hsum(c30);
- END_KERNEL()
- }
-
- dontinline void gemm1x1(int m0, int m, int n0, int n) {
- BEGIN_KERNEL(1, 1)
- D c = {0}, e = {0};
- for (int l = 0; l < k; l += KN)
- c = madder(load(A + lda * i + l), load(B + ldb * j + l), c, &e);
- C[ldc * j + i] = hsum(c);
- END_KERNEL()
}
const TA *const A;
diff --git a/llamafile/varith.h b/llamafile/varith.h
index 1d5a1abf..8fb4d414 100644
--- a/llamafile/varith.h
+++ b/llamafile/varith.h
@@ -3,8 +3,16 @@
// clang-format off
#pragma once
+#ifdef __ARM_NEON
#include <arm_neon.h>
+#endif
+#ifdef __x86_64__
#include <immintrin.h>
+#endif
+
+inline float add(float x, float y) { return x + y; }
+inline float sub(float x, float y) { return x - y; }
+inline float mul(float x, float y) { return x * y; }
#ifdef __SSE__
inline __m128 add(__m128 x, __m128 y) { return _mm_add_ps(x, y); }