Skip to content

Distribute and run LLMs with a single file.

Boost reading speed on Raspberry Pi 5

Justine Tunney1fecf91b4336
diff --git a/build/objdump b/build/objdump
index 27916ef5..7db7eda2 100755
--- a/build/objdump
+++ b/build/objdump
@@ -1,6 +1,6 @@
#!/bin/sh
if printf '%s\n' "$*" | grep aarch64 >/dev/null 2>&1; then
- exec aarch64-unknown-cosmo-objdump "$@"
+ exec aarch64-unknown-cosmo-objdump $1 ${2%/*}/.aarch64/${2##*/}
else
exec x86_64-unknown-cosmo-objdump "$@"
fi
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index efea3060..bc5dd65b 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -10716,13 +10716,13 @@ static void ggml_compute_forward_mul_mat(
}
#endif
-#ifdef __x86_64__
- bool can_use_tinyblas = ggml_is_contiguous(src0) &&
- ggml_is_contiguous(src1) &&
+ bool can_use_tinyblas = src1_cont &&
((ne2 == 1 && ne3 == 1) &&
(ne02 == 1 && ne03 == 1) &&
(ne12 == 1 && ne13 == 1));
+
if (can_use_tinyblas &&
+ src0->type == GGML_TYPE_F16 &&
llamafile_sgemm(ne01, ne11, ne00,
src0->data, nb01/ggml_type_size(src0->type),
src1->data, nb11/ggml_type_size(src1->type),
@@ -10734,7 +10734,6 @@ static void ggml_compute_forward_mul_mat(
dst->type)) {
return;
}
-#endif
if (params->type == GGML_TASK_TYPE_INIT) {
if (ith != 0) {
@@ -10767,7 +10766,6 @@ static void ggml_compute_forward_mul_mat(
const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
const size_t row_size = ggml_row_size(vec_dot_type, ne10);
-#ifdef __x86_64__
if (can_use_tinyblas &&
llamafile_sgemm(ne01, ne11, ne00,
src0->data, nb01/ggml_type_size(src0->type),
@@ -10780,7 +10778,6 @@ static void ggml_compute_forward_mul_mat(
dst->type)) {
return;
}
-#endif
const int64_t nr0 = ne01; // src0 rows
const int64_t nr1 = ne1*ne12*ne13; // src1 rows
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 92379c6e..d8901515 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -48,6 +48,7 @@ o/$(MODE)/llamafile/sgemm_sss_fma.o: private TARGET_ARCH += -Xx86_64-mfma
o/$(MODE)/llamafile/sgemm_sss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f
o/$(MODE)/llamafile/sgemm_hss_f16c.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mf16c
o/$(MODE)/llamafile/sgemm_hss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f
+o/$(MODE)/llamafile/sgemm_hhs_neon.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+fp16
o/$(MODE)/llamafile/sgemm_q0q0s_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni
o/$(MODE)/llamafile/sgemm_q0q0s_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni
o/$(MODE)/llamafile/sgemm_q0q0s_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma
diff --git a/llamafile/hsum.h b/llamafile/hsum.h
index 6295ebec..75af50ff 100644
--- a/llamafile/hsum.h
+++ b/llamafile/hsum.h
@@ -1,7 +1,23 @@
// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
#pragma once
-#ifdef __SSE__
+
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)
+#include <arm_neon.h>
+
+inline float hsum(float32x4_t x) {
+ return vaddvq_f32(x);
+}
+
+#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+inline float hsum(float16x8_t x) {
+ float32x4_t t = vcvt_f32_f16(vget_low_f16(x));
+ float32x4_t u = vcvt_f32_f16(vget_high_f16(x));
+ return vaddvq_f32(vaddq_f32(t, u));
+}
+#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+
+#elif defined(__SSE__)
#include <immintrin.h>
inline float hsum(__m128 x) {
@@ -18,8 +34,6 @@ inline float hsum(__m256 x) {
#ifdef __AVX512F__
-#include <immintrin.h>
-
inline float hsum(__m512 x) {
return _mm512_reduce_add_ps(x);
}
@@ -27,3 +41,10 @@ inline float hsum(__m512 x) {
#endif // __AVX512F__
#endif // __AVX__
#endif // __SSE__
+
+template <typename T> float hsums(const T *x, int n) {
+ float sum = 0;
+ for (int i = 0; i < n; ++i)
+ sum += hsum(x[i]);
+ return sum;
+}
diff --git a/llamafile/kernel.h b/llamafile/kernel.h
index dced6c4d..9318eb71 100644
--- a/llamafile/kernel.h
+++ b/llamafile/kernel.h
@@ -1,6 +1,6 @@
#pragma once
-#ifdef __AVX512F__
+#if defined(__ARM_NEON) || defined(__AVX512F__)
#define VECTOR_REGISTERS 32
#else
#define VECTOR_REGISTERS 16
diff --git a/llamafile/kernelcount.sh b/llamafile/kernelcount.sh
new file mode 100755
index 00000000..2bc4844e
--- /dev/null
+++ b/llamafile/kernelcount.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+# prints number of llamafile_sgemm kernels
+
+echo $(( $(aarch64-unknown-cosmo-nm \
+ -C o//llama.cpp/main/main.aarch64.elf |
+ grep GEMMER.*::gemm |
+ wc -l) +
+ $(x86_64-unknown-cosmo-nm \
+ -C o//llama.cpp/main/main.com.dbg |
+ grep GEMMER.*::gemm |
+ wc -l) ))
diff --git a/llamafile/madd.h b/llamafile/madd.h
index 1777c883..3994b306 100644
--- a/llamafile/madd.h
+++ b/llamafile/madd.h
@@ -1,7 +1,20 @@
// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
#pragma once
-#ifdef __SSE__
+#ifdef __ARM_NEON
+#include <arm_neon.h>
+
+inline float32x4_t madd(float32x4_t x, float32x4_t y, float32x4_t z) {
+ return vaddq_f32(vmulq_f32(x, y), z);
+}
+
+#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+inline float16x8_t madd(float16x8_t x, float16x8_t y, float16x8_t z) {
+ return vaddq_f16(vmulq_f16(x, y), z);
+}
+#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+
+#elif defined(__SSE__)
#include <immintrin.h>
inline __m128 madd(__m128 x, __m128 y, __m128 z) {
diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cpp
index 3430e6dc..76ae8a76 100644
--- a/llamafile/sgemm.cpp
+++ b/llamafile/sgemm.cpp
@@ -17,13 +17,47 @@
#include "sgemm.h"
#include "llamafile.h"
+#include <cassert>
#include <cosmo.h>
+#include <libc/sysv/consts/hwcap.h>
+#include <sys/auxv.h>
+static const long hwcap = getauxval(AT_HWCAP);
+
+/**
+ * Multiplies matrices.
+ *
+ * This is a column major GEMM subroutine for computing C = α*A*B + β*C.
+ *
+ * @param m is rows in `A` and `C`
+ * @param n is cols in `B` and `C`
+ * @param k is cols in `A` and rows in `B`
+ * @param A is first input matrix (always transposed)
+ * @param lda is row stride of `A`
+ * @param B is second input matrix (never transposed)
+ * @param ldb is row stride of `B`
+ * @param C is input/output array of output matrices
+ * @param ldc is row stride of `C`
+ * @param ith is thread id (must be less than `nth`)
+ * @param nth is number of threads (must be greater than zero)
+ * @param task is GGML task type
+ * @param Atype is GGML data type of `A`
+ * @param Btype is GGML data type of `B`
+ * @param Ctype is GGML data type of `C`
+ * @return true if this function was able to service the matmul request
+ */
bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B, int ldb, void *C,
int ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype) {
- if (!X86_HAVE(AVX))
- return false;
+ assert(m >= 0);
+ assert(n >= 0);
+ assert(k >= 0);
+ assert(lda >= k);
+ assert(ldb >= k);
+ assert(ldc >= m);
+ assert(nth > 0);
+ assert(ith < nth);
+
if (Ctype != GGML_TYPE_F32)
return false;
@@ -32,6 +66,9 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,
case GGML_TYPE_F32:
if (Btype != GGML_TYPE_F32)
return false;
+#ifdef __x86_64__
+ if (!X86_HAVE(AVX))
+ return false;
if (X86_HAVE(AVX512F) && !(k % 16))
return llamafile_sgemm_sss_avx512f(m, n, k, (const float *)A, lda, (const float *)B,
ldb, (float *)C, ldc, ith, nth, task);
@@ -41,9 +78,17 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,
if (!(k % 8))
return llamafile_sgemm_sss_avx(m, n, k, (const float *)A, lda, (const float *)B, ldb,
(float *)C, ldc, ith, nth, task);
+#elif defined(__aarch64__)
+ if (n > 1 && !(k % 4))
+ return llamafile_sgemm_sss_neon(m, n, k, (const float *)A, lda, (const float *)B, ldb,
+ (float *)C, ldc, ith, nth, task);
+#endif
return false;
case GGML_TYPE_F16:
+#ifdef __x86_64__
+ if (!X86_HAVE(AVX))
+ return false;
if (Btype != GGML_TYPE_F32)
return false;
if (X86_HAVE(AVX512F) && !(k % 16))
@@ -53,11 +98,23 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,
if (X86_HAVE(FMA) && X86_HAVE(F16C) && !(k % 8))
return llamafile_sgemm_hss_f16c(m, n, k, (const unsigned short *)A, lda,
(const float *)B, ldb, (float *)C, ldc, ith, nth, task);
+#elif defined(__aarch64__)
+ if (n > 1 && !(k % 8) && (hwcap & HWCAP_FPHP) && Btype == GGML_TYPE_F16)
+ return llamafile_sgemm_hhs_neon(m, n, k, (const unsigned short *)A, lda,
+ (const unsigned short *)B, ldb, (float *)C, ldc, ith,
+ nth, task);
+ if (n > 1 && !(k % 4) && !(hwcap & HWCAP_FPHP) && Btype == GGML_TYPE_F32)
+ return llamafile_sgemm_hss_neon(m, n, k, (const unsigned short *)A, lda,
+ (const float *)B, ldb, (float *)C, ldc, ith, nth, task);
+#endif
return false;
case GGML_TYPE_Q8_0:
if (Btype != GGML_TYPE_Q8_0)
return false;
+#ifdef __x86_64__
+ if (!X86_HAVE(AVX))
+ return false;
if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))
return llamafile_sgemm_q0q0s_avx512vnni(m, n, k, (const block_q8_0 *)A, lda,
(const block_q8_0 *)B, ldb, (float *)C, ldc,
@@ -70,11 +127,15 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,
return llamafile_sgemm_q0q0s_fma(m, n, k, (const block_q8_0 *)A, lda,
(const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,
task);
+#endif
return false;
case GGML_TYPE_Q4_0:
if (Btype != GGML_TYPE_Q8_0)
return false;
+#ifdef __x86_64__
+ if (!X86_HAVE(AVX))
+ return false;
if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))
return llamafile_sgemm_e0q0s_avx512vnni(m, n, k, (const block_q4_0 *)A, lda,
(const block_q8_0 *)B, ldb, (float *)C, ldc,
@@ -87,11 +148,15 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,
return llamafile_sgemm_e0q0s_fma(m, n, k, (const block_q4_0 *)A, lda,
(const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,
task);
+#endif
return false;
case GGML_TYPE_Q4_1:
if (Btype != GGML_TYPE_Q8_1)
return false;
+#ifdef __x86_64__
+ if (!X86_HAVE(AVX))
+ return false;
if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))
return llamafile_sgemm_e1q1s_avx512vnni(m, n, k, (const block_q4_1 *)A, lda,
(const block_q8_1 *)B, ldb, (float *)C, ldc,
@@ -104,6 +169,7 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,
return llamafile_sgemm_e1q1s_fma(m, n, k, (const block_q4_1 *)A, lda,
(const block_q8_1 *)B, ldb, (float *)C, ldc, ith, nth,
task);
+#endif
return false;
default:
diff --git a/llamafile/sgemm.h b/llamafile/sgemm.h
index 5c2c2db2..034989fc 100644
--- a/llamafile/sgemm.h
+++ b/llamafile/sgemm.h
@@ -11,11 +11,17 @@ bool llamafile_sgemm_sss_fma(int, int, int, const float *, int, const float *, i
int, int, int);
bool llamafile_sgemm_sss_avx512f(int, int, int, const float *, int, const float *, int, float *,
int, int, int, int);
+bool llamafile_sgemm_sss_neon(int, int, int, const float *, int, const float *, int, float *, int,
+ int, int, int);
bool llamafile_sgemm_hss_f16c(int, int, int, const unsigned short *, int, const float *, int,
float *, int, int, int, int);
bool llamafile_sgemm_hss_avx512f(int, int, int, const unsigned short *, int, const float *, int,
float *, int, int, int, int);
+bool llamafile_sgemm_hss_neon(int, int, int, const unsigned short *, int, const float *, int,
+ float *, int, int, int, int);
+bool llamafile_sgemm_hhs_neon(int, int, int, const unsigned short *, int, const unsigned short *,
+ int, float *, int, int, int, int);
bool llamafile_sgemm_q0q0s_avx512vnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *,
int, float *, int, int, int, int);
diff --git a/llamafile/sgemm_e0q0s_avx512vnni.cpp b/llamafile/sgemm_e0q0s_avx512vnni.cpp
index 18445ec0..1e2a1fd9 100644
--- a/llamafile/sgemm_e0q0s_avx512vnni.cpp
+++ b/llamafile/sgemm_e0q0s_avx512vnni.cpp
@@ -19,9 +19,10 @@
#include "sgemm.h"
-#define TA block_q4_0
-#define TB block_q8_0
-#define TC float
+typedef block_q4_0 TA;
+typedef block_q8_0 TB;
+typedef float TC;
+
#include "sgemmer0.inc"
bool llamafile_sgemm_e0q0s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,
diff --git a/llamafile/sgemm_e0q0s_avxvnni.cpp b/llamafile/sgemm_e0q0s_avxvnni.cpp
index 92598457..6682938c 100644
--- a/llamafile/sgemm_e0q0s_avxvnni.cpp
+++ b/llamafile/sgemm_e0q0s_avxvnni.cpp
@@ -19,9 +19,10 @@
#include "sgemm.h"
-#define TA block_q4_0
-#define TB block_q8_0
-#define TC float
+typedef block_q4_0 TA;
+typedef block_q8_0 TB;
+typedef float TC;
+
#include "sgemmer0.inc"
bool llamafile_sgemm_e0q0s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
diff --git a/llamafile/sgemm_e0q0s_fma.cpp b/llamafile/sgemm_e0q0s_fma.cpp
index bd69b661..fad68bf0 100644
--- a/llamafile/sgemm_e0q0s_fma.cpp
+++ b/llamafile/sgemm_e0q0s_fma.cpp
@@ -19,9 +19,10 @@
#include "sgemm.h"
-#define TA block_q4_0
-#define TB block_q8_0
-#define TC float
+typedef block_q4_0 TA;
+typedef block_q8_0 TB;
+typedef float TC;
+
#include "sgemmer0.inc"
bool llamafile_sgemm_e0q0s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
diff --git a/llamafile/sgemm_e1q1s_avx512vnni.cpp b/llamafile/sgemm_e1q1s_avx512vnni.cpp
index e2a4836b..83f50c81 100644
--- a/llamafile/sgemm_e1q1s_avx512vnni.cpp
+++ b/llamafile/sgemm_e1q1s_avx512vnni.cpp
@@ -19,9 +19,10 @@
#include "sgemm.h"
-#define TA block_q4_1
-#define TB block_q8_1
-#define TC float
+typedef block_q4_1 TA;
+typedef block_q8_1 TB;
+typedef float TC;
+
#include "sgemmer1.inc"
bool llamafile_sgemm_e1q1s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,
diff --git a/llamafile/sgemm_e1q1s_avxvnni.cpp b/llamafile/sgemm_e1q1s_avxvnni.cpp
index 9114427e..fe41b49b 100644
--- a/llamafile/sgemm_e1q1s_avxvnni.cpp
+++ b/llamafile/sgemm_e1q1s_avxvnni.cpp
@@ -19,9 +19,10 @@
#include "sgemm.h"
-#define TA block_q4_1
-#define TB block_q8_1
-#define TC float
+typedef block_q4_1 TA;
+typedef block_q8_1 TB;
+typedef float TC;
+
#include "sgemmer1.inc"
bool llamafile_sgemm_e1q1s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
diff --git a/llamafile/sgemm_e1q1s_fma.cpp b/llamafile/sgemm_e1q1s_fma.cpp
index 399232e2..585e7587 100644
--- a/llamafile/sgemm_e1q1s_fma.cpp
+++ b/llamafile/sgemm_e1q1s_fma.cpp
@@ -19,9 +19,10 @@
#include "sgemm.h"
-#define TA block_q4_1
-#define TB block_q8_1
-#define TC float
+typedef block_q4_1 TA;
+typedef block_q8_1 TB;
+typedef float TC;
+
#include "sgemmer1.inc"
bool llamafile_sgemm_e1q1s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
diff --git a/llamafile/sgemm_hhs_neon.cpp b/llamafile/sgemm_hhs_neon.cpp
new file mode 100644
index 00000000..f1f31c2b
--- /dev/null
+++ b/llamafile/sgemm_hhs_neon.cpp
@@ -0,0 +1,49 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __aarch64__
+
+#include "sgemm.h"
+#include <arm_neon.h>
+
+#define KN 8
+
+typedef float16x8_t V;
+typedef unsigned short TA;
+typedef unsigned short TB;
+typedef float TC;
+
+static inline V zero() {
+ return vdupq_n_f16(0.f);
+}
+
+static inline V load(const unsigned short *p) {
+ return vld1q_f16((const __fp16 *)p);
+}
+
+#include "sgemmer.inc"
+
+bool llamafile_sgemm_hhs_neon(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __aarch64__
diff --git a/llamafile/sgemm_hss_avx512f.cpp b/llamafile/sgemm_hss_avx512f.cpp
index 394f4124..d1584919 100644
--- a/llamafile/sgemm_hss_avx512f.cpp
+++ b/llamafile/sgemm_hss_avx512f.cpp
@@ -18,15 +18,27 @@
#ifdef __x86_64__
#include "sgemm.h"
+#include <immintrin.h>
#define KN 16
-#define V __m512
-#define TA unsigned short
-#define TB float
-#define TC float
-#define zero() _mm512_setzero_ps()
-#define load_a(p) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(p)))
-#define load_b(p) _mm512_loadu_ps(p)
+
+typedef __m512 V;
+typedef unsigned short TA;
+typedef float TB;
+typedef float TC;
+
+static inline V zero() {
+ return _mm512_setzero_ps();
+}
+
+static inline V load(const float *p) {
+ return _mm512_loadu_ps(p);
+}
+
+static inline V load(const unsigned short *p) {
+ return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(p)));
+}
+
#include "sgemmer.inc"
bool llamafile_sgemm_hss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
diff --git a/llamafile/sgemm_hss_f16c.cpp b/llamafile/sgemm_hss_f16c.cpp
index f2f45473..87bc04cd 100644
--- a/llamafile/sgemm_hss_f16c.cpp
+++ b/llamafile/sgemm_hss_f16c.cpp
@@ -18,15 +18,27 @@
#ifdef __x86_64__
#include "sgemm.h"
+#include <immintrin.h>
#define KN 8
-#define V __m256
-#define TA unsigned short
-#define TB float
-#define TC float
-#define zero() _mm256_setzero_ps()
-#define load_a(p) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(p)))
-#define load_b(p) _mm256_loadu_ps(p)
+
+typedef __m256 V;
+typedef unsigned short TA;
+typedef float TB;
+typedef float TC;
+
+static inline V zero() {
+ return _mm256_setzero_ps();
+}
+
+static inline V load(const float *p) {
+ return _mm256_loadu_ps(p);
+}
+
+static inline V load(const unsigned short *p) {
+ return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(p)));
+}
+
#include "sgemmer.inc"
bool llamafile_sgemm_hss_f16c(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
diff --git a/llamafile/sgemm_hss_neon.cpp b/llamafile/sgemm_hss_neon.cpp
new file mode 100644
index 00000000..aec6c67d
--- /dev/null
+++ b/llamafile/sgemm_hss_neon.cpp
@@ -0,0 +1,53 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __aarch64__
+
+#include "sgemm.h"
+#include <arm_neon.h>
+
+#define KN 4
+
+typedef float32x4_t V;
+typedef unsigned short TA;
+typedef float TB;
+typedef float TC;
+
+static inline V zero() {
+ return vdupq_n_f32(0.f);
+}
+
+static inline V load(const float *p) {
+ return vld1q_f32(p);
+}
+
+static inline V load(const unsigned short *p) {
+ return vcvt_f32_f16(vld1_f16((const __fp16 *)p));
+}
+
+#include "sgemmer.inc"
+
+bool llamafile_sgemm_hss_neon(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __aarch64__
diff --git a/llamafile/sgemm_q0q0s_avx512vnni.cpp b/llamafile/sgemm_q0q0s_avx512vnni.cpp
index 839b5c2d..aac59018 100644
--- a/llamafile/sgemm_q0q0s_avx512vnni.cpp
+++ b/llamafile/sgemm_q0q0s_avx512vnni.cpp
@@ -19,9 +19,10 @@
#include "sgemm.h"
-#define TA block_q8_0
-#define TB block_q8_0
-#define TC float
+typedef block_q8_0 TA;
+typedef block_q8_0 TB;
+typedef float TC;
+
#include "sgemmer0.inc"
bool llamafile_sgemm_q0q0s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,
diff --git a/llamafile/sgemm_q0q0s_avxvnni.cpp b/llamafile/sgemm_q0q0s_avxvnni.cpp
index 4c6eb23c..34c5307a 100644
--- a/llamafile/sgemm_q0q0s_avxvnni.cpp
+++ b/llamafile/sgemm_q0q0s_avxvnni.cpp
@@ -19,9 +19,10 @@
#include "sgemm.h"
-#define TA block_q8_0
-#define TB block_q8_0
-#define TC float
+typedef block_q8_0 TA;
+typedef block_q8_0 TB;
+typedef float TC;
+
#include "sgemmer0.inc"
bool llamafile_sgemm_q0q0s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
diff --git a/llamafile/sgemm_q0q0s_fma.cpp b/llamafile/sgemm_q0q0s_fma.cpp
index 99f71dd5..ff6eeda6 100644
--- a/llamafile/sgemm_q0q0s_fma.cpp
+++ b/llamafile/sgemm_q0q0s_fma.cpp
@@ -19,9 +19,10 @@
#include "sgemm.h"
-#define TA block_q8_0
-#define TB block_q8_0
-#define TC float
+typedef block_q8_0 TA;
+typedef block_q8_0 TB;
+typedef float TC;
+
#include "sgemmer0.inc"
bool llamafile_sgemm_q0q0s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
diff --git a/llamafile/sgemm_sss_avx.cpp b/llamafile/sgemm_sss_avx.cpp
index 97bd377d..73c4b90f 100644
--- a/llamafile/sgemm_sss_avx.cpp
+++ b/llamafile/sgemm_sss_avx.cpp
@@ -18,15 +18,23 @@
#ifdef __x86_64__
#include "sgemm.h"
+#include <immintrin.h>
#define KN 8
-#define V __m256
-#define TA float
-#define TB float
-#define TC float
-#define zero() _mm256_setzero_ps()
-#define load_a(p) _mm256_loadu_ps(p)
-#define load_b(p) _mm256_loadu_ps(p)
+
+typedef __m256 V;
+typedef float TA;
+typedef float TB;
+typedef float TC;
+
+static inline V zero() {
+ return _mm256_setzero_ps();
+}
+
+static inline V load(const float *p) {
+ return _mm256_loadu_ps(p);
+}
+
#include "sgemmer.inc"
bool llamafile_sgemm_sss_avx(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,
diff --git a/llamafile/sgemm_sss_avx512f.cpp b/llamafile/sgemm_sss_avx512f.cpp
index 7ccb188c..94b4e234 100644
--- a/llamafile/sgemm_sss_avx512f.cpp
+++ b/llamafile/sgemm_sss_avx512f.cpp
@@ -18,15 +18,23 @@
#ifdef __x86_64__
#include "sgemm.h"
+#include <immintrin.h>
#define KN 16
-#define V __m512
-#define TA float
-#define TB float
-#define TC float
-#define zero() _mm512_setzero_ps()
-#define load_a(p) _mm512_loadu_ps(p)
-#define load_b(p) _mm512_loadu_ps(p)
+
+typedef __m512 V;
+typedef float TA;
+typedef float TB;
+typedef float TC;
+
+static inline V zero() {
+ return _mm512_setzero_ps();
+}
+
+static inline V load(const float *p) {
+ return _mm512_loadu_ps(p);
+}
+
#include "sgemmer.inc"
bool llamafile_sgemm_sss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
diff --git a/llamafile/sgemm_sss_fma.cpp b/llamafile/sgemm_sss_fma.cpp
index f018bfe6..1e925b80 100644
--- a/llamafile/sgemm_sss_fma.cpp
+++ b/llamafile/sgemm_sss_fma.cpp
@@ -18,15 +18,23 @@
#ifdef __x86_64__
#include "sgemm.h"
+#include <immintrin.h>
#define KN 8
-#define V __m256
-#define TA float
-#define TB float
-#define TC float
-#define zero() _mm256_setzero_ps()
-#define load_a(p) _mm256_loadu_ps(p)
-#define load_b(p) _mm256_loadu_ps(p)
+
+typedef __m256 V;
+typedef float TA;
+typedef float TB;
+typedef float TC;
+
+static inline V zero() {
+ return _mm256_setzero_ps();
+}
+
+static inline V load(const float *p) {
+ return _mm256_loadu_ps(p);
+}
+
#include "sgemmer.inc"
bool llamafile_sgemm_sss_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,
diff --git a/llamafile/sgemm_sss_neon.cpp b/llamafile/sgemm_sss_neon.cpp
new file mode 100644
index 00000000..16f7f0a3
--- /dev/null
+++ b/llamafile/sgemm_sss_neon.cpp
@@ -0,0 +1,49 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __aarch64__
+
+#include "sgemm.h"
+#include <arm_neon.h>
+
+#define KN 4
+
+typedef float32x4_t V;
+typedef float TA;
+typedef float TB;
+typedef float TC;
+
+static inline V zero() {
+ return vdupq_n_f32(0.f);
+}
+
+static inline V load(const float *p) {
+ return vld1q_f32(p);
+}
+
+#include "sgemmer.inc"
+
+bool llamafile_sgemm_sss_neon(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __aarch64__
diff --git a/llamafile/sgemm_test.cpp b/llamafile/sgemm_test.cpp
index e633f725..0553f650 100644
--- a/llamafile/sgemm_test.cpp
+++ b/llamafile/sgemm_test.cpp
@@ -15,24 +15,22 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-#ifdef __x86_64__
-
#include "gemm.h"
#include "llama.cpp/ggml.h"
#include "llamafile.h"
-int rand32(void) {
+static int rand32(void) {
static unsigned long long lcg = 1;
lcg *= 6364136223846793005;
lcg += 1442695040888963407;
return lcg >> 32;
}
-float float01(unsigned x) { // (0,1)
+static float float01(unsigned x) { // (0,1)
return 1.f / 8388608 * ((x >> 9) + .5f);
}
-float numba(void) { // (-1,1)
+static float numba(void) { // (-1,1)
return float01(rand32()) * 2 - 1;
}
@@ -48,7 +46,7 @@ template <typename T> void randomize(int m, int n, T *A, int lda) {
}
int main(int argc, char *argv[]) {
- float tolerance = 1e-5;
+ float tolerance = 2e-5;
int m = 510;
int n = 513;
int k = 512;
@@ -66,28 +64,38 @@ int main(int argc, char *argv[]) {
broadcast(G, ldc * n, NAN);
randomize(k, m, A, lda);
randomize(k, n, B, ldb);
+
gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc);
if (!llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,
GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32))
return 1;
- for (int i = 0; i < m; ++i) {
+
+ for (int i = 0; i < m; ++i)
for (int j = 0; j < n; ++j) {
auto g = G[ldc * j + i];
auto c = C[ldc * j + i];
- if (isnan(g))
+ if (isnan(g)) {
+ fprintf(stderr, "%s:%d: found nan in reference matrix: i=%d j=%d\n", __FILE__,
+ __LINE__, i, j);
return 3;
- if (isnan(c))
+ }
+ if (isnan(c)) {
+ fprintf(stderr, "%s:%d: found nan in output matrix: i=%d j=%d\n", __FILE__,
+ __LINE__, i, j);
return 4;
- auto diff = g - c;
- if (diff < 0)
- diff = -diff;
- if (diff > tolerance)
+ }
+ auto d = g - c;
+ if (d < 0)
+ d = -d;
+ if (d > tolerance) {
+ fprintf(stderr, "%s:%d: difference exceeds tolerance: %g vs. %g (%g) i=%d j=%d\n",
+ __FILE__, __LINE__, g, c, d, i, j);
return 5;
+ }
}
- }
-}
-#else
-int main(int argc, char *argv[]) {
+ delete[] G;
+ delete[] C;
+ delete[] B;
+ delete[] A;
}
-#endif // __x86_64__
diff --git a/llamafile/sgemmer.inc b/llamafile/sgemmer.inc
index 54b9e5e5..8ee7da03 100644
--- a/llamafile/sgemmer.inc
+++ b/llamafile/sgemmer.inc
@@ -94,36 +94,36 @@ class SGEMMER {
V c43 = zero();
V c44 = zero();
for (int l = 0; l < k; l += KN) {
- V k0 = load_b(B + ldb * (j + 0) + l);
- V k1 = load_b(B + ldb * (j + 1) + l);
- V k2 = load_b(B + ldb * (j + 2) + l);
- V k3 = load_b(B + ldb * (j + 3) + l);
- V k4 = load_b(B + ldb * (j + 4) + l);
- V a0 = load_a(A + lda * (i + 0) + l);
+ V k0 = load(B + ldb * (j + 0) + l);
+ V k1 = load(B + ldb * (j + 1) + l);
+ V k2 = load(B + ldb * (j + 2) + l);
+ V k3 = load(B + ldb * (j + 3) + l);
+ V k4 = load(B + ldb * (j + 4) + l);
+ V a0 = load(A + lda * (i + 0) + l);
c00 = madd(a0, k0, c00);
c01 = madd(a0, k1, c01);
c02 = madd(a0, k2, c02);
c03 = madd(a0, k3, c03);
c04 = madd(a0, k4, c04);
- V a1 = load_a(A + lda * (i + 1) + l);
+ V a1 = load(A + lda * (i + 1) + l);
c10 = madd(a1, k0, c10);
c11 = madd(a1, k1, c11);
c12 = madd(a1, k2, c12);
c13 = madd(a1, k3, c13);
c14 = madd(a1, k4, c14);
- V a2 = load_a(A + lda * (i + 2) + l);
+ V a2 = load(A + lda * (i + 2) + l);
c20 = madd(a2, k0, c20);
c21 = madd(a2, k1, c21);
c22 = madd(a2, k2, c22);
c23 = madd(a2, k3, c23);
c24 = madd(a2, k4, c24);
- V a3 = load_a(A + lda * (i + 3) + l);
+ V a3 = load(A + lda * (i + 3) + l);
c30 = madd(a3, k0, c30);
c31 = madd(a3, k1, c31);
c32 = madd(a3, k2, c32);
c33 = madd(a3, k3, c33);
c34 = madd(a3, k4, c34);
- V a4 = load_a(A + lda * (i + 4) + l);
+ V a4 = load(A + lda * (i + 4) + l);
c40 = madd(a4, k0, c40);
c41 = madd(a4, k1, c41);
c42 = madd(a4, k2, c42);
@@ -158,6 +158,82 @@ class SGEMMER {
END_KERNEL()
}
+ dontinline void gemm5x4(int m0, int m, int n0, int n) {
+ BEGIN_KERNEL(5, 4)
+ V c00 = zero();
+ V c01 = zero();
+ V c02 = zero();
+ V c03 = zero();
+ V c10 = zero();
+ V c11 = zero();
+ V c12 = zero();
+ V c13 = zero();
+ V c20 = zero();
+ V c21 = zero();
+ V c22 = zero();
+ V c23 = zero();
+ V c30 = zero();
+ V c31 = zero();
+ V c32 = zero();
+ V c33 = zero();
+ V c40 = zero();
+ V c41 = zero();
+ V c42 = zero();
+ V c43 = zero();
+ for (int l = 0; l < k; l += KN) {
+ V k0 = load(B + ldb * (j + 0) + l);
+ V k1 = load(B + ldb * (j + 1) + l);
+ V k2 = load(B + ldb * (j + 2) + l);
+ V k3 = load(B + ldb * (j + 3) + l);
+ V a0 = load(A + lda * (i + 0) + l);
+ c00 = madd(a0, k0, c00);
+ c01 = madd(a0, k1, c01);
+ c02 = madd(a0, k2, c02);
+ c03 = madd(a0, k3, c03);
+ V a1 = load(A + lda * (i + 1) + l);
+ c10 = madd(a1, k0, c10);
+ c11 = madd(a1, k1, c11);
+ c12 = madd(a1, k2, c12);
+ c13 = madd(a1, k3, c13);
+ V a2 = load(A + lda * (i + 2) + l);
+ c20 = madd(a2, k0, c20);
+ c21 = madd(a2, k1, c21);
+ c22 = madd(a2, k2, c22);
+ c23 = madd(a2, k3, c23);
+ V a3 = load(A + lda * (i + 3) + l);
+ c30 = madd(a3, k0, c30);
+ c31 = madd(a3, k1, c31);
+ c32 = madd(a3, k2, c32);
+ c33 = madd(a3, k3, c33);
+ V a4 = load(A + lda * (i + 4) + l);
+ c40 = madd(a4, k0, c40);
+ c41 = madd(a4, k1, c41);
+ c42 = madd(a4, k2, c42);
+ c43 = madd(a4, k3, c43);
+ }
+ C[ldc * (j + 0) + (i + 0)] = hsum(c00);
+ C[ldc * (j + 0) + (i + 1)] = hsum(c10);
+ C[ldc * (j + 0) + (i + 2)] = hsum(c20);
+ C[ldc * (j + 0) + (i + 3)] = hsum(c30);
+ C[ldc * (j + 0) + (i + 4)] = hsum(c40);
+ C[ldc * (j + 1) + (i + 0)] = hsum(c01);
+ C[ldc * (j + 1) + (i + 1)] = hsum(c11);
+ C[ldc * (j + 1) + (i + 2)] = hsum(c21);
+ C[ldc * (j + 1) + (i + 3)] = hsum(c31);
+ C[ldc * (j + 1) + (i + 4)] = hsum(c41);
+ C[ldc * (j + 2) + (i + 0)] = hsum(c02);
+ C[ldc * (j + 2) + (i + 1)] = hsum(c12);
+ C[ldc * (j + 2) + (i + 2)] = hsum(c22);
+ C[ldc * (j + 2) + (i + 3)] = hsum(c32);
+ C[ldc * (j + 2) + (i + 4)] = hsum(c42);
+ C[ldc * (j + 3) + (i + 0)] = hsum(c03);
+ C[ldc * (j + 3) + (i + 1)] = hsum(c13);
+ C[ldc * (j + 3) + (i + 2)] = hsum(c23);
+ C[ldc * (j + 3) + (i + 3)] = hsum(c33);
+ C[ldc * (j + 3) + (i + 4)] = hsum(c43);
+ END_KERNEL()
+ }
+
dontinline void gemm3x4(int m0, int m, int n0, int n) {
BEGIN_KERNEL(3, 4)
V c00 = zero();
@@ -173,21 +249,21 @@ class SGEMMER {
V c22 = zero();
V c23 = zero();
for (int l = 0; l < k; l += KN) {
- V k0 = load_b(B + ldb * (j + 0) + l);
- V k1 = load_b(B + ldb * (j + 1) + l);
- V k2 = load_b(B + ldb * (j + 2) + l);
- V k3 = load_b(B + ldb * (j + 3) + l);
- V a0 = load_a(A + lda * (i + 0) + l);
+ V k0 = load(B + ldb * (j + 0) + l);
+ V k1 = load(B + ldb * (j + 1) + l);
+ V k2 = load(B + ldb * (j + 2) + l);
+ V k3 = load(B + ldb * (j + 3) + l);
+ V a0 = load(A + lda * (i + 0) + l);
c00 = madd(a0, k0, c00);
c01 = madd(a0, k1, c01);
c02 = madd(a0, k2, c02);
c03 = madd(a0, k3, c03);
- V a1 = load_a(A + lda * (i + 1) + l);
+ V a1 = load(A + lda * (i + 1) + l);
c10 = madd(a1, k0, c10);
c11 = madd(a1, k1, c11);
c12 = madd(a1, k2, c12);
c13 = madd(a1, k3, c13);
- V a2 = load_a(A + lda * (i + 2) + l);
+ V a2 = load(A + lda * (i + 2) + l);
c20 = madd(a2, k0, c20);
c21 = madd(a2, k1, c21);
c22 = madd(a2, k2, c22);
@@ -215,11 +291,11 @@ class SGEMMER {
V c02 = zero();
V c03 = zero();
for (int l = 0; l < k; l += KN) {
- V a0 = load_a(A + lda * (i + 0) + l);
- V k0 = load_b(B + ldb * (j + 0) + l);
- V k1 = load_b(B + ldb * (j + 1) + l);
- V k2 = load_b(B + ldb * (j + 2) + l);
- V k3 = load_b(B + ldb * (j + 3) + l);
+ V a0 = load(A + lda * (i + 0) + l);
+ V k0 = load(B + ldb * (j + 0) + l);
+ V k1 = load(B + ldb * (j + 1) + l);
+ V k2 = load(B + ldb * (j + 2) + l);
+ V k3 = load(B + ldb * (j + 3) + l);
c00 = madd(a0, k0, c00);
c01 = madd(a0, k1, c01);
c02 = madd(a0, k2, c02);
@@ -239,14 +315,14 @@ class SGEMMER {
V c20 = zero();
V c30 = zero();
for (int l = 0; l < k; l += KN) {
- V k0 = load_b(B + ldb * (j + 0) + l);
- V a0 = load_a(A + lda * (i + 0) + l);
+ V k0 = load(B + ldb * (j + 0) + l);
+ V a0 = load(A + lda * (i + 0) + l);
c00 = madd(a0, k0, c00);
- V a1 = load_a(A + lda * (i + 1) + l);
+ V a1 = load(A + lda * (i + 1) + l);
c10 = madd(a1, k0, c10);
- V a2 = load_a(A + lda * (i + 2) + l);
+ V a2 = load(A + lda * (i + 2) + l);
c20 = madd(a2, k0, c20);
- V a3 = load_a(A + lda * (i + 3) + l);
+ V a3 = load(A + lda * (i + 3) + l);
c30 = madd(a3, k0, c30);
}
C[ldc * (j + 0) + (i + 0)] = hsum(c00);
@@ -260,7 +336,7 @@ class SGEMMER {
BEGIN_KERNEL(1, 1)
V c = zero();
for (int l = 0; l < k; l += KN)
- c = madd(load_a(A + lda * i + l), load_b(B + ldb * j + l), c);
+ c = madd(load(A + lda * i + l), load(B + ldb * j + l), c);
C[ldc * j + i] = hsum(c);
END_KERNEL()
}
diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cu
index f988f2ad..ef4dbc74 100644
--- a/llamafile/tinyblas.cu
+++ b/llamafile/tinyblas.cu
@@ -867,11 +867,11 @@ static tinyblasStatus_t tinyblasGSBE_launch(tinyblasHandle_t handle, tinyblasOpe
* @param k is cols in `A` and rows in `B`
* @param alpha points to scalar that's multiplied against input
* @param A is input array of first matrices
- * @param Atype is data type of `C`
+ * @param Atype is data type of `A`
* @param lda is row stride of `A`
* @param strideA is distance between matrices in `A`
* @param B is input array of second matrices
- * @param Btype is data type of `C`
+ * @param Btype is data type of `B`
* @param ldb is row stride of `B`
* @param strideB is distance between matrices in `B`
* @param beta points to scalar that's multiplied against the existing