Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Boost reading speed on Raspberry Pi 5
Justine Tunney1fecf91b4336
diff --git a/build/objdump b/build/objdumpindex 27916ef5..7db7eda2 100755--- a/build/objdump+++ b/build/objdump@@ -1,6 +1,6 @@#!/bin/shif printf '%s\n' "$*" | grep aarch64 >/dev/null 2>&1; then- exec aarch64-unknown-cosmo-objdump "$@"+ exec aarch64-unknown-cosmo-objdump $1 ${2%/*}/.aarch64/${2##*/}elseexec x86_64-unknown-cosmo-objdump "$@"fidiff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex efea3060..bc5dd65b 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -10716,13 +10716,13 @@ static void ggml_compute_forward_mul_mat(}#endif-#ifdef __x86_64__- bool can_use_tinyblas = ggml_is_contiguous(src0) &&- ggml_is_contiguous(src1) &&+ bool can_use_tinyblas = src1_cont &&((ne2 == 1 && ne3 == 1) &&(ne02 == 1 && ne03 == 1) &&(ne12 == 1 && ne13 == 1));+if (can_use_tinyblas &&+ src0->type == GGML_TYPE_F16 &&llamafile_sgemm(ne01, ne11, ne00,src0->data, nb01/ggml_type_size(src0->type),src1->data, nb11/ggml_type_size(src1->type),@@ -10734,7 +10734,6 @@ static void ggml_compute_forward_mul_mat(dst->type)) {return;}-#endifif (params->type == GGML_TASK_TYPE_INIT) {if (ith != 0) {@@ -10767,7 +10766,6 @@ static void ggml_compute_forward_mul_mat(const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;const size_t row_size = ggml_row_size(vec_dot_type, ne10);-#ifdef __x86_64__if (can_use_tinyblas &&llamafile_sgemm(ne01, ne11, ne00,src0->data, nb01/ggml_type_size(src0->type),@@ -10780,7 +10778,6 @@ static void ggml_compute_forward_mul_mat(dst->type)) {return;}-#endifconst int64_t nr0 = ne01; // src0 rowsconst int64_t nr1 = ne1*ne12*ne13; // src1 rowsdiff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 92379c6e..d8901515 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -48,6 +48,7 @@ o/$(MODE)/llamafile/sgemm_sss_fma.o: private TARGET_ARCH += -Xx86_64-mfmao/$(MODE)/llamafile/sgemm_sss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512fo/$(MODE)/llamafile/sgemm_hss_f16c.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mf16co/$(MODE)/llamafile/sgemm_hss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f+o/$(MODE)/llamafile/sgemm_hhs_neon.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+fp16o/$(MODE)/llamafile/sgemm_q0q0s_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnnio/$(MODE)/llamafile/sgemm_q0q0s_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnnio/$(MODE)/llamafile/sgemm_q0q0s_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfmadiff --git a/llamafile/hsum.h b/llamafile/hsum.hindex 6295ebec..75af50ff 100644--- a/llamafile/hsum.h+++ b/llamafile/hsum.h@@ -1,7 +1,23 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi#pragma once-#ifdef __SSE__++#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)+#include <arm_neon.h>++inline float hsum(float32x4_t x) {+ return vaddvq_f32(x);+}++#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC+inline float hsum(float16x8_t x) {+ float32x4_t t = vcvt_f32_f16(vget_low_f16(x));+ float32x4_t u = vcvt_f32_f16(vget_high_f16(x));+ return vaddvq_f32(vaddq_f32(t, u));+}+#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC++#elif defined(__SSE__)#include <immintrin.h>inline float hsum(__m128 x) {@@ -18,8 +34,6 @@ inline float hsum(__m256 x) {#ifdef __AVX512F__-#include <immintrin.h>-inline float hsum(__m512 x) {return _mm512_reduce_add_ps(x);}@@ -27,3 +41,10 @@ inline float hsum(__m512 x) {#endif // __AVX512F__#endif // __AVX__#endif // __SSE__++template <typename T> float hsums(const T *x, int n) {+ float sum = 0;+ for (int i = 0; i < n; ++i)+ sum += hsum(x[i]);+ return sum;+}diff --git a/llamafile/kernel.h b/llamafile/kernel.hindex dced6c4d..9318eb71 100644--- a/llamafile/kernel.h+++ b/llamafile/kernel.h@@ -1,6 +1,6 @@#pragma once-#ifdef __AVX512F__+#if defined(__ARM_NEON) || defined(__AVX512F__)#define VECTOR_REGISTERS 32#else#define VECTOR_REGISTERS 16diff --git a/llamafile/kernelcount.sh b/llamafile/kernelcount.shnew file mode 100755index 00000000..2bc4844e--- /dev/null+++ b/llamafile/kernelcount.sh@@ -0,0 +1,11 @@+#!/bin/sh+# prints number of llamafile_sgemm kernels++echo $(( $(aarch64-unknown-cosmo-nm \+ -C o//llama.cpp/main/main.aarch64.elf |+ grep GEMMER.*::gemm |+ wc -l) ++ $(x86_64-unknown-cosmo-nm \+ -C o//llama.cpp/main/main.com.dbg |+ grep GEMMER.*::gemm |+ wc -l) ))diff --git a/llamafile/madd.h b/llamafile/madd.hindex 1777c883..3994b306 100644--- a/llamafile/madd.h+++ b/llamafile/madd.h@@ -1,7 +1,20 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi#pragma once-#ifdef __SSE__+#ifdef __ARM_NEON+#include <arm_neon.h>++inline float32x4_t madd(float32x4_t x, float32x4_t y, float32x4_t z) {+ return vaddq_f32(vmulq_f32(x, y), z);+}++#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC+inline float16x8_t madd(float16x8_t x, float16x8_t y, float16x8_t z) {+ return vaddq_f16(vmulq_f16(x, y), z);+}+#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC++#elif defined(__SSE__)#include <immintrin.h>inline __m128 madd(__m128 x, __m128 y, __m128 z) {diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppindex 3430e6dc..76ae8a76 100644--- a/llamafile/sgemm.cpp+++ b/llamafile/sgemm.cpp@@ -17,13 +17,47 @@#include "sgemm.h"#include "llamafile.h"+#include <cassert>#include <cosmo.h>+#include <libc/sysv/consts/hwcap.h>+#include <sys/auxv.h>+static const long hwcap = getauxval(AT_HWCAP);++/**+ * Multiplies matrices.+ *+ * This is a column major GEMM subroutine for computing C = α*A*B + β*C.+ *+ * @param m is rows in `A` and `C`+ * @param n is cols in `B` and `C`+ * @param k is cols in `A` and rows in `B`+ * @param A is first input matrix (always transposed)+ * @param lda is row stride of `A`+ * @param B is second input matrix (never transposed)+ * @param ldb is row stride of `B`+ * @param C is input/output array of output matrices+ * @param ldc is row stride of `C`+ * @param ith is thread id (must be less than `nth`)+ * @param nth is number of threads (must be greater than zero)+ * @param task is GGML task type+ * @param Atype is GGML data type of `A`+ * @param Btype is GGML data type of `B`+ * @param Ctype is GGML data type of `C`+ * @return true if this function was able to service the matmul request+ */bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B, int ldb, void *C,int ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype) {- if (!X86_HAVE(AVX))- return false;+ assert(m >= 0);+ assert(n >= 0);+ assert(k >= 0);+ assert(lda >= k);+ assert(ldb >= k);+ assert(ldc >= m);+ assert(nth > 0);+ assert(ith < nth);+if (Ctype != GGML_TYPE_F32)return false;@@ -32,6 +66,9 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,case GGML_TYPE_F32:if (Btype != GGML_TYPE_F32)return false;+#ifdef __x86_64__+ if (!X86_HAVE(AVX))+ return false;if (X86_HAVE(AVX512F) && !(k % 16))return llamafile_sgemm_sss_avx512f(m, n, k, (const float *)A, lda, (const float *)B,ldb, (float *)C, ldc, ith, nth, task);@@ -41,9 +78,17 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,if (!(k % 8))return llamafile_sgemm_sss_avx(m, n, k, (const float *)A, lda, (const float *)B, ldb,(float *)C, ldc, ith, nth, task);+#elif defined(__aarch64__)+ if (n > 1 && !(k % 4))+ return llamafile_sgemm_sss_neon(m, n, k, (const float *)A, lda, (const float *)B, ldb,+ (float *)C, ldc, ith, nth, task);+#endifreturn false;case GGML_TYPE_F16:+#ifdef __x86_64__+ if (!X86_HAVE(AVX))+ return false;if (Btype != GGML_TYPE_F32)return false;if (X86_HAVE(AVX512F) && !(k % 16))@@ -53,11 +98,23 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,if (X86_HAVE(FMA) && X86_HAVE(F16C) && !(k % 8))return llamafile_sgemm_hss_f16c(m, n, k, (const unsigned short *)A, lda,(const float *)B, ldb, (float *)C, ldc, ith, nth, task);+#elif defined(__aarch64__)+ if (n > 1 && !(k % 8) && (hwcap & HWCAP_FPHP) && Btype == GGML_TYPE_F16)+ return llamafile_sgemm_hhs_neon(m, n, k, (const unsigned short *)A, lda,+ (const unsigned short *)B, ldb, (float *)C, ldc, ith,+ nth, task);+ if (n > 1 && !(k % 4) && !(hwcap & HWCAP_FPHP) && Btype == GGML_TYPE_F32)+ return llamafile_sgemm_hss_neon(m, n, k, (const unsigned short *)A, lda,+ (const float *)B, ldb, (float *)C, ldc, ith, nth, task);+#endifreturn false;case GGML_TYPE_Q8_0:if (Btype != GGML_TYPE_Q8_0)return false;+#ifdef __x86_64__+ if (!X86_HAVE(AVX))+ return false;if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))return llamafile_sgemm_q0q0s_avx512vnni(m, n, k, (const block_q8_0 *)A, lda,(const block_q8_0 *)B, ldb, (float *)C, ldc,@@ -70,11 +127,15 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,return llamafile_sgemm_q0q0s_fma(m, n, k, (const block_q8_0 *)A, lda,(const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,task);+#endifreturn false;case GGML_TYPE_Q4_0:if (Btype != GGML_TYPE_Q8_0)return false;+#ifdef __x86_64__+ if (!X86_HAVE(AVX))+ return false;if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))return llamafile_sgemm_e0q0s_avx512vnni(m, n, k, (const block_q4_0 *)A, lda,(const block_q8_0 *)B, ldb, (float *)C, ldc,@@ -87,11 +148,15 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,return llamafile_sgemm_e0q0s_fma(m, n, k, (const block_q4_0 *)A, lda,(const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,task);+#endifreturn false;case GGML_TYPE_Q4_1:if (Btype != GGML_TYPE_Q8_1)return false;+#ifdef __x86_64__+ if (!X86_HAVE(AVX))+ return false;if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))return llamafile_sgemm_e1q1s_avx512vnni(m, n, k, (const block_q4_1 *)A, lda,(const block_q8_1 *)B, ldb, (float *)C, ldc,@@ -104,6 +169,7 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,return llamafile_sgemm_e1q1s_fma(m, n, k, (const block_q4_1 *)A, lda,(const block_q8_1 *)B, ldb, (float *)C, ldc, ith, nth,task);+#endifreturn false;default:diff --git a/llamafile/sgemm.h b/llamafile/sgemm.hindex 5c2c2db2..034989fc 100644--- a/llamafile/sgemm.h+++ b/llamafile/sgemm.h@@ -11,11 +11,17 @@ bool llamafile_sgemm_sss_fma(int, int, int, const float *, int, const float *, iint, int, int);bool llamafile_sgemm_sss_avx512f(int, int, int, const float *, int, const float *, int, float *,int, int, int, int);+bool llamafile_sgemm_sss_neon(int, int, int, const float *, int, const float *, int, float *, int,+ int, int, int);bool llamafile_sgemm_hss_f16c(int, int, int, const unsigned short *, int, const float *, int,float *, int, int, int, int);bool llamafile_sgemm_hss_avx512f(int, int, int, const unsigned short *, int, const float *, int,float *, int, int, int, int);+bool llamafile_sgemm_hss_neon(int, int, int, const unsigned short *, int, const float *, int,+ float *, int, int, int, int);+bool llamafile_sgemm_hhs_neon(int, int, int, const unsigned short *, int, const unsigned short *,+ int, float *, int, int, int, int);bool llamafile_sgemm_q0q0s_avx512vnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *,int, float *, int, int, int, int);diff --git a/llamafile/sgemm_e0q0s_avx512vnni.cpp b/llamafile/sgemm_e0q0s_avx512vnni.cppindex 18445ec0..1e2a1fd9 100644--- a/llamafile/sgemm_e0q0s_avx512vnni.cpp+++ b/llamafile/sgemm_e0q0s_avx512vnni.cpp@@ -19,9 +19,10 @@#include "sgemm.h"-#define TA block_q4_0-#define TB block_q8_0-#define TC float+typedef block_q4_0 TA;+typedef block_q8_0 TB;+typedef float TC;+#include "sgemmer0.inc"bool llamafile_sgemm_e0q0s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,diff --git a/llamafile/sgemm_e0q0s_avxvnni.cpp b/llamafile/sgemm_e0q0s_avxvnni.cppindex 92598457..6682938c 100644--- a/llamafile/sgemm_e0q0s_avxvnni.cpp+++ b/llamafile/sgemm_e0q0s_avxvnni.cpp@@ -19,9 +19,10 @@#include "sgemm.h"-#define TA block_q4_0-#define TB block_q8_0-#define TC float+typedef block_q4_0 TA;+typedef block_q8_0 TB;+typedef float TC;+#include "sgemmer0.inc"bool llamafile_sgemm_e0q0s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,diff --git a/llamafile/sgemm_e0q0s_fma.cpp b/llamafile/sgemm_e0q0s_fma.cppindex bd69b661..fad68bf0 100644--- a/llamafile/sgemm_e0q0s_fma.cpp+++ b/llamafile/sgemm_e0q0s_fma.cpp@@ -19,9 +19,10 @@#include "sgemm.h"-#define TA block_q4_0-#define TB block_q8_0-#define TC float+typedef block_q4_0 TA;+typedef block_q8_0 TB;+typedef float TC;+#include "sgemmer0.inc"bool llamafile_sgemm_e0q0s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,diff --git a/llamafile/sgemm_e1q1s_avx512vnni.cpp b/llamafile/sgemm_e1q1s_avx512vnni.cppindex e2a4836b..83f50c81 100644--- a/llamafile/sgemm_e1q1s_avx512vnni.cpp+++ b/llamafile/sgemm_e1q1s_avx512vnni.cpp@@ -19,9 +19,10 @@#include "sgemm.h"-#define TA block_q4_1-#define TB block_q8_1-#define TC float+typedef block_q4_1 TA;+typedef block_q8_1 TB;+typedef float TC;+#include "sgemmer1.inc"bool llamafile_sgemm_e1q1s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,diff --git a/llamafile/sgemm_e1q1s_avxvnni.cpp b/llamafile/sgemm_e1q1s_avxvnni.cppindex 9114427e..fe41b49b 100644--- a/llamafile/sgemm_e1q1s_avxvnni.cpp+++ b/llamafile/sgemm_e1q1s_avxvnni.cpp@@ -19,9 +19,10 @@#include "sgemm.h"-#define TA block_q4_1-#define TB block_q8_1-#define TC float+typedef block_q4_1 TA;+typedef block_q8_1 TB;+typedef float TC;+#include "sgemmer1.inc"bool llamafile_sgemm_e1q1s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,diff --git a/llamafile/sgemm_e1q1s_fma.cpp b/llamafile/sgemm_e1q1s_fma.cppindex 399232e2..585e7587 100644--- a/llamafile/sgemm_e1q1s_fma.cpp+++ b/llamafile/sgemm_e1q1s_fma.cpp@@ -19,9 +19,10 @@#include "sgemm.h"-#define TA block_q4_1-#define TB block_q8_1-#define TC float+typedef block_q4_1 TA;+typedef block_q8_1 TB;+typedef float TC;+#include "sgemmer1.inc"bool llamafile_sgemm_e1q1s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,diff --git a/llamafile/sgemm_hhs_neon.cpp b/llamafile/sgemm_hhs_neon.cppnew file mode 100644index 00000000..f1f31c2b--- /dev/null+++ b/llamafile/sgemm_hhs_neon.cpp@@ -0,0 +1,49 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __aarch64__++#include "sgemm.h"+#include <arm_neon.h>++#define KN 8++typedef float16x8_t V;+typedef unsigned short TA;+typedef unsigned short TB;+typedef float TC;++static inline V zero() {+ return vdupq_n_f16(0.f);+}++static inline V load(const unsigned short *p) {+ return vld1q_f16((const __fp16 *)p);+}++#include "sgemmer.inc"++bool llamafile_sgemm_hhs_neon(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __aarch64__diff --git a/llamafile/sgemm_hss_avx512f.cpp b/llamafile/sgemm_hss_avx512f.cppindex 394f4124..d1584919 100644--- a/llamafile/sgemm_hss_avx512f.cpp+++ b/llamafile/sgemm_hss_avx512f.cpp@@ -18,15 +18,27 @@#ifdef __x86_64__#include "sgemm.h"+#include <immintrin.h>#define KN 16-#define V __m512-#define TA unsigned short-#define TB float-#define TC float-#define zero() _mm512_setzero_ps()-#define load_a(p) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(p)))-#define load_b(p) _mm512_loadu_ps(p)++typedef __m512 V;+typedef unsigned short TA;+typedef float TB;+typedef float TC;++static inline V zero() {+ return _mm512_setzero_ps();+}++static inline V load(const float *p) {+ return _mm512_loadu_ps(p);+}++static inline V load(const unsigned short *p) {+ return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(p)));+}+#include "sgemmer.inc"bool llamafile_sgemm_hss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,diff --git a/llamafile/sgemm_hss_f16c.cpp b/llamafile/sgemm_hss_f16c.cppindex f2f45473..87bc04cd 100644--- a/llamafile/sgemm_hss_f16c.cpp+++ b/llamafile/sgemm_hss_f16c.cpp@@ -18,15 +18,27 @@#ifdef __x86_64__#include "sgemm.h"+#include <immintrin.h>#define KN 8-#define V __m256-#define TA unsigned short-#define TB float-#define TC float-#define zero() _mm256_setzero_ps()-#define load_a(p) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(p)))-#define load_b(p) _mm256_loadu_ps(p)++typedef __m256 V;+typedef unsigned short TA;+typedef float TB;+typedef float TC;++static inline V zero() {+ return _mm256_setzero_ps();+}++static inline V load(const float *p) {+ return _mm256_loadu_ps(p);+}++static inline V load(const unsigned short *p) {+ return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(p)));+}+#include "sgemmer.inc"bool llamafile_sgemm_hss_f16c(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,diff --git a/llamafile/sgemm_hss_neon.cpp b/llamafile/sgemm_hss_neon.cppnew file mode 100644index 00000000..aec6c67d--- /dev/null+++ b/llamafile/sgemm_hss_neon.cpp@@ -0,0 +1,53 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __aarch64__++#include "sgemm.h"+#include <arm_neon.h>++#define KN 4++typedef float32x4_t V;+typedef unsigned short TA;+typedef float TB;+typedef float TC;++static inline V zero() {+ return vdupq_n_f32(0.f);+}++static inline V load(const float *p) {+ return vld1q_f32(p);+}++static inline V load(const unsigned short *p) {+ return vcvt_f32_f16(vld1_f16((const __fp16 *)p));+}++#include "sgemmer.inc"++bool llamafile_sgemm_hss_neon(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __aarch64__diff --git a/llamafile/sgemm_q0q0s_avx512vnni.cpp b/llamafile/sgemm_q0q0s_avx512vnni.cppindex 839b5c2d..aac59018 100644--- a/llamafile/sgemm_q0q0s_avx512vnni.cpp+++ b/llamafile/sgemm_q0q0s_avx512vnni.cpp@@ -19,9 +19,10 @@#include "sgemm.h"-#define TA block_q8_0-#define TB block_q8_0-#define TC float+typedef block_q8_0 TA;+typedef block_q8_0 TB;+typedef float TC;+#include "sgemmer0.inc"bool llamafile_sgemm_q0q0s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,diff --git a/llamafile/sgemm_q0q0s_avxvnni.cpp b/llamafile/sgemm_q0q0s_avxvnni.cppindex 4c6eb23c..34c5307a 100644--- a/llamafile/sgemm_q0q0s_avxvnni.cpp+++ b/llamafile/sgemm_q0q0s_avxvnni.cpp@@ -19,9 +19,10 @@#include "sgemm.h"-#define TA block_q8_0-#define TB block_q8_0-#define TC float+typedef block_q8_0 TA;+typedef block_q8_0 TB;+typedef float TC;+#include "sgemmer0.inc"bool llamafile_sgemm_q0q0s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,diff --git a/llamafile/sgemm_q0q0s_fma.cpp b/llamafile/sgemm_q0q0s_fma.cppindex 99f71dd5..ff6eeda6 100644--- a/llamafile/sgemm_q0q0s_fma.cpp+++ b/llamafile/sgemm_q0q0s_fma.cpp@@ -19,9 +19,10 @@#include "sgemm.h"-#define TA block_q8_0-#define TB block_q8_0-#define TC float+typedef block_q8_0 TA;+typedef block_q8_0 TB;+typedef float TC;+#include "sgemmer0.inc"bool llamafile_sgemm_q0q0s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,diff --git a/llamafile/sgemm_sss_avx.cpp b/llamafile/sgemm_sss_avx.cppindex 97bd377d..73c4b90f 100644--- a/llamafile/sgemm_sss_avx.cpp+++ b/llamafile/sgemm_sss_avx.cpp@@ -18,15 +18,23 @@#ifdef __x86_64__#include "sgemm.h"+#include <immintrin.h>#define KN 8-#define V __m256-#define TA float-#define TB float-#define TC float-#define zero() _mm256_setzero_ps()-#define load_a(p) _mm256_loadu_ps(p)-#define load_b(p) _mm256_loadu_ps(p)++typedef __m256 V;+typedef float TA;+typedef float TB;+typedef float TC;++static inline V zero() {+ return _mm256_setzero_ps();+}++static inline V load(const float *p) {+ return _mm256_loadu_ps(p);+}+#include "sgemmer.inc"bool llamafile_sgemm_sss_avx(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,diff --git a/llamafile/sgemm_sss_avx512f.cpp b/llamafile/sgemm_sss_avx512f.cppindex 7ccb188c..94b4e234 100644--- a/llamafile/sgemm_sss_avx512f.cpp+++ b/llamafile/sgemm_sss_avx512f.cpp@@ -18,15 +18,23 @@#ifdef __x86_64__#include "sgemm.h"+#include <immintrin.h>#define KN 16-#define V __m512-#define TA float-#define TB float-#define TC float-#define zero() _mm512_setzero_ps()-#define load_a(p) _mm512_loadu_ps(p)-#define load_b(p) _mm512_loadu_ps(p)++typedef __m512 V;+typedef float TA;+typedef float TB;+typedef float TC;++static inline V zero() {+ return _mm512_setzero_ps();+}++static inline V load(const float *p) {+ return _mm512_loadu_ps(p);+}+#include "sgemmer.inc"bool llamafile_sgemm_sss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,diff --git a/llamafile/sgemm_sss_fma.cpp b/llamafile/sgemm_sss_fma.cppindex f018bfe6..1e925b80 100644--- a/llamafile/sgemm_sss_fma.cpp+++ b/llamafile/sgemm_sss_fma.cpp@@ -18,15 +18,23 @@#ifdef __x86_64__#include "sgemm.h"+#include <immintrin.h>#define KN 8-#define V __m256-#define TA float-#define TB float-#define TC float-#define zero() _mm256_setzero_ps()-#define load_a(p) _mm256_loadu_ps(p)-#define load_b(p) _mm256_loadu_ps(p)++typedef __m256 V;+typedef float TA;+typedef float TB;+typedef float TC;++static inline V zero() {+ return _mm256_setzero_ps();+}++static inline V load(const float *p) {+ return _mm256_loadu_ps(p);+}+#include "sgemmer.inc"bool llamafile_sgemm_sss_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,diff --git a/llamafile/sgemm_sss_neon.cpp b/llamafile/sgemm_sss_neon.cppnew file mode 100644index 00000000..16f7f0a3--- /dev/null+++ b/llamafile/sgemm_sss_neon.cpp@@ -0,0 +1,49 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __aarch64__++#include "sgemm.h"+#include <arm_neon.h>++#define KN 4++typedef float32x4_t V;+typedef float TA;+typedef float TB;+typedef float TC;++static inline V zero() {+ return vdupq_n_f32(0.f);+}++static inline V load(const float *p) {+ return vld1q_f32(p);+}++#include "sgemmer.inc"++bool llamafile_sgemm_sss_neon(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __aarch64__diff --git a/llamafile/sgemm_test.cpp b/llamafile/sgemm_test.cppindex e633f725..0553f650 100644--- a/llamafile/sgemm_test.cpp+++ b/llamafile/sgemm_test.cpp@@ -15,24 +15,22 @@// See the License for the specific language governing permissions and// limitations under the License.-#ifdef __x86_64__-#include "gemm.h"#include "llama.cpp/ggml.h"#include "llamafile.h"-int rand32(void) {+static int rand32(void) {static unsigned long long lcg = 1;lcg *= 6364136223846793005;lcg += 1442695040888963407;return lcg >> 32;}-float float01(unsigned x) { // (0,1)+static float float01(unsigned x) { // (0,1)return 1.f / 8388608 * ((x >> 9) + .5f);}-float numba(void) { // (-1,1)+static float numba(void) { // (-1,1)return float01(rand32()) * 2 - 1;}@@ -48,7 +46,7 @@ template <typename T> void randomize(int m, int n, T *A, int lda) {}int main(int argc, char *argv[]) {- float tolerance = 1e-5;+ float tolerance = 2e-5;int m = 510;int n = 513;int k = 512;@@ -66,28 +64,38 @@ int main(int argc, char *argv[]) {broadcast(G, ldc * n, NAN);randomize(k, m, A, lda);randomize(k, n, B, ldb);+gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc);if (!llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32))return 1;- for (int i = 0; i < m; ++i) {++ for (int i = 0; i < m; ++i)for (int j = 0; j < n; ++j) {auto g = G[ldc * j + i];auto c = C[ldc * j + i];- if (isnan(g))+ if (isnan(g)) {+ fprintf(stderr, "%s:%d: found nan in reference matrix: i=%d j=%d\n", __FILE__,+ __LINE__, i, j);return 3;- if (isnan(c))+ }+ if (isnan(c)) {+ fprintf(stderr, "%s:%d: found nan in output matrix: i=%d j=%d\n", __FILE__,+ __LINE__, i, j);return 4;- auto diff = g - c;- if (diff < 0)- diff = -diff;- if (diff > tolerance)+ }+ auto d = g - c;+ if (d < 0)+ d = -d;+ if (d > tolerance) {+ fprintf(stderr, "%s:%d: difference exceeds tolerance: %g vs. %g (%g) i=%d j=%d\n",+ __FILE__, __LINE__, g, c, d, i, j);return 5;+ }}- }-}-#else-int main(int argc, char *argv[]) {+ delete[] G;+ delete[] C;+ delete[] B;+ delete[] A;}-#endif // __x86_64__diff --git a/llamafile/sgemmer.inc b/llamafile/sgemmer.incindex 54b9e5e5..8ee7da03 100644--- a/llamafile/sgemmer.inc+++ b/llamafile/sgemmer.inc@@ -94,36 +94,36 @@ class SGEMMER {V c43 = zero();V c44 = zero();for (int l = 0; l < k; l += KN) {- V k0 = load_b(B + ldb * (j + 0) + l);- V k1 = load_b(B + ldb * (j + 1) + l);- V k2 = load_b(B + ldb * (j + 2) + l);- V k3 = load_b(B + ldb * (j + 3) + l);- V k4 = load_b(B + ldb * (j + 4) + l);- V a0 = load_a(A + lda * (i + 0) + l);+ V k0 = load(B + ldb * (j + 0) + l);+ V k1 = load(B + ldb * (j + 1) + l);+ V k2 = load(B + ldb * (j + 2) + l);+ V k3 = load(B + ldb * (j + 3) + l);+ V k4 = load(B + ldb * (j + 4) + l);+ V a0 = load(A + lda * (i + 0) + l);c00 = madd(a0, k0, c00);c01 = madd(a0, k1, c01);c02 = madd(a0, k2, c02);c03 = madd(a0, k3, c03);c04 = madd(a0, k4, c04);- V a1 = load_a(A + lda * (i + 1) + l);+ V a1 = load(A + lda * (i + 1) + l);c10 = madd(a1, k0, c10);c11 = madd(a1, k1, c11);c12 = madd(a1, k2, c12);c13 = madd(a1, k3, c13);c14 = madd(a1, k4, c14);- V a2 = load_a(A + lda * (i + 2) + l);+ V a2 = load(A + lda * (i + 2) + l);c20 = madd(a2, k0, c20);c21 = madd(a2, k1, c21);c22 = madd(a2, k2, c22);c23 = madd(a2, k3, c23);c24 = madd(a2, k4, c24);- V a3 = load_a(A + lda * (i + 3) + l);+ V a3 = load(A + lda * (i + 3) + l);c30 = madd(a3, k0, c30);c31 = madd(a3, k1, c31);c32 = madd(a3, k2, c32);c33 = madd(a3, k3, c33);c34 = madd(a3, k4, c34);- V a4 = load_a(A + lda * (i + 4) + l);+ V a4 = load(A + lda * (i + 4) + l);c40 = madd(a4, k0, c40);c41 = madd(a4, k1, c41);c42 = madd(a4, k2, c42);@@ -158,6 +158,82 @@ class SGEMMER {END_KERNEL()}+ dontinline void gemm5x4(int m0, int m, int n0, int n) {+ BEGIN_KERNEL(5, 4)+ V c00 = zero();+ V c01 = zero();+ V c02 = zero();+ V c03 = zero();+ V c10 = zero();+ V c11 = zero();+ V c12 = zero();+ V c13 = zero();+ V c20 = zero();+ V c21 = zero();+ V c22 = zero();+ V c23 = zero();+ V c30 = zero();+ V c31 = zero();+ V c32 = zero();+ V c33 = zero();+ V c40 = zero();+ V c41 = zero();+ V c42 = zero();+ V c43 = zero();+ for (int l = 0; l < k; l += KN) {+ V k0 = load(B + ldb * (j + 0) + l);+ V k1 = load(B + ldb * (j + 1) + l);+ V k2 = load(B + ldb * (j + 2) + l);+ V k3 = load(B + ldb * (j + 3) + l);+ V a0 = load(A + lda * (i + 0) + l);+ c00 = madd(a0, k0, c00);+ c01 = madd(a0, k1, c01);+ c02 = madd(a0, k2, c02);+ c03 = madd(a0, k3, c03);+ V a1 = load(A + lda * (i + 1) + l);+ c10 = madd(a1, k0, c10);+ c11 = madd(a1, k1, c11);+ c12 = madd(a1, k2, c12);+ c13 = madd(a1, k3, c13);+ V a2 = load(A + lda * (i + 2) + l);+ c20 = madd(a2, k0, c20);+ c21 = madd(a2, k1, c21);+ c22 = madd(a2, k2, c22);+ c23 = madd(a2, k3, c23);+ V a3 = load(A + lda * (i + 3) + l);+ c30 = madd(a3, k0, c30);+ c31 = madd(a3, k1, c31);+ c32 = madd(a3, k2, c32);+ c33 = madd(a3, k3, c33);+ V a4 = load(A + lda * (i + 4) + l);+ c40 = madd(a4, k0, c40);+ c41 = madd(a4, k1, c41);+ c42 = madd(a4, k2, c42);+ c43 = madd(a4, k3, c43);+ }+ C[ldc * (j + 0) + (i + 0)] = hsum(c00);+ C[ldc * (j + 0) + (i + 1)] = hsum(c10);+ C[ldc * (j + 0) + (i + 2)] = hsum(c20);+ C[ldc * (j + 0) + (i + 3)] = hsum(c30);+ C[ldc * (j + 0) + (i + 4)] = hsum(c40);+ C[ldc * (j + 1) + (i + 0)] = hsum(c01);+ C[ldc * (j + 1) + (i + 1)] = hsum(c11);+ C[ldc * (j + 1) + (i + 2)] = hsum(c21);+ C[ldc * (j + 1) + (i + 3)] = hsum(c31);+ C[ldc * (j + 1) + (i + 4)] = hsum(c41);+ C[ldc * (j + 2) + (i + 0)] = hsum(c02);+ C[ldc * (j + 2) + (i + 1)] = hsum(c12);+ C[ldc * (j + 2) + (i + 2)] = hsum(c22);+ C[ldc * (j + 2) + (i + 3)] = hsum(c32);+ C[ldc * (j + 2) + (i + 4)] = hsum(c42);+ C[ldc * (j + 3) + (i + 0)] = hsum(c03);+ C[ldc * (j + 3) + (i + 1)] = hsum(c13);+ C[ldc * (j + 3) + (i + 2)] = hsum(c23);+ C[ldc * (j + 3) + (i + 3)] = hsum(c33);+ C[ldc * (j + 3) + (i + 4)] = hsum(c43);+ END_KERNEL()+ }+dontinline void gemm3x4(int m0, int m, int n0, int n) {BEGIN_KERNEL(3, 4)V c00 = zero();@@ -173,21 +249,21 @@ class SGEMMER {V c22 = zero();V c23 = zero();for (int l = 0; l < k; l += KN) {- V k0 = load_b(B + ldb * (j + 0) + l);- V k1 = load_b(B + ldb * (j + 1) + l);- V k2 = load_b(B + ldb * (j + 2) + l);- V k3 = load_b(B + ldb * (j + 3) + l);- V a0 = load_a(A + lda * (i + 0) + l);+ V k0 = load(B + ldb * (j + 0) + l);+ V k1 = load(B + ldb * (j + 1) + l);+ V k2 = load(B + ldb * (j + 2) + l);+ V k3 = load(B + ldb * (j + 3) + l);+ V a0 = load(A + lda * (i + 0) + l);c00 = madd(a0, k0, c00);c01 = madd(a0, k1, c01);c02 = madd(a0, k2, c02);c03 = madd(a0, k3, c03);- V a1 = load_a(A + lda * (i + 1) + l);+ V a1 = load(A + lda * (i + 1) + l);c10 = madd(a1, k0, c10);c11 = madd(a1, k1, c11);c12 = madd(a1, k2, c12);c13 = madd(a1, k3, c13);- V a2 = load_a(A + lda * (i + 2) + l);+ V a2 = load(A + lda * (i + 2) + l);c20 = madd(a2, k0, c20);c21 = madd(a2, k1, c21);c22 = madd(a2, k2, c22);@@ -215,11 +291,11 @@ class SGEMMER {V c02 = zero();V c03 = zero();for (int l = 0; l < k; l += KN) {- V a0 = load_a(A + lda * (i + 0) + l);- V k0 = load_b(B + ldb * (j + 0) + l);- V k1 = load_b(B + ldb * (j + 1) + l);- V k2 = load_b(B + ldb * (j + 2) + l);- V k3 = load_b(B + ldb * (j + 3) + l);+ V a0 = load(A + lda * (i + 0) + l);+ V k0 = load(B + ldb * (j + 0) + l);+ V k1 = load(B + ldb * (j + 1) + l);+ V k2 = load(B + ldb * (j + 2) + l);+ V k3 = load(B + ldb * (j + 3) + l);c00 = madd(a0, k0, c00);c01 = madd(a0, k1, c01);c02 = madd(a0, k2, c02);@@ -239,14 +315,14 @@ class SGEMMER {V c20 = zero();V c30 = zero();for (int l = 0; l < k; l += KN) {- V k0 = load_b(B + ldb * (j + 0) + l);- V a0 = load_a(A + lda * (i + 0) + l);+ V k0 = load(B + ldb * (j + 0) + l);+ V a0 = load(A + lda * (i + 0) + l);c00 = madd(a0, k0, c00);- V a1 = load_a(A + lda * (i + 1) + l);+ V a1 = load(A + lda * (i + 1) + l);c10 = madd(a1, k0, c10);- V a2 = load_a(A + lda * (i + 2) + l);+ V a2 = load(A + lda * (i + 2) + l);c20 = madd(a2, k0, c20);- V a3 = load_a(A + lda * (i + 3) + l);+ V a3 = load(A + lda * (i + 3) + l);c30 = madd(a3, k0, c30);}C[ldc * (j + 0) + (i + 0)] = hsum(c00);@@ -260,7 +336,7 @@ class SGEMMER {BEGIN_KERNEL(1, 1)V c = zero();for (int l = 0; l < k; l += KN)- c = madd(load_a(A + lda * i + l), load_b(B + ldb * j + l), c);+ c = madd(load(A + lda * i + l), load(B + ldb * j + l), c);C[ldc * j + i] = hsum(c);END_KERNEL()}diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cuindex f988f2ad..ef4dbc74 100644--- a/llamafile/tinyblas.cu+++ b/llamafile/tinyblas.cu@@ -867,11 +867,11 @@ static tinyblasStatus_t tinyblasGSBE_launch(tinyblasHandle_t handle, tinyblasOpe* @param k is cols in `A` and rows in `B`* @param alpha points to scalar that's multiplied against input* @param A is input array of first matrices- * @param Atype is data type of `C`+ * @param Atype is data type of `A`* @param lda is row stride of `A`* @param strideA is distance between matrices in `A`* @param B is input array of second matrices- * @param Btype is data type of `C`+ * @param Btype is data type of `B`* @param ldb is row stride of `B`* @param strideB is distance between matrices in `B`* @param beta points to scalar that's multiplied against the existing