Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Refactor new hardware detection code
Justine Tunney32cd7cdea8fc
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 2409004f..8e136974 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -43,6 +43,7 @@ o/$(MODE)/llamafile: \################################################################################# microarchitectures+o/$(MODE)/llamafile/sgemm.o: private CXXFLAGS += -Oso/$(MODE)/llamafile/tinyblas_cpu_amd_fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mfmao/$(MODE)/llamafile/tinyblas_cpu_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mavx2 -Xx86_64-mfmao/$(MODE)/llamafile/tinyblas_cpu_amd_avxvnni.o: private TARGET_ARCH += -Xx86_64-mtune=alderlake -Xx86_64-mavx2 -Xx86_64-mfma -Xx86_64-mavxvnnidiff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppindex b34cbe82..3d8b5022 100644--- a/llamafile/sgemm.cpp+++ b/llamafile/sgemm.cpp@@ -23,7 +23,70 @@#include <libc/sysv/consts/hwcap.h>#include <sys/auxv.h>-static const long hwcap = getauxval(AT_HWCAP);+static const struct GemmFuncs {+ typeof(llamafile_sgemm) *sgemm;+ typeof(llamafile_mixmul) *mixmul;+ GemmFuncs() {+#ifdef __x86_64__+ if (X86_HAVE(AVX)) {+ if (X86_HAVE(FMA)) {+ if (X86_HAVE(AVX2)) {+ if (X86_HAVE(AVX512F)) {+ if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && X86_HAVE(AVX512_BF16)) {+ // AMD Zen4+ (2023-)+ sgemm = llamafile_sgemm_amd_zen4;+ mixmul = llamafile_mixmul_amd_zen4;+ } else {+ // Intel Xeon Skylake+ (2015-)+ sgemm = llamafile_sgemm_amd_avx512f;+ mixmul = llamafile_mixmul_amd_avx512f;+ }+ } else if (X86_HAVE(AVXVNNI)) {+ // Intel Alderlake (2021-)+ sgemm = llamafile_sgemm_amd_avxvnni;+ mixmul = llamafile_mixmul_amd_avxvnni;+ } else {+ // Intel Haswell/Broadwell/Skylake (2013-2020)+ // AMD Excavator (2015-2022)+ sgemm = llamafile_sgemm_amd_avx2;+ mixmul = llamafile_mixmul_amd_avx2;+ }+ } else {+ // AMD Piledriver (2011-2014)+ sgemm = llamafile_sgemm_amd_fma;+ mixmul = llamafile_mixmul_amd_fma;+ }+ } else {+ // Intel Sandybridge/Ivybridge (2010-2012)+ // AMD Bulldozer (2011)+ sgemm = llamafile_sgemm_amd_avx;+ mixmul = llamafile_mixmul_amd_avx;+ }+ } else {+ // AMD K8/Barcelona (2003-2010)+ // Intel Core/Nehalem (2006-2009)+ sgemm = llamafile_sgemm_unsupported;+ mixmul = llamafile_mixmul_unsupported;+ }+#elif defined(__aarch64__)+ long hwcap = getauxval(AT_HWCAP);+ if ((hwcap & HWCAP_FPHP) && // fp16 scalar isa (ID_AA64PFR0_EL1.FP == 1)+ (hwcap & HWCAP_ASIMDHP) && // fp16 vector isa (ID_AA64PFR0_EL1.AdvSIMD == 1)+ (hwcap & HWCAP_ASIMDDP)) { // dotprod isa (ID_AA64ISAR0_EL1.DP == 1)+ // e.g. Apple M1, Raspberry Pi 5+ sgemm = llamafile_sgemm_arm82;+ mixmul = llamafile_mixmul_arm82;+ } else {+ // ARM64 baseline ISA+ sgemm = llamafile_sgemm_arm80;+ mixmul = llamafile_mixmul_arm80;+ }+#else+ sgemm = llamafile_sgemm_unsupported;+ mixmul = llamafile_mixmul_unsupported;+#endif+ }+} funcs;/*** Performs optimized matrix multiplication on CPU.@@ -52,59 +115,7 @@ static const long hwcap = getauxval(AT_HWCAP);*/bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B, int ldb, void *C,int ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype) {-#ifdef __x86_64__- if (X86_HAVE(AVX)) {- if (X86_HAVE(FMA)) {- if (X86_HAVE(AVX2)) {- if (X86_HAVE(AVX512F)) {- if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && X86_HAVE(AVX512_BF16)) {- // AMD Zen4+ (2023-)- return llamafile_sgemm_amd_zen4(m, n, k, A, lda, B, ldb, C, ldc, ith, nth,- task, Atype, Btype, Ctype);- } else {- // Intel Xeon Skylake+ (2015-)- return llamafile_sgemm_amd_avx512f(m, n, k, A, lda, B, ldb, C, ldc, ith,- nth, task, Atype, Btype, Ctype);- }- } else if (X86_HAVE(AVXVNNI)) {- // Intel Alderlake (2021-)- return llamafile_sgemm_amd_avxvnni(m, n, k, A, lda, B, ldb, C, ldc, ith, nth,- task, Atype, Btype, Ctype);- } else {- // Intel Haswell/Broadwell/Skylake (2013-2020)- // AMD Excavator (2015-2022)- return llamafile_sgemm_amd_avx2(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task,- Atype, Btype, Ctype);- }- } else {- // AMD Piledriver (2011-2014)- return llamafile_sgemm_amd_fma(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task,- Atype, Btype, Ctype);- }- } else {- // Intel Sandybridge/Ivybridge (2010-2012)- // AMD Bulldozer (2011)- return llamafile_sgemm_amd_avx(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype,- Btype, Ctype);- }- } else {- // AMD K8/Barcelona (2003-2010)- // Intel Core/Nehalem (2006-2009)- return false;- }-#else- if ((hwcap & HWCAP_FPHP) && // fp16 scalar isa (ID_AA64PFR0_EL1.FP == 1)- (hwcap & HWCAP_ASIMDHP) && // fp16 vector isa (ID_AA64PFR0_EL1.AdvSIMD == 1)- (hwcap & HWCAP_ASIMDDP)) { // dotprod isa (ID_AA64ISAR0_EL1.DP == 1)- // e.g. Apple M1, Raspberry Pi 5- return llamafile_sgemm_arm82(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,- Ctype);- } else {- // ARM64 baseline ISA- return llamafile_sgemm_arm80(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,- Ctype);- }-#endif+ return funcs.sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype, Ctype);}/**@@ -112,37 +123,5 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,*/bool llamafile_mixmul(const ggml_compute_params *params, const ggml_tensor *weights,const ggml_tensor *thought, const ggml_tensor *plan, ggml_tensor *result) {-#ifdef __x86_64__- if (X86_HAVE(AVX)) {- if (X86_HAVE(FMA)) {- if (X86_HAVE(AVX2)) {- if (X86_HAVE(AVX512F)) {- if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && X86_HAVE(AVX512_BF16)) {- return llamafile_mixmul_amd_zen4(params, weights, thought, plan, result);- } else {- return llamafile_mixmul_amd_avx512f(params, weights, thought, plan, result);- }- } else if (X86_HAVE(AVXVNNI)) {- return llamafile_mixmul_amd_avxvnni(params, weights, thought, plan, result);- } else {- return llamafile_mixmul_amd_avx2(params, weights, thought, plan, result);- }- } else {- return llamafile_mixmul_amd_fma(params, weights, thought, plan, result);- }- } else {- return llamafile_mixmul_amd_avx(params, weights, thought, plan, result);- }- } else {- return false;- }-#elif defined(__aarch64__)- if ((hwcap & HWCAP_FPHP) && // fp16 scalar isa (ID_AA64PFR0_EL1.FP == 1)- (hwcap & HWCAP_ASIMDHP) && // fp16 vector isa (ID_AA64PFR0_EL1.AdvSIMD == 1)- (hwcap & HWCAP_ASIMDDP)) { // dotprod isa (ID_AA64ISAR0_EL1.DP == 1)- return llamafile_mixmul_arm82(params, weights, thought, plan, result);- } else {- return llamafile_mixmul_arm80(params, weights, thought, plan, result);- }-#endif+ return funcs.mixmul(params, weights, thought, plan, result);}diff --git a/llamafile/sgemm.h b/llamafile/sgemm.hindex f384f890..4ff2e0c8 100644--- a/llamafile/sgemm.h+++ b/llamafile/sgemm.h@@ -1,4 +1,5 @@#pragma once+#include <stdbool.h>#ifdef __cplusplusextern "C" {#endif@@ -6,6 +7,8 @@ extern "C" {struct ggml_tensor;struct ggml_compute_params;+bool llamafile_sgemm_unsupported(int, int, int, const void *, int, const void *, int, void *, int,+ int, int, int, int, int, int);bool llamafile_sgemm_amd_avx(int, int, int, const void *, int, const void *, int, void *, int, int,int, int, int, int, int);bool llamafile_sgemm_amd_fma(int, int, int, const void *, int, const void *, int, void *, int, int,@@ -23,6 +26,9 @@ bool llamafile_sgemm_arm80(int, int, int, const void *, int, const void *, int,bool llamafile_sgemm_arm82(int, int, int, const void *, int, const void *, int, void *, int, int,int, int, int, int, int);+bool llamafile_mixmul_unsupported(const struct ggml_compute_params *, const struct ggml_tensor *,+ const struct ggml_tensor *, const struct ggml_tensor *,+ struct ggml_tensor *);bool llamafile_mixmul_amd_avx(const struct ggml_compute_params *, const struct ggml_tensor *,const struct ggml_tensor *, const struct ggml_tensor *,struct ggml_tensor *);diff --git a/llamafile/sgemm_sss_test.cpp b/llamafile/sgemm_sss_test.cppnew file mode 100644index 00000000..3825989c--- /dev/null+++ b/llamafile/sgemm_sss_test.cpp@@ -0,0 +1,80 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "bench.h"+#include "float.h"+#include "gemm.h"+#include "llama.cpp/ggml.h"+#include "llamafile.h"+#include "macros.h"+#include "numba.h"++#define ITERATIONS 5+#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))++int main(int argc, char *argv[]) {+ float tolerance = 2e-5;+ int m = 510;+ int n = 513;+ int k = 512;+ int lda = ROUNDUP(k, 16);+ int ldb = ROUNDUP(k, 16);+ int ldc = ROUNDUP(m, 16);+ float *A = ALLOC(lda * m);+ float *B = ALLOC(ldb * n);+ float *C = ALLOC(ldc * n);+ float *G = ALLOC(ldc * n);+ broadcast(A, lda * m, NAN);+ broadcast(B, ldb * n, NAN);+ broadcast(C, ldc * n, NAN);+ broadcast(G, ldc * n, NAN);+ randomize(k, m, A, lda);+ randomize(k, n, B, ldb);++ BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));+ BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,+ GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));++ for (int i = 0; i < m; ++i)+ for (int j = 0; j < n; ++j) {+ auto g = G[ldc * j + i];+ auto c = C[ldc * j + i];+ if (flt::isnan(g)) {+ fprintf(stderr, "%s:%d: found nan in reference matrix: i=%d j=%d\n", __FILE__,+ __LINE__, i, j);+ return 3;+ }+ if (flt::isnan(c)) {+ fprintf(stderr, "%s:%d: found nan in output matrix: i=%d j=%d\n", __FILE__,+ __LINE__, i, j);+ return 4;+ }+ auto d = g - c;+ if (d < 0)+ d = -d;+ if (d > tolerance) {+ fprintf(stderr, "%s:%d: difference exceeds tolerance: %g vs. %g (%g) i=%d j=%d\n",+ __FILE__, __LINE__, g, c, d, i, j);+ return 5;+ }+ }++ free(G);+ free(C);+ free(B);+ free(A);+}diff --git a/llamafile/sgemm_vecdot_test.cpp b/llamafile/sgemm_vecdot_test.cppindex 78c7587b..283973ad 100644--- a/llamafile/sgemm_vecdot_test.cpp+++ b/llamafile/sgemm_vecdot_test.cpp@@ -20,22 +20,23 @@#include "gemm.h"#include "llama.cpp/ggml.h"#include "llamafile.h"+#include "macros.h"#include "numba.h"#define ITERATIONS 30+#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))int main(int argc, char *argv[]) {int m = 1025;int n = 1;int k = 32768;- int l = 3;- int lda = k + l;- int ldb = k + l;- int ldc = m + l;- float *A = new float[lda * m];- float *B = new float[ldb * n];- float *C = new float[ldc * n];- float *G = new float[ldc * n];+ int lda = ROUNDUP(k, 16);+ int ldb = ROUNDUP(k, 16);+ int ldc = ROUNDUP(m, 16);+ float *A = ALLOC(lda * m);+ float *B = ALLOC(ldb * n);+ float *C = ALLOC(ldc * n);+ float *G = ALLOC(ldc * n);broadcast(A, lda * m, NAN);broadcast(B, ldb * n, NAN);broadcast(C, ldc * n, NAN);@@ -98,8 +99,8 @@ int main(int argc, char *argv[]) {if (err_worst > 134)return 6;- delete[] G;- delete[] C;- delete[] B;- delete[] A;+ free(G);+ free(C);+ free(B);+ free(A);}diff --git a/llamafile/tinyblas_cpu_unsupported.cpp b/llamafile/tinyblas_cpu_unsupported.cppnew file mode 100644index 00000000..5510e15c--- /dev/null+++ b/llamafile/tinyblas_cpu_unsupported.cpp@@ -0,0 +1,31 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "sgemm.h"++bool llamafile_sgemm_unsupported(int m, int n, int k, const void *A, int lda, const void *B,+ int ldb, void *C, int ldc, int ith, int nth, int task, int Atype,+ int Btype, int Ctype) {+ return false;+}++bool llamafile_mixmul_unsupported(const struct ggml_compute_params *params,+ const struct ggml_tensor *weights,+ const struct ggml_tensor *thought, const struct ggml_tensor *plan,+ struct ggml_tensor *result) {+ return false;+}