Skip to content

Distribute and run LLMs with a single file.

Refactor new hardware detection code

Justine Tunney32cd7cdea8fc
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 2409004f..8e136974 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -43,6 +43,7 @@ o/$(MODE)/llamafile: \
################################################################################
# microarchitectures
+o/$(MODE)/llamafile/sgemm.o: private CXXFLAGS += -Os
o/$(MODE)/llamafile/tinyblas_cpu_amd_fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mfma
o/$(MODE)/llamafile/tinyblas_cpu_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mavx2 -Xx86_64-mfma
o/$(MODE)/llamafile/tinyblas_cpu_amd_avxvnni.o: private TARGET_ARCH += -Xx86_64-mtune=alderlake -Xx86_64-mavx2 -Xx86_64-mfma -Xx86_64-mavxvnni
diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cpp
index b34cbe82..3d8b5022 100644
--- a/llamafile/sgemm.cpp
+++ b/llamafile/sgemm.cpp
@@ -23,7 +23,70 @@
#include <libc/sysv/consts/hwcap.h>
#include <sys/auxv.h>
-static const long hwcap = getauxval(AT_HWCAP);
+static const struct GemmFuncs {
+ typeof(llamafile_sgemm) *sgemm;
+ typeof(llamafile_mixmul) *mixmul;
+ GemmFuncs() {
+#ifdef __x86_64__
+ if (X86_HAVE(AVX)) {
+ if (X86_HAVE(FMA)) {
+ if (X86_HAVE(AVX2)) {
+ if (X86_HAVE(AVX512F)) {
+ if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && X86_HAVE(AVX512_BF16)) {
+ // AMD Zen4+ (2023-)
+ sgemm = llamafile_sgemm_amd_zen4;
+ mixmul = llamafile_mixmul_amd_zen4;
+ } else {
+ // Intel Xeon Skylake+ (2015-)
+ sgemm = llamafile_sgemm_amd_avx512f;
+ mixmul = llamafile_mixmul_amd_avx512f;
+ }
+ } else if (X86_HAVE(AVXVNNI)) {
+ // Intel Alderlake (2021-)
+ sgemm = llamafile_sgemm_amd_avxvnni;
+ mixmul = llamafile_mixmul_amd_avxvnni;
+ } else {
+ // Intel Haswell/Broadwell/Skylake (2013-2020)
+ // AMD Excavator (2015-2022)
+ sgemm = llamafile_sgemm_amd_avx2;
+ mixmul = llamafile_mixmul_amd_avx2;
+ }
+ } else {
+ // AMD Piledriver (2011-2014)
+ sgemm = llamafile_sgemm_amd_fma;
+ mixmul = llamafile_mixmul_amd_fma;
+ }
+ } else {
+ // Intel Sandybridge/Ivybridge (2010-2012)
+ // AMD Bulldozer (2011)
+ sgemm = llamafile_sgemm_amd_avx;
+ mixmul = llamafile_mixmul_amd_avx;
+ }
+ } else {
+ // AMD K8/Barcelona (2003-2010)
+ // Intel Core/Nehalem (2006-2009)
+ sgemm = llamafile_sgemm_unsupported;
+ mixmul = llamafile_mixmul_unsupported;
+ }
+#elif defined(__aarch64__)
+ long hwcap = getauxval(AT_HWCAP);
+ if ((hwcap & HWCAP_FPHP) && // fp16 scalar isa (ID_AA64PFR0_EL1.FP == 1)
+ (hwcap & HWCAP_ASIMDHP) && // fp16 vector isa (ID_AA64PFR0_EL1.AdvSIMD == 1)
+ (hwcap & HWCAP_ASIMDDP)) { // dotprod isa (ID_AA64ISAR0_EL1.DP == 1)
+ // e.g. Apple M1, Raspberry Pi 5
+ sgemm = llamafile_sgemm_arm82;
+ mixmul = llamafile_mixmul_arm82;
+ } else {
+ // ARM64 baseline ISA
+ sgemm = llamafile_sgemm_arm80;
+ mixmul = llamafile_mixmul_arm80;
+ }
+#else
+ sgemm = llamafile_sgemm_unsupported;
+ mixmul = llamafile_mixmul_unsupported;
+#endif
+ }
+} funcs;
/**
* Performs optimized matrix multiplication on CPU.
@@ -52,59 +115,7 @@ static const long hwcap = getauxval(AT_HWCAP);
*/
bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B, int ldb, void *C,
int ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype) {
-#ifdef __x86_64__
- if (X86_HAVE(AVX)) {
- if (X86_HAVE(FMA)) {
- if (X86_HAVE(AVX2)) {
- if (X86_HAVE(AVX512F)) {
- if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && X86_HAVE(AVX512_BF16)) {
- // AMD Zen4+ (2023-)
- return llamafile_sgemm_amd_zen4(m, n, k, A, lda, B, ldb, C, ldc, ith, nth,
- task, Atype, Btype, Ctype);
- } else {
- // Intel Xeon Skylake+ (2015-)
- return llamafile_sgemm_amd_avx512f(m, n, k, A, lda, B, ldb, C, ldc, ith,
- nth, task, Atype, Btype, Ctype);
- }
- } else if (X86_HAVE(AVXVNNI)) {
- // Intel Alderlake (2021-)
- return llamafile_sgemm_amd_avxvnni(m, n, k, A, lda, B, ldb, C, ldc, ith, nth,
- task, Atype, Btype, Ctype);
- } else {
- // Intel Haswell/Broadwell/Skylake (2013-2020)
- // AMD Excavator (2015-2022)
- return llamafile_sgemm_amd_avx2(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task,
- Atype, Btype, Ctype);
- }
- } else {
- // AMD Piledriver (2011-2014)
- return llamafile_sgemm_amd_fma(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task,
- Atype, Btype, Ctype);
- }
- } else {
- // Intel Sandybridge/Ivybridge (2010-2012)
- // AMD Bulldozer (2011)
- return llamafile_sgemm_amd_avx(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype,
- Btype, Ctype);
- }
- } else {
- // AMD K8/Barcelona (2003-2010)
- // Intel Core/Nehalem (2006-2009)
- return false;
- }
-#else
- if ((hwcap & HWCAP_FPHP) && // fp16 scalar isa (ID_AA64PFR0_EL1.FP == 1)
- (hwcap & HWCAP_ASIMDHP) && // fp16 vector isa (ID_AA64PFR0_EL1.AdvSIMD == 1)
- (hwcap & HWCAP_ASIMDDP)) { // dotprod isa (ID_AA64ISAR0_EL1.DP == 1)
- // e.g. Apple M1, Raspberry Pi 5
- return llamafile_sgemm_arm82(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,
- Ctype);
- } else {
- // ARM64 baseline ISA
- return llamafile_sgemm_arm80(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,
- Ctype);
- }
-#endif
+ return funcs.sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype, Ctype);
}
/**
@@ -112,37 +123,5 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,
*/
bool llamafile_mixmul(const ggml_compute_params *params, const ggml_tensor *weights,
const ggml_tensor *thought, const ggml_tensor *plan, ggml_tensor *result) {
-#ifdef __x86_64__
- if (X86_HAVE(AVX)) {
- if (X86_HAVE(FMA)) {
- if (X86_HAVE(AVX2)) {
- if (X86_HAVE(AVX512F)) {
- if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && X86_HAVE(AVX512_BF16)) {
- return llamafile_mixmul_amd_zen4(params, weights, thought, plan, result);
- } else {
- return llamafile_mixmul_amd_avx512f(params, weights, thought, plan, result);
- }
- } else if (X86_HAVE(AVXVNNI)) {
- return llamafile_mixmul_amd_avxvnni(params, weights, thought, plan, result);
- } else {
- return llamafile_mixmul_amd_avx2(params, weights, thought, plan, result);
- }
- } else {
- return llamafile_mixmul_amd_fma(params, weights, thought, plan, result);
- }
- } else {
- return llamafile_mixmul_amd_avx(params, weights, thought, plan, result);
- }
- } else {
- return false;
- }
-#elif defined(__aarch64__)
- if ((hwcap & HWCAP_FPHP) && // fp16 scalar isa (ID_AA64PFR0_EL1.FP == 1)
- (hwcap & HWCAP_ASIMDHP) && // fp16 vector isa (ID_AA64PFR0_EL1.AdvSIMD == 1)
- (hwcap & HWCAP_ASIMDDP)) { // dotprod isa (ID_AA64ISAR0_EL1.DP == 1)
- return llamafile_mixmul_arm82(params, weights, thought, plan, result);
- } else {
- return llamafile_mixmul_arm80(params, weights, thought, plan, result);
- }
-#endif
+ return funcs.mixmul(params, weights, thought, plan, result);
}
diff --git a/llamafile/sgemm.h b/llamafile/sgemm.h
index f384f890..4ff2e0c8 100644
--- a/llamafile/sgemm.h
+++ b/llamafile/sgemm.h
@@ -1,4 +1,5 @@
#pragma once
+#include <stdbool.h>
#ifdef __cplusplus
extern "C" {
#endif
@@ -6,6 +7,8 @@ extern "C" {
struct ggml_tensor;
struct ggml_compute_params;
+bool llamafile_sgemm_unsupported(int, int, int, const void *, int, const void *, int, void *, int,
+ int, int, int, int, int, int);
bool llamafile_sgemm_amd_avx(int, int, int, const void *, int, const void *, int, void *, int, int,
int, int, int, int, int);
bool llamafile_sgemm_amd_fma(int, int, int, const void *, int, const void *, int, void *, int, int,
@@ -23,6 +26,9 @@ bool llamafile_sgemm_arm80(int, int, int, const void *, int, const void *, int,
bool llamafile_sgemm_arm82(int, int, int, const void *, int, const void *, int, void *, int, int,
int, int, int, int, int);
+bool llamafile_mixmul_unsupported(const struct ggml_compute_params *, const struct ggml_tensor *,
+ const struct ggml_tensor *, const struct ggml_tensor *,
+ struct ggml_tensor *);
bool llamafile_mixmul_amd_avx(const struct ggml_compute_params *, const struct ggml_tensor *,
const struct ggml_tensor *, const struct ggml_tensor *,
struct ggml_tensor *);
diff --git a/llamafile/sgemm_sss_test.cpp b/llamafile/sgemm_sss_test.cpp
new file mode 100644
index 00000000..3825989c
--- /dev/null
+++ b/llamafile/sgemm_sss_test.cpp
@@ -0,0 +1,80 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "bench.h"
+#include "float.h"
+#include "gemm.h"
+#include "llama.cpp/ggml.h"
+#include "llamafile.h"
+#include "macros.h"
+#include "numba.h"
+
+#define ITERATIONS 5
+#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))
+
+int main(int argc, char *argv[]) {
+ float tolerance = 2e-5;
+ int m = 510;
+ int n = 513;
+ int k = 512;
+ int lda = ROUNDUP(k, 16);
+ int ldb = ROUNDUP(k, 16);
+ int ldc = ROUNDUP(m, 16);
+ float *A = ALLOC(lda * m);
+ float *B = ALLOC(ldb * n);
+ float *C = ALLOC(ldc * n);
+ float *G = ALLOC(ldc * n);
+ broadcast(A, lda * m, NAN);
+ broadcast(B, ldb * n, NAN);
+ broadcast(C, ldc * n, NAN);
+ broadcast(G, ldc * n, NAN);
+ randomize(k, m, A, lda);
+ randomize(k, n, B, ldb);
+
+ BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));
+ BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,
+ GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));
+
+ for (int i = 0; i < m; ++i)
+ for (int j = 0; j < n; ++j) {
+ auto g = G[ldc * j + i];
+ auto c = C[ldc * j + i];
+ if (flt::isnan(g)) {
+ fprintf(stderr, "%s:%d: found nan in reference matrix: i=%d j=%d\n", __FILE__,
+ __LINE__, i, j);
+ return 3;
+ }
+ if (flt::isnan(c)) {
+ fprintf(stderr, "%s:%d: found nan in output matrix: i=%d j=%d\n", __FILE__,
+ __LINE__, i, j);
+ return 4;
+ }
+ auto d = g - c;
+ if (d < 0)
+ d = -d;
+ if (d > tolerance) {
+ fprintf(stderr, "%s:%d: difference exceeds tolerance: %g vs. %g (%g) i=%d j=%d\n",
+ __FILE__, __LINE__, g, c, d, i, j);
+ return 5;
+ }
+ }
+
+ free(G);
+ free(C);
+ free(B);
+ free(A);
+}
diff --git a/llamafile/sgemm_vecdot_test.cpp b/llamafile/sgemm_vecdot_test.cpp
index 78c7587b..283973ad 100644
--- a/llamafile/sgemm_vecdot_test.cpp
+++ b/llamafile/sgemm_vecdot_test.cpp
@@ -20,22 +20,23 @@
#include "gemm.h"
#include "llama.cpp/ggml.h"
#include "llamafile.h"
+#include "macros.h"
#include "numba.h"
#define ITERATIONS 30
+#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))
int main(int argc, char *argv[]) {
int m = 1025;
int n = 1;
int k = 32768;
- int l = 3;
- int lda = k + l;
- int ldb = k + l;
- int ldc = m + l;
- float *A = new float[lda * m];
- float *B = new float[ldb * n];
- float *C = new float[ldc * n];
- float *G = new float[ldc * n];
+ int lda = ROUNDUP(k, 16);
+ int ldb = ROUNDUP(k, 16);
+ int ldc = ROUNDUP(m, 16);
+ float *A = ALLOC(lda * m);
+ float *B = ALLOC(ldb * n);
+ float *C = ALLOC(ldc * n);
+ float *G = ALLOC(ldc * n);
broadcast(A, lda * m, NAN);
broadcast(B, ldb * n, NAN);
broadcast(C, ldc * n, NAN);
@@ -98,8 +99,8 @@ int main(int argc, char *argv[]) {
if (err_worst > 134)
return 6;
- delete[] G;
- delete[] C;
- delete[] B;
- delete[] A;
+ free(G);
+ free(C);
+ free(B);
+ free(A);
}
diff --git a/llamafile/tinyblas_cpu_unsupported.cpp b/llamafile/tinyblas_cpu_unsupported.cpp
new file mode 100644
index 00000000..5510e15c
--- /dev/null
+++ b/llamafile/tinyblas_cpu_unsupported.cpp
@@ -0,0 +1,31 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "sgemm.h"
+
+bool llamafile_sgemm_unsupported(int m, int n, int k, const void *A, int lda, const void *B,
+ int ldb, void *C, int ldc, int ith, int nth, int task, int Atype,
+ int Btype, int Ctype) {
+ return false;
+}
+
+bool llamafile_mixmul_unsupported(const struct ggml_compute_params *params,
+ const struct ggml_tensor *weights,
+ const struct ggml_tensor *thought, const struct ggml_tensor *plan,
+ struct ggml_tensor *result) {
+ return false;
+}