Skip to content

Distribute and run LLMs with a single file.

Restore Alderlake performance

Justine Tunney38cf08ff4fb5
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cpp
index 99897db0..c1793a2f 100644
--- a/llama.cpp/common.cpp
+++ b/llama.cpp/common.cpp
@@ -22,6 +22,7 @@
#include <vector>
#include <cinttypes>
#include <codecvt>
+#include <cosmo.h>
#if defined(__APPLE__) && defined(__MACH__)
#include <sys/types.h>
@@ -76,7 +77,7 @@
using json = nlohmann::ordered_json;
int32_t get_num_physical_cores() {
-#ifdef __linux__
+ if (IsLinux()) {
// enumerate the set of thread siblings, num entries is num cores
std::unordered_set<std::string> siblings;
for (uint32_t cpu=0; cpu < UINT32_MAX; ++cpu) {
@@ -93,25 +94,12 @@ int32_t get_num_physical_cores() {
if (!siblings.empty()) {
return static_cast<int32_t>(siblings.size());
}
-#elif defined(__APPLE__) && defined(__MACH__)
- int32_t num_physical_cores;
- size_t len = sizeof(num_physical_cores);
- int result = sysctlbyname("hw.perflevel0.physicalcpu", &num_physical_cores, &len, NULL, 0);
- if (result == 0) {
- return num_physical_cores;
- }
- result = sysctlbyname("hw.physicalcpu", &num_physical_cores, &len, NULL, 0);
- if (result == 0) {
- return num_physical_cores;
}
-#elif defined(_WIN32)
- //TODO: Implement
-#endif
unsigned int n_threads = std::thread::hardware_concurrency();
return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4;
}
-#if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__)
+#if defined(__x86_64__) && (defined(__linux__) || defined(__COSMOPOLITAN__)) && !defined(__ANDROID__)
#include <pthread.h>
static void cpuid(unsigned leaf, unsigned subleaf,
@@ -165,7 +153,7 @@ static int count_math_cpus(int cpu_count) {
* Returns number of CPUs on system that are useful for math.
*/
int get_math_cpu_count() {
-#if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__)
+#if defined(__x86_64__) && (defined(__linux__) || defined(__COSMOPOLITAN__)) && !defined(__ANDROID__)
int cpu_count = sysconf(_SC_NPROCESSORS_ONLN);
if (cpu_count < 1) {
return get_num_physical_cores();
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index a7482051..8dcf2363 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -42,13 +42,54 @@ o/$(MODE)/llamafile: \
################################################################################
# microarchitectures
+#
+#### Intel CPU Line
+#
+# - 2006 core X64 SSE4.1 (only on 45nm variety) (-march=core2)
+# - 2008 nehalem SSE4.2 VT-x VT-d RDTSCP POPCNT (-march=nehalem)
+# - 2010 westmere CLMUL AES (-march=westmere)
+# - 2011 sandybridge AVX TXT (-march=sandybridge)
+# - 2012 ivybridge F16C MOVBE FSGSBASE (-march=ivybridge)
+# - 2013 haswell AVX2 TSX BMI1 BMI2 FMA (-march=haswell)
+# - 2014 broadwell RDSEED ADX PREFETCHW (-march=broadwell)
+# - 2015 skylake SGX ADX MPX AVX-512[xeon-only] (-march=skylake / -march=skylake-avx512)
+# - 2018 cannonlake SHA (-march=cannonlake)
+# - 2019 cascadelake VNNI
+# - 2021 alderlake efficiency cores
+#
+#### AMD CPU Line
+#
+# - 2003 k8 SSE SSE2 (-march=k8)
+# - 2005 k8 (Venus) SSE3 (-march=k8-sse3)
+# - 2008 barcelona SSE4a?! (-march=barcelona)
+# - 2011 bulldozer SSSE3 SSE4.1 SSE4.2 CLMUL AVX AES FMA4?! (-march=bdver1)
+# - 2011 piledriver BMI1 FMA (-march=bdver2)
+# - 2015 excavator AVX2 BMI2 MOVBE (-march=bdver4)
+# - 2017 ryzen F16C SHA ADX (-march=znver1)
+# - 2023 zen4 AVX512F AVX512VL AVX512VNNI AVX512BF16 (-march=znver4)
+#
+#### ARM Revisions
+#
+# - armv8.0-a raspberry pi 4
+# - armv8.2-a raspberry pi 5
+# - armv8.5-a apple m1
+# - armv8.6-a apple m2
+#
+#### ARM Features
+#
+# - HWCAP_CRC32 +crc32 (e.g. m1, rpi4) __ARM_FEATRUE_CRC32
+# - HWCAP_FPHP +fp16 (e.g. m1, rpi5) __ARM_FEATURE_FP16_SCALAR_ARITHMETIC
+# - HWCAP_ASIMDHP +fp16 (e.g. m1, rpi5) __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+# - HWCAP_ASIMDDP +dotprod (e.g. m1, rpi5) __ARM_FEATURE_DOTPROD
+#
o/$(MODE)/llamafile/sgemm.o: private CXXFLAGS += -Os
-o/$(MODE)/llamafile/tinyblas_cpu_amd_fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mfma
-o/$(MODE)/llamafile/tinyblas_cpu_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mfma
-o/$(MODE)/llamafile/tinyblas_cpu_amd_avxvnni.o: private TARGET_ARCH += -Xx86_64-mtune=alderlake -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavxvnni
-o/$(MODE)/llamafile/tinyblas_cpu_amd_avx512f.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f
-o/$(MODE)/llamafile/tinyblas_cpu_amd_zen4.o: private TARGET_ARCH += -Xx86_64-mtune=znver4 -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512vl -Xx86_64-mavx512vnni -Xx86_64-mavx512bf16
+o/$(MODE)/llamafile/tinyblas_cpu_amd_avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge -Xx86_64-mf16c
+o/$(MODE)/llamafile/tinyblas_cpu_amd_fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mf16c -Xx86_64-mfma
+o/$(MODE)/llamafile/tinyblas_cpu_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mfma
+o/$(MODE)/llamafile/tinyblas_cpu_amd_avxvnni.o: private TARGET_ARCH += -Xx86_64-mtune=alderlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavxvnni
+o/$(MODE)/llamafile/tinyblas_cpu_amd_avx512f.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f
+o/$(MODE)/llamafile/tinyblas_cpu_amd_zen4.o: private TARGET_ARCH += -Xx86_64-mtune=znver4 -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512vl -Xx86_64-mavx512vnni -Xx86_64-mavx512bf16
o/$(MODE)/llamafile/tinyblas_cpu_arm82.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+dotprod+fp16
################################################################################
diff --git a/llamafile/tinyblas_cpu.inc b/llamafile/tinyblas_cpu.inc
index 1ff3225e..a25fb23a 100644
--- a/llamafile/tinyblas_cpu.inc
+++ b/llamafile/tinyblas_cpu.inc
@@ -42,6 +42,7 @@
#include "llama.cpp/ggml-quants.h"
#include "llamafile.h"
#include <algorithm>
+#include <cosmo.h>
#pragma GCC diagnostic ignored "-Wpedantic"
#pragma GCC diagnostic ignored "-Wignored-attributes"
@@ -1022,14 +1023,18 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,
tb.matmul(m, n, task);
return true;
#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)
- if (k % 8)
- return false;
- if (Btype != GGML_TYPE_F32)
+ if (X86_HAVE(F16C)) {
+ if (k % 8)
+ return false;
+ if (Btype != GGML_TYPE_F32)
+ return false;
+ tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, float> tb{
+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+ } else {
return false;
- tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, float> tb{
- k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
- return true;
+ }
#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
if (n < 8)
return false;
@@ -1295,9 +1300,13 @@ class MixMul {
tinyBLAS<NCB | NCC, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, float>,
ggml_fp16_t, ggml_fp16_t, float>();
#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)
- return mixmat<8, 1,
- tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float>,
- ggml_fp16_t, ggml_fp16_t, float>();
+ if (X86_HAVE(F16C)) {
+ return mixmat<
+ 8, 1, tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float>,
+ ggml_fp16_t, ggml_fp16_t, float>();
+ } else {
+ return false;
+ }
#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
return mixmat<
8, 1,