Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Restore Alderlake performance
Justine Tunney38cf08ff4fb5
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex 99897db0..c1793a2f 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -22,6 +22,7 @@#include <vector>#include <cinttypes>#include <codecvt>+#include <cosmo.h>#if defined(__APPLE__) && defined(__MACH__)#include <sys/types.h>@@ -76,7 +77,7 @@using json = nlohmann::ordered_json;int32_t get_num_physical_cores() {-#ifdef __linux__+ if (IsLinux()) {// enumerate the set of thread siblings, num entries is num coresstd::unordered_set<std::string> siblings;for (uint32_t cpu=0; cpu < UINT32_MAX; ++cpu) {@@ -93,25 +94,12 @@ int32_t get_num_physical_cores() {if (!siblings.empty()) {return static_cast<int32_t>(siblings.size());}-#elif defined(__APPLE__) && defined(__MACH__)- int32_t num_physical_cores;- size_t len = sizeof(num_physical_cores);- int result = sysctlbyname("hw.perflevel0.physicalcpu", &num_physical_cores, &len, NULL, 0);- if (result == 0) {- return num_physical_cores;- }- result = sysctlbyname("hw.physicalcpu", &num_physical_cores, &len, NULL, 0);- if (result == 0) {- return num_physical_cores;}-#elif defined(_WIN32)- //TODO: Implement-#endifunsigned int n_threads = std::thread::hardware_concurrency();return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4;}-#if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__)+#if defined(__x86_64__) && (defined(__linux__) || defined(__COSMOPOLITAN__)) && !defined(__ANDROID__)#include <pthread.h>static void cpuid(unsigned leaf, unsigned subleaf,@@ -165,7 +153,7 @@ static int count_math_cpus(int cpu_count) {* Returns number of CPUs on system that are useful for math.*/int get_math_cpu_count() {-#if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__)+#if defined(__x86_64__) && (defined(__linux__) || defined(__COSMOPOLITAN__)) && !defined(__ANDROID__)int cpu_count = sysconf(_SC_NPROCESSORS_ONLN);if (cpu_count < 1) {return get_num_physical_cores();diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex a7482051..8dcf2363 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -42,13 +42,54 @@ o/$(MODE)/llamafile: \################################################################################# microarchitectures+#+#### Intel CPU Line+#+# - 2006 core X64 SSE4.1 (only on 45nm variety) (-march=core2)+# - 2008 nehalem SSE4.2 VT-x VT-d RDTSCP POPCNT (-march=nehalem)+# - 2010 westmere CLMUL AES (-march=westmere)+# - 2011 sandybridge AVX TXT (-march=sandybridge)+# - 2012 ivybridge F16C MOVBE FSGSBASE (-march=ivybridge)+# - 2013 haswell AVX2 TSX BMI1 BMI2 FMA (-march=haswell)+# - 2014 broadwell RDSEED ADX PREFETCHW (-march=broadwell)+# - 2015 skylake SGX ADX MPX AVX-512[xeon-only] (-march=skylake / -march=skylake-avx512)+# - 2018 cannonlake SHA (-march=cannonlake)+# - 2019 cascadelake VNNI+# - 2021 alderlake efficiency cores+#+#### AMD CPU Line+#+# - 2003 k8 SSE SSE2 (-march=k8)+# - 2005 k8 (Venus) SSE3 (-march=k8-sse3)+# - 2008 barcelona SSE4a?! (-march=barcelona)+# - 2011 bulldozer SSSE3 SSE4.1 SSE4.2 CLMUL AVX AES FMA4?! (-march=bdver1)+# - 2011 piledriver BMI1 FMA (-march=bdver2)+# - 2015 excavator AVX2 BMI2 MOVBE (-march=bdver4)+# - 2017 ryzen F16C SHA ADX (-march=znver1)+# - 2023 zen4 AVX512F AVX512VL AVX512VNNI AVX512BF16 (-march=znver4)+#+#### ARM Revisions+#+# - armv8.0-a raspberry pi 4+# - armv8.2-a raspberry pi 5+# - armv8.5-a apple m1+# - armv8.6-a apple m2+#+#### ARM Features+#+# - HWCAP_CRC32 +crc32 (e.g. m1, rpi4) __ARM_FEATRUE_CRC32+# - HWCAP_FPHP +fp16 (e.g. m1, rpi5) __ARM_FEATURE_FP16_SCALAR_ARITHMETIC+# - HWCAP_ASIMDHP +fp16 (e.g. m1, rpi5) __ARM_FEATURE_FP16_VECTOR_ARITHMETIC+# - HWCAP_ASIMDDP +dotprod (e.g. m1, rpi5) __ARM_FEATURE_DOTPROD+#o/$(MODE)/llamafile/sgemm.o: private CXXFLAGS += -Os-o/$(MODE)/llamafile/tinyblas_cpu_amd_fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mfma-o/$(MODE)/llamafile/tinyblas_cpu_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mfma-o/$(MODE)/llamafile/tinyblas_cpu_amd_avxvnni.o: private TARGET_ARCH += -Xx86_64-mtune=alderlake -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavxvnni-o/$(MODE)/llamafile/tinyblas_cpu_amd_avx512f.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f-o/$(MODE)/llamafile/tinyblas_cpu_amd_zen4.o: private TARGET_ARCH += -Xx86_64-mtune=znver4 -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512vl -Xx86_64-mavx512vnni -Xx86_64-mavx512bf16+o/$(MODE)/llamafile/tinyblas_cpu_amd_avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge -Xx86_64-mf16c+o/$(MODE)/llamafile/tinyblas_cpu_amd_fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mf16c -Xx86_64-mfma+o/$(MODE)/llamafile/tinyblas_cpu_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mfma+o/$(MODE)/llamafile/tinyblas_cpu_amd_avxvnni.o: private TARGET_ARCH += -Xx86_64-mtune=alderlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavxvnni+o/$(MODE)/llamafile/tinyblas_cpu_amd_avx512f.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f+o/$(MODE)/llamafile/tinyblas_cpu_amd_zen4.o: private TARGET_ARCH += -Xx86_64-mtune=znver4 -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512vl -Xx86_64-mavx512vnni -Xx86_64-mavx512bf16o/$(MODE)/llamafile/tinyblas_cpu_arm82.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+dotprod+fp16################################################################################diff --git a/llamafile/tinyblas_cpu.inc b/llamafile/tinyblas_cpu.incindex 1ff3225e..a25fb23a 100644--- a/llamafile/tinyblas_cpu.inc+++ b/llamafile/tinyblas_cpu.inc@@ -42,6 +42,7 @@#include "llama.cpp/ggml-quants.h"#include "llamafile.h"#include <algorithm>+#include <cosmo.h>#pragma GCC diagnostic ignored "-Wpedantic"#pragma GCC diagnostic ignored "-Wignored-attributes"@@ -1022,14 +1023,18 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,tb.matmul(m, n, task);return true;#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)- if (k % 8)- return false;- if (Btype != GGML_TYPE_F32)+ if (X86_HAVE(F16C)) {+ if (k % 8)+ return false;+ if (Btype != GGML_TYPE_F32)+ return false;+ tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, float> tb{+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+ } else {return false;- tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, float> tb{- k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);- return true;+ }#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)if (n < 8)return false;@@ -1295,9 +1300,13 @@ class MixMul {tinyBLAS<NCB | NCC, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, float>,ggml_fp16_t, ggml_fp16_t, float>();#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)- return mixmat<8, 1,- tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float>,- ggml_fp16_t, ggml_fp16_t, float>();+ if (X86_HAVE(F16C)) {+ return mixmat<+ 8, 1, tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float>,+ ggml_fp16_t, ggml_fp16_t, float>();+ } else {+ return false;+ }#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)return mixmat<8, 1,