Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add vectorized expf() functions
Justine Tunneye2b3cb2cccdf
diff --git a/llama.cpp/ggml-vector-amd-avx.c b/llama.cpp/ggml-vector-amd-avx.cindex a47d29b5..17d0f136 100644--- a/llama.cpp/ggml-vector-amd-avx.c+++ b/llama.cpp/ggml-vector-amd-avx.c@@ -52,6 +52,7 @@#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_avx#define GGML_VECTOR#include "ggml-vector.inc"#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-avx2.c b/llama.cpp/ggml-vector-amd-avx2.cindex 2daeb599..bf00db96 100644--- a/llama.cpp/ggml-vector-amd-avx2.c+++ b/llama.cpp/ggml-vector-amd-avx2.c@@ -52,6 +52,7 @@#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx2#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx2#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx2+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_avx2#define GGML_VECTOR#include "ggml-vector.inc"#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-avx512.c b/llama.cpp/ggml-vector-amd-avx512.cindex 325e6bb3..c73f2bf2 100644--- a/llama.cpp/ggml-vector-amd-avx512.c+++ b/llama.cpp/ggml-vector-amd-avx512.c@@ -52,6 +52,7 @@#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx512#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx512#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx512+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_avx512#define GGML_VECTOR#include "ggml-vector.inc"#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-avx512bf16.c b/llama.cpp/ggml-vector-amd-avx512bf16.cindex 5f7a3a65..835803eb 100644--- a/llama.cpp/ggml-vector-amd-avx512bf16.c+++ b/llama.cpp/ggml-vector-amd-avx512bf16.c@@ -52,6 +52,7 @@#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx512bf16#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx512bf16#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx512bf16+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_avx512bf16#define GGML_VECTOR#include "ggml-vector.inc"#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-f16c.c b/llama.cpp/ggml-vector-amd-f16c.cindex 5a0f2c9f..2344e8e9 100644--- a/llama.cpp/ggml-vector-amd-f16c.c+++ b/llama.cpp/ggml-vector-amd-f16c.c@@ -52,6 +52,7 @@#define ggml_vec_max_f32 ggml_vec_max_f32_amd_f16c#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_f16c#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_f16c+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_f16c#define GGML_VECTOR#include "ggml-vector.inc"#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-fma.c b/llama.cpp/ggml-vector-amd-fma.cindex 77e72ada..fbfd057d 100644--- a/llama.cpp/ggml-vector-amd-fma.c+++ b/llama.cpp/ggml-vector-amd-fma.c@@ -52,6 +52,7 @@#define ggml_vec_max_f32 ggml_vec_max_f32_amd_fma#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_fma#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_fma+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_fma#define GGML_VECTOR#include "ggml-vector.inc"#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-arm80.c b/llama.cpp/ggml-vector-arm80.cindex 64d51d24..bc4595a1 100644--- a/llama.cpp/ggml-vector-arm80.c+++ b/llama.cpp/ggml-vector-arm80.c@@ -52,6 +52,7 @@#define ggml_vec_max_f32 ggml_vec_max_f32_arm80#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_arm80#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_arm80+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_arm80#define GGML_VECTOR#include "ggml-vector.inc"#endif // __aarch64__diff --git a/llama.cpp/ggml-vector-arm82.c b/llama.cpp/ggml-vector-arm82.cindex fa07a23f..20fb8d44 100644--- a/llama.cpp/ggml-vector-arm82.c+++ b/llama.cpp/ggml-vector-arm82.c@@ -52,6 +52,7 @@#define ggml_vec_max_f32 ggml_vec_max_f32_arm82#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_arm82#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_arm82+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_arm82#define GGML_VECTOR#include "ggml-vector.inc"#endif // __aarch64__diff --git a/llama.cpp/ggml-vector.cpp b/llama.cpp/ggml-vector.cppindex a6aed0cb..3a9a5b8e 100644--- a/llama.cpp/ggml-vector.cpp+++ b/llama.cpp/ggml-vector.cpp@@ -480,6 +480,15 @@ extern "C" void ggml_vec_argmax_f32_amd_avx(const int n, int * s, const float *extern "C" void ggml_vec_argmax_f32_arm82(const int n, int * s, const float * x);extern "C" void ggml_vec_argmax_f32_arm80(const int n, int * s, const float * x);+extern "C" float ggml_vec_soft_max_f32_amd_avx512bf16(const int n, float * y, const float * x, float max);+extern "C" float ggml_vec_soft_max_f32_amd_avx512(const int n, float * y, const float * x, float max);+extern "C" float ggml_vec_soft_max_f32_amd_avx2(const int n, float * y, const float * x, float max);+extern "C" float ggml_vec_soft_max_f32_amd_f16c(const int n, float * y, const float * x, float max);+extern "C" float ggml_vec_soft_max_f32_amd_fma(const int n, float * y, const float * x, float max);+extern "C" float ggml_vec_soft_max_f32_amd_avx(const int n, float * y, const float * x, float max);+extern "C" float ggml_vec_soft_max_f32_arm82(const int n, float * y, const float * x, float max);+extern "C" float ggml_vec_soft_max_f32_arm80(const int n, float * y, const float * x, float max);+static const struct VectorFuncs {typeof(ggml_fp16_to_fp32_row) *ptr_ggml_fp16_to_fp32_row;typeof(ggml_fp32_to_fp16_row) *ptr_ggml_fp32_to_fp16_row;@@ -534,6 +543,7 @@ static const struct VectorFuncs {typeof(ggml_vec_max_f32) *ptr_ggml_vec_max_f32;typeof(ggml_vec_norm_inv_f32) *ptr_ggml_vec_norm_inv_f32;typeof(ggml_vec_argmax_f32) *ptr_ggml_vec_argmax_f32;+ typeof(ggml_vec_soft_max_f32) *ptr_ggml_vec_soft_max_f32;VectorFuncs() {#ifdef __x86_64__@@ -591,6 +601,7 @@ static const struct VectorFuncs {ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx512bf16;ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx512bf16;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx512bf16;+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_avx512bf16;return;}#endif@@ -649,6 +660,7 @@ static const struct VectorFuncs {ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx512;ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx512;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx512;+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_avx512;return;}#endif@@ -707,6 +719,7 @@ static const struct VectorFuncs {ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx2;ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx2;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx2;+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_avx2;return;}#endif@@ -765,6 +778,7 @@ static const struct VectorFuncs {ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_f16c;ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_f16c;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_f16c;+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_f16c;return;}#endif@@ -823,6 +837,7 @@ static const struct VectorFuncs {ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_fma;ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_fma;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_fma;+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_fma;return;}#endif@@ -881,6 +896,7 @@ static const struct VectorFuncs {ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx;ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx;+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_avx;return;}#endif@@ -939,6 +955,7 @@ static const struct VectorFuncs {ptr_ggml_vec_max_f32 = ggml_vec_max_f32_arm82;ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_arm82;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_arm82;+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_arm82;return;}#endif@@ -997,6 +1014,7 @@ static const struct VectorFuncs {ptr_ggml_vec_max_f32 = ggml_vec_max_f32_arm80;ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_arm80;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_arm80;+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_arm80;return;}#endif@@ -1215,3 +1233,7 @@ void ggml_vec_argmax_f32(const int n, int * s, const float * x) {return funcs.ptr_ggml_vec_argmax_f32(n, s, x);}+float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {+ return funcs.ptr_ggml_vec_soft_max_f32(n, y, x, max);+}+diff --git a/llama.cpp/ggml-vector.h b/llama.cpp/ggml-vector.hindex cdaff979..c6aa5e04 100644--- a/llama.cpp/ggml-vector.h+++ b/llama.cpp/ggml-vector.h@@ -61,6 +61,7 @@ void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x);void ggml_vec_max_f32(const int n, float * s, const float * x);void ggml_vec_norm_inv_f32(const int n, float * s, const float * x);void ggml_vec_argmax_f32(const int n, int * s, const float * x);+float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max);#ifdef __cplusplus}diff --git a/llama.cpp/ggml-vector.inc b/llama.cpp/ggml-vector.incindex 244c63c2..62614a77 100644--- a/llama.cpp/ggml-vector.inc+++ b/llama.cpp/ggml-vector.inc@@ -3,6 +3,8 @@#include "ggml-vector.h"#include "ggml-impl.h"+#include "llamafile/expf.h"+#include "llamafile/llamafile.h"#include <stdatomic.h>void ggml_once(atomic_uint *, void (*)(void));@@ -1184,7 +1186,33 @@ void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {}void ggml_vec_silu_f32(const int n, float * y, const float * x) {- for (int i = 0; i < n; ++i) {+ int i = 0;++#if defined(__AVX512F__)+ if (!FLAG_trap) {+ for (; i + 15 < n; i += 16) {+ _mm512_storeu_ps(y + i, llamafile_silu_avx512(_mm512_loadu_ps(x + i)));+ }+ }+#endif++#if defined(__AVX2__) && defined(__FMA__)+ if (!FLAG_trap) {+ for (; i + 7 < n; i += 8) {+ _mm256_storeu_ps(y + i, llamafile_silu_avx2fma(_mm256_loadu_ps(x + i)));+ }+ }+#endif++#if defined(__ARM_NEON)+ if (!FLAG_trap) {+ for (; i + 3 < n; i += 4) {+ vst1q_f32(y + i, llamafile_silu_neon(vld1q_f32(x + i)));+ }+ }+#endif++ for (; i < n; ++i) {y[i] = ggml_silu_f32(x[i]);}}@@ -1254,3 +1282,49 @@ void ggml_vec_argmax_f32(const int n, int * s, const float * x) {}*s = idx;}++float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {+ int i = 0;+ ggml_float sum = 0;++#if defined(__AVX512F__)+ if (!FLAG_trap) {+ for (; i + 15 < n; i += 16) {+ __m512 val = llamafile_expf_avx512(_mm512_sub_ps(_mm512_loadu_ps(x + i),+ _mm512_set1_ps(max)));+ _mm512_storeu_ps(y + i, val);+ sum += _mm512_reduce_add_ps(val);+ }+ }+#endif++#if defined(__AVX2__) && defined(__FMA__)+ if (!FLAG_trap) {+ for (; i + 7 < n; i += 8) {+ __m256 val = llamafile_expf_avx2fma(_mm256_sub_ps(_mm256_loadu_ps(x + i),+ _mm256_set1_ps(max)));+ _mm256_storeu_ps(y + i, val);+ __m128 v2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),+ _mm256_castps256_ps128(val));+ v2 = _mm_add_ps(v2, _mm_movehl_ps(v2, v2));+ v2 = _mm_add_ss(v2, _mm_movehdup_ps(v2));+ sum += _mm_cvtss_f32(v2);+ }+ }+#endif++#ifdef __ARM_NEON+ for (; i + 3 < n; i += 4) {+ float32x4_t val = llamafile_expf_neon(vsubq_f32(vld1q_f32(x + i), vdupq_n_f32(max)));+ vst1q_f32(y + i, val);+ sum += vaddvq_f32(val);+ }+#endif++ for (; i < n; ++i) {+ float val = expf(x[i] - max);+ sum += (ggml_float)val;+ y[i] = val;+ }+ return sum;+}diff --git a/llama.cpp/ggml-vector.py b/llama.cpp/ggml-vector.pyindex 528bf60d..31f20426 100644--- a/llama.cpp/ggml-vector.py+++ b/llama.cpp/ggml-vector.py@@ -3,7 +3,7 @@ import rewith open('llama.cpp/ggml-vector.h') as f:prototypes = f.read()prototypes = [line.replace(';', '') for line in prototypes.split('\n')- if line.endswith(';') and not line.startswith('//')]+ if line.endswith(';') and not line.startswith('//') and '(' in line]FUNCS = [(re.search(r'(?<= )\w+', proto).group(0), proto)for proto in prototypes]diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 5eaf8852..338f80be 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -11867,14 +11867,7 @@ static void ggml_compute_forward_soft_max_f32(float max = -INFINITY;ggml_vec_max_f32(nc, &max, wp);- ggml_float sum = 0.0;-- for (int i = 0; i < nc; i++) {- const float val = expf(wp[i] - max);- sum += (ggml_float)val;- dp[i] = val;- }-+ float sum = ggml_vec_soft_max_f32(nc, dp, wp, max);assert(sum > 0.0);sum = 1.0/sum;diff --git a/llamafile/debug.cpp b/llamafile/debug.cppindex 15512354..c68788b0 100644--- a/llamafile/debug.cpp+++ b/llamafile/debug.cpp@@ -30,15 +30,13 @@#include "llama.cpp/ggml.h"-static int get_config(void) {- int config = FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW;- // TODO(jart): enable if quantization isn't being used- // or possibly suppress exceptions in quants- // config |= FE_UNDERFLOW;- return config;-}+#define TRAPS (FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW)+#define UNDERFLOW_ALARM "\e[s\e[H\e[7;1;49;31mUNDERFLOW\e[0m\e[u"+#define UNDERFLOW_RESET "\e[s\e[H \e[u"+#define UNDERFLOW_DELAY 2bool FLAG_trap;+static atomic_llong g_underflowed;static thread_local int g_enabled;thread_local int llamafile_debug_op_index;const struct ggml_cgraph *llamafile_debug_graph;@@ -46,15 +44,34 @@ const struct ggml_cgraph *llamafile_debug_graph;static struct TerminalBuddy {TerminalBuddy() {active = !tcgetattr(0, &state);+ is_terminal = active && isatty(2);}void restore() {if (active)tcsetattr(0, TCSANOW, &state);}bool active;+ bool is_terminal;struct termios state;} g_terminal_buddy;+static long long millis(void) {+ return timespec_tomillis(timespec_real());+}++static inline void spinlock(atomic_uint *lock) {+ int x;+ for (;;) {+ x = atomic_exchange_explicit(lock, 1, memory_order_acquire);+ if (!x)+ break;+ }+}++static inline void spunlock(atomic_uint *lock) {+ atomic_store_explicit(lock, 0, memory_order_release);+}+static const char *describe_vertex(struct ggml_tensor *t) {if (t->op == GGML_OP_UNARY)return ggml_unary_op_name((enum ggml_unary_op)t->op_params[0]);@@ -99,41 +116,70 @@ int fedisableexcept(int excepts) {return 0;}-static inline void spinlock(atomic_uint *lock) {- int x;- for (;;) {- x = atomic_exchange_explicit(lock, 1, memory_order_acquire);- if (!x)- break;+// recover from trap so that execution may resume+// without this the same signal will just keep getting raised+static void recover(ucontext_t *ctx, int traps) {+#ifdef __x86_64__+ if (ctx->uc_mcontext.fpregs) {+ ctx->uc_mcontext.fpregs->mxcsr |= traps << 7; // disable traps+ ctx->uc_mcontext.fpregs->mxcsr &= ~traps; // clear cages+ return;}-}--static inline void spunlock(atomic_uint *lock) {- atomic_store_explicit(lock, 0, memory_order_release);+#elif defined(__aarch64__)+ struct _aarch64_ctx *ac;+ for (ac = (struct _aarch64_ctx *)ctx->uc_mcontext.__reserved; ac->magic;+ ac = (struct _aarch64_ctx *)((char *)ac + ac->size)) {+ if (ac->magic == FPSIMD_MAGIC) {+ struct fpsimd_context *sm = (struct fpsimd_context *)ac;+ sm->fpcr &= ~(traps << 8); // disable traps+ sm->fpsr &= ~traps; // clear cages+ return;+ }+ }+#endif+ // time to die+ g_terminal_buddy.restore();+ _exit(128 + SIGFPE);}static void on_sigfpe(int sig, siginfo_t *si, void *arg) {+ ucontext_t *ctx = (ucontext_t *)arg;+ int reason = si->si_code;++ // underflows are something we expect to happen, particularly with+ // quantization, so we'll display a check engine light whenever it+ // happens, provided we're in teletypewriter mode.+ if (reason == FPE_FLTUND) {+ if (g_terminal_buddy.is_terminal) {+ long long now = millis();+ if ((now - atomic_exchange_explicit(&g_underflowed, now, memory_order_relaxed)) >+ UNDERFLOW_DELAY) {+ write(2, UNDERFLOW_ALARM, strlen(UNDERFLOW_ALARM));+ }+ }+ recover(ctx, FE_UNDERFLOW);+ return;+ }static atomic_uint lock;spinlock(&lock);const char *issue;- ucontext_t *ctx = (ucontext_t *)arg;- if (si->si_code == FPE_INTDIV)+ if (reason == FPE_INTDIV)issue = "integer divide by zero";- else if (si->si_code == FPE_INTOVF)+ else if (reason == FPE_INTOVF)issue = "integer overflow";- else if (si->si_code == FPE_FLTDIV)+ else if (reason == FPE_FLTDIV)issue = "floating point divide by zero";- else if (si->si_code == FPE_FLTOVF)+ else if (reason == FPE_FLTOVF)issue = "floating point overflow";- else if (si->si_code == FPE_FLTUND)+ else if (reason == FPE_FLTUND)issue = "floating point underflow";- else if (si->si_code == FPE_FLTRES)+ else if (reason == FPE_FLTRES)issue = "floating point inexact";- else if (si->si_code == FPE_FLTINV)+ else if (reason == FPE_FLTINV)issue = "invalid floating point operation";- else if (si->si_code == FPE_FLTSUB)+ else if (reason == FPE_FLTSUB)issue = "subscript out of range";elseissue = "sigfpe";@@ -165,33 +211,8 @@ static void on_sigfpe(int sig, siginfo_t *si, void *arg) {once = true;}- // recover from trap so that execution may resume- // without this the same signal will just keep getting raised- int traps = get_config();-#ifdef __x86_64__- if (ctx->uc_mcontext.fpregs) {- ctx->uc_mcontext.fpregs->mxcsr |= traps << 7; // disable traps- ctx->uc_mcontext.fpregs->mxcsr &= ~traps; // clear cages- spunlock(&lock);- return;- }-#elif defined(__aarch64__)- struct _aarch64_ctx *ac;- for (ac = (struct _aarch64_ctx *)ctx->uc_mcontext.__reserved; ac->magic;- ac = (struct _aarch64_ctx *)((char *)ac + ac->size)) {- if (ac->magic == FPSIMD_MAGIC) {- struct fpsimd_context *sm = (struct fpsimd_context *)ac;- sm->fpcr &= ~(traps << 8); // disable traps- sm->fpsr &= ~traps; // clear cages- spunlock(&lock);- return;- }- }-#endif-- // time to die- g_terminal_buddy.restore();- _exit(128 + SIGFPE);+ recover(ctx, TRAPS);+ spunlock(&lock);}static void setup_sigfpe(void) {@@ -209,14 +230,13 @@ int llamafile_trapping_enabled(int delta) {static atomic_uint once;bool was_enabled = g_enabled > 0;bool is_enabled = (g_enabled += delta) > 0;- int config = get_config();feclearexcept(FE_ALL_EXCEPT);if (is_enabled && !was_enabled) {cosmo_once(&once, setup_sigfpe);- feenableexcept(config);+ feenableexcept(TRAPS);}if (!is_enabled && was_enabled) {- fedisableexcept(config);+ fedisableexcept(TRAPS);}return g_enabled;}@@ -224,6 +244,16 @@ int llamafile_trapping_enabled(int delta) {void llamafile_trapping_restore(void) {if (g_enabled > 0) {feclearexcept(FE_ALL_EXCEPT);- feenableexcept(get_config());+ feenableexcept(TRAPS);+ long long last;+ if (g_terminal_buddy.is_terminal &&+ (last = atomic_load_explicit(&g_underflowed, memory_order_relaxed))) {+ long long now = millis();+ if (now - last > UNDERFLOW_DELAY &&+ now - atomic_exchange_explicit(&g_underflowed, 0, memory_order_relaxed) >+ UNDERFLOW_DELAY) {+ write(2, UNDERFLOW_RESET, strlen(UNDERFLOW_RESET));+ }+ }}}diff --git a/llamafile/expf.h b/llamafile/expf.hnew file mode 100644index 00000000..20082954--- /dev/null+++ b/llamafile/expf.h@@ -0,0 +1,146 @@+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2023 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once++#ifdef __AVX512F__+#include <immintrin.h>++// computes expf() for each element in vector.+//+// the maximum error is 1.45358 +0.5 ulp. the only difference between+// this function and expf(), is that there's currently no support for+// subnormals. input values are clamped to range: [-87.6831, 88.3763]+// whereas expf() allows inputs as low as -103.972. therefore numbers+// will be flushed to zero sooner than they otherwise would with this+// function. nearest rounding mode is always used. exception trapping+// isn't supported although this function does a good job avoiding it+//+static inline __m512 llamafile_expf_avx512(__m512 x) {+ __m512 a, b, c, d, e, f, g;+ __m512 will_turn_into_inf = _mm512_set1_ps(0x1.62e44p+6f);+ __m512 max_before_overflow = _mm512_set1_ps(0x1.61814cp+6f);+ __m512 min_before_underflow = _mm512_set1_ps(-0x1.5ebb86p+6f);+ x = _mm512_mask_blend_ps(_mm512_cmp_ps_mask(x, max_before_overflow, _CMP_GE_OQ), x,+ will_turn_into_inf);+ x = _mm512_mask_blend_ps(_mm512_cmp_ps_mask(x, min_before_underflow, _CMP_LE_OQ), x,+ min_before_underflow);+ a = _mm512_fmadd_round_ps(_mm512_set1_ps(0x1.715476p+0f), x, _mm512_set1_ps(0x1.8p23f),+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);+ b = _mm512_sub_ps(a, _mm512_set1_ps(0x1.8p23f));+ c = _mm512_fnmadd_round_ps(_mm512_set1_ps(0x1.62e4p-1f), b, x,+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);+ c = _mm512_fnmadd_round_ps(_mm512_set1_ps(0x1.7f7d1cp-20f), b, c,+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);+ d = _mm512_castsi512_ps(_mm512_add_epi32(_mm512_slli_epi32(_mm512_castps_si512(a), 23),+ _mm512_set1_epi32(0x3f800000u)));+ e = _mm512_mul_round_ps(c, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);+ f = _mm512_fmadd_round_ps(_mm512_set1_ps(0x1.0e4020p-7f), c, _mm512_set1_ps(0x1.573e2ep-5f),+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);+ g = _mm512_fmadd_round_ps(_mm512_set1_ps(0x1.555e66p-3f), c, _mm512_set1_ps(0x1.fffdb6p-2f),+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);+ g = _mm512_fmadd_round_ps(f, e, g, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);+ f = _mm512_mul_round_ps(_mm512_set1_ps(0x1.ffffecp-1f), c,+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);+ return _mm512_fmadd_round_ps(+ _mm512_fmadd_round_ps(g, e, f, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC), d, d,+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);+}++// computes silu x/(1+exp(-x)) in single precision+static inline __m512 llamafile_silu_avx512(__m512 x) {+ __m512 one = _mm512_set1_ps(1);+ __m512 zero = _mm512_setzero_ps();+ __m512 neg_x = _mm512_sub_ps(zero, x);+ __m512 exp_neg_x = llamafile_expf_avx512(neg_x);+ __m512 one_plus_exp_neg_x = _mm512_add_ps(one, exp_neg_x);+ return _mm512_div_ps(x, one_plus_exp_neg_x);+}++#endif // __AVX512F__++#if defined(__AVX2__) && defined(__FMA__)+#include <immintrin.h>++// computes expf() for each element in vector.+//+// the maximum error is 1.45358 +0.5 ulp. the only difference between+// this function and expf(), is that there's currently no support for+// subnormals. input values are clamped to range: [-87.6831, 88.3763]+// whereas expf() allows inputs as low as -103.972. therefore numbers+// will be flushed to zero sooner than they otherwise would with this+// function. exception trapping isnt supported although this function+// does a good job avoiding it.+//+static inline __m256 llamafile_expf_avx2fma(__m256 x) {+ __m256 a, b, c, d, e, f, g;+ __m256 will_turn_into_inf = _mm256_set1_ps(0x1.62e44p+6f);+ __m256 max_before_overflow = _mm256_set1_ps(0x1.61814cp+6f);+ __m256 min_before_underflow = _mm256_set1_ps(-0x1.5ebb86p+6f);+ __m256 min_mask = _mm256_cmp_ps(x, min_before_underflow, _CMP_LE_OQ);+ __m256 max_mask = _mm256_cmp_ps(x, max_before_overflow, _CMP_GE_OQ);+ x = _mm256_or_ps(_mm256_and_ps(min_mask, min_before_underflow), _mm256_andnot_ps(min_mask, x));+ x = _mm256_or_ps(_mm256_and_ps(max_mask, will_turn_into_inf), _mm256_andnot_ps(max_mask, x));+ a = _mm256_fmadd_ps(_mm256_set1_ps(0x1.715476p+0f), x, _mm256_set1_ps(0x1.8p23f));+ b = _mm256_sub_ps(a, _mm256_set1_ps(0x1.8p23f));+ c = _mm256_fnmadd_ps(_mm256_set1_ps(0x1.62e4p-1f), b, x);+ c = _mm256_fnmadd_ps(_mm256_set1_ps(0x1.7f7d1cp-20f), b, c);+ d = _mm256_castsi256_ps(_mm256_add_epi32(_mm256_slli_epi32(_mm256_castps_si256(a), 23),+ _mm256_set1_epi32(0x3f800000u)));+ e = _mm256_mul_ps(c, c);+ f = _mm256_fmadd_ps(_mm256_set1_ps(0x1.0e4020p-7f), c, _mm256_set1_ps(0x1.573e2ep-5f));+ g = _mm256_fmadd_ps(_mm256_set1_ps(0x1.555e66p-3f), c, _mm256_set1_ps(0x1.fffdb6p-2f));+ g = _mm256_fmadd_ps(f, e, g);+ f = _mm256_mul_ps(_mm256_set1_ps(0x1.ffffecp-1f), c);+ return _mm256_fmadd_ps(_mm256_fmadd_ps(g, e, f), d, d);+}++// computes silu x/(1+exp(-x)) in single precision+static inline __m256 llamafile_silu_avx2fma(__m256 x) {+ __m256 one = _mm256_set1_ps(1);+ __m256 zero = _mm256_setzero_ps();+ __m256 neg_x = _mm256_sub_ps(zero, x);+ __m256 exp_neg_x = llamafile_expf_avx2fma(neg_x);+ __m256 one_plus_exp_neg_x = _mm256_add_ps(one, exp_neg_x);+ return _mm256_div_ps(x, one_plus_exp_neg_x);+}++#endif // __AVX2__++#ifdef __ARM_NEON+#include <arm_neon.h>++float32x4_t v_expf(float32x4_t);++static inline float32x4_t llamafile_expf_neon(float32x4_t x) {+ return v_expf(x);+}++static inline float32x4_t llamafile_silu_neon(float32x4_t x) {+ float32x4_t one = vdupq_n_f32(1.0f);+ float32x4_t zero = vdupq_n_f32(0.0f);+ float32x4_t neg_x = vsubq_f32(zero, x);+ float32x4_t exp_neg_x = llamafile_expf_neon(neg_x);+ float32x4_t one_plus_exp_neg_x = vaddq_f32(one, exp_neg_x);+ return vdivq_f32(x, one_plus_exp_neg_x);+}++#endif++#ifdef __cplusplus+}+#endifdiff --git a/llamafile/v_expf.c b/llamafile/v_expf.cnew file mode 100644index 00000000..4127df92--- /dev/null+++ b/llamafile/v_expf.c@@ -0,0 +1,145 @@+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 1999-2022 Arm Limited+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __aarch64__++#include <arm_neon.h>++#define V4(X) {X, X, X, X}+#define NOINLINE __attribute__((__noinline__))+#define VPCS_ATTR __attribute__((aarch64_vector_pcs))+#define likely(x) __builtin_expect(!!(x), 1)+#define unlikely(x) __builtin_expect(x, 0)++static inline uint32x4_t v_u32(uint32_t x) {+ return (uint32x4_t)V4(x);+}++/* true if any elements of a vector compare result is non-zero. */+static inline int v_any_u32(uint32x4_t x) {+ /* assume elements in x are either 0 or -1u. */+ return vpaddd_u64(vreinterpretq_u64_u32(x)) != 0;+}++static const struct data {+ float32x4_t poly[5];+ float32x4_t shift, inv_ln2, ln2_hi, ln2_lo;+ uint32x4_t exponent_bias;+#if !WANT_SIMD_EXCEPT+ float32x4_t special_bound, scale_thresh;+#endif+} data = {+ /* maxerr: 1.45358 +0.5 ulp. */+ .poly = {V4(0x1.0e4020p-7f), V4(0x1.573e2ep-5f), V4(0x1.555e66p-3f), V4(0x1.fffdb6p-2f),+ V4(0x1.ffffecp-1f)},+ .shift = V4(0x1.8p23f),+ .inv_ln2 = V4(0x1.715476p+0f),+ .ln2_hi = V4(0x1.62e4p-1f),+ .ln2_lo = V4(0x1.7f7d1cp-20f),+ .exponent_bias = V4(0x3f800000),+#if !WANT_SIMD_EXCEPT+ .special_bound = V4(126.0f),+ .scale_thresh = V4(192.0f),+#endif+};++#define C(i) d->poly[i]++#if WANT_SIMD_EXCEPT++#define TinyBound v_u32(0x20000000) /* asuint (0x1p-63). */+#define BigBound v_u32(0x42800000) /* asuint (0x1p6). */+#define SpecialBound v_u32(0x22800000) /* BigBound - TinyBound. */++static float32x4_t VPCS_ATTR NOINLINE special_case(float32x4_t x, float32x4_t y, uint32x4_t cmp) {+ /* If fenv exceptions are to be triggered correctly, fall back to the scalar+ routine to special lanes. */+ return v_call_f32(expf, x, y, cmp);+}++#else++#define SpecialOffset v_u32(0x82000000)+#define SpecialBias v_u32(0x7f000000)++static float32x4_t VPCS_ATTR NOINLINE special_case(float32x4_t poly, float32x4_t n, uint32x4_t e,+ uint32x4_t cmp1, float32x4_t scale,+ const struct data *d) {+ /* 2^n may overflow, break it up into s1*s2. */+ uint32x4_t b = vandq_u32(vclezq_f32(n), SpecialOffset);+ float32x4_t s1 = vreinterpretq_f32_u32(vaddq_u32(b, SpecialBias));+ float32x4_t s2 = vreinterpretq_f32_u32(vsubq_u32(e, b));+ uint32x4_t cmp2 = vcagtq_f32(n, d->scale_thresh);+ float32x4_t r2 = vmulq_f32(s1, s1);+ float32x4_t r1 = vmulq_f32(vfmaq_f32(s2, poly, s2), s1);+ /* Similar to r1 but avoids double rounding in the subnormal range. */+ float32x4_t r0 = vfmaq_f32(scale, poly, scale);+ float32x4_t r = vbslq_f32(cmp1, r1, r0);+ return vbslq_f32(cmp2, r2, r);+}++#endif++float32x4_t v_expf(float32x4_t x) {+ const struct data *d = &data;+ __asm__("" : "+r"(d));+ float32x4_t n, r, r2, scale, p, q, poly, z;+ uint32x4_t cmp, e;++#if WANT_SIMD_EXCEPT+ /* asuint(x) - TinyBound >= BigBound - TinyBound. */+ cmp = vcgeq_u32(vsubq_u32(vandq_u32(vreinterpretq_u32_f32(x), v_u32(0x7fffffff)), TinyBound),+ SpecialBound);+ float32x4_t xm = x;+ /* If any lanes are special, mask them with 1 and retain a copy of x to allow+ special case handler to fix special lanes later. This is only necessary if+ fenv exceptions are to be triggered correctly. */+ if (unlikely(v_any_u32(cmp)))+ x = vbslq_f32(cmp, v_f32(1), x);+#endif++ /* exp(x) = 2^n (1 + poly(r)), with 1 + poly(r) in [1/sqrt(2),sqrt(2)]+ x = ln2*n + r, with r in [-ln2/2, ln2/2]. */+ z = vfmaq_f32(d->shift, x, d->inv_ln2);+ n = vsubq_f32(z, d->shift);+ r = vfmsq_f32(x, n, d->ln2_hi);+ r = vfmsq_f32(r, n, d->ln2_lo);+ e = vshlq_n_u32(vreinterpretq_u32_f32(z), 23);+ scale = vreinterpretq_f32_u32(vaddq_u32(e, d->exponent_bias));++#if !WANT_SIMD_EXCEPT+ cmp = vcagtq_f32(n, d->special_bound);+#endif++ r2 = vmulq_f32(r, r);+ p = vfmaq_f32(C(1), C(0), r);+ q = vfmaq_f32(C(3), C(2), r);+ q = vfmaq_f32(q, p, r2);+ p = vmulq_f32(C(4), r);+ poly = vfmaq_f32(p, q, r2);++ if (unlikely(v_any_u32(cmp)))+#if WANT_SIMD_EXCEPT+ return special_case(xm, vfmaq_f32(scale, poly, scale), cmp);+#else+ return special_case(poly, n, e, cmp, scale, d);+#endif++ return vfmaq_f32(scale, poly, scale);+}++#endif // __aarch64__