Skip to content

Distribute and run LLMs with a single file.

Add vectorized expf() functions

Justine Tunneye2b3cb2cccdf
diff --git a/llama.cpp/ggml-vector-amd-avx.c b/llama.cpp/ggml-vector-amd-avx.c
index a47d29b5..17d0f136 100644
--- a/llama.cpp/ggml-vector-amd-avx.c
+++ b/llama.cpp/ggml-vector-amd-avx.c
@@ -52,6 +52,7 @@
#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx
#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx
#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx
+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_avx
#define GGML_VECTOR
#include "ggml-vector.inc"
#endif // __x86_64__
diff --git a/llama.cpp/ggml-vector-amd-avx2.c b/llama.cpp/ggml-vector-amd-avx2.c
index 2daeb599..bf00db96 100644
--- a/llama.cpp/ggml-vector-amd-avx2.c
+++ b/llama.cpp/ggml-vector-amd-avx2.c
@@ -52,6 +52,7 @@
#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx2
#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx2
#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx2
+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_avx2
#define GGML_VECTOR
#include "ggml-vector.inc"
#endif // __x86_64__
diff --git a/llama.cpp/ggml-vector-amd-avx512.c b/llama.cpp/ggml-vector-amd-avx512.c
index 325e6bb3..c73f2bf2 100644
--- a/llama.cpp/ggml-vector-amd-avx512.c
+++ b/llama.cpp/ggml-vector-amd-avx512.c
@@ -52,6 +52,7 @@
#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx512
#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx512
#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx512
+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_avx512
#define GGML_VECTOR
#include "ggml-vector.inc"
#endif // __x86_64__
diff --git a/llama.cpp/ggml-vector-amd-avx512bf16.c b/llama.cpp/ggml-vector-amd-avx512bf16.c
index 5f7a3a65..835803eb 100644
--- a/llama.cpp/ggml-vector-amd-avx512bf16.c
+++ b/llama.cpp/ggml-vector-amd-avx512bf16.c
@@ -52,6 +52,7 @@
#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx512bf16
#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx512bf16
#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx512bf16
+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_avx512bf16
#define GGML_VECTOR
#include "ggml-vector.inc"
#endif // __x86_64__
diff --git a/llama.cpp/ggml-vector-amd-f16c.c b/llama.cpp/ggml-vector-amd-f16c.c
index 5a0f2c9f..2344e8e9 100644
--- a/llama.cpp/ggml-vector-amd-f16c.c
+++ b/llama.cpp/ggml-vector-amd-f16c.c
@@ -52,6 +52,7 @@
#define ggml_vec_max_f32 ggml_vec_max_f32_amd_f16c
#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_f16c
#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_f16c
+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_f16c
#define GGML_VECTOR
#include "ggml-vector.inc"
#endif // __x86_64__
diff --git a/llama.cpp/ggml-vector-amd-fma.c b/llama.cpp/ggml-vector-amd-fma.c
index 77e72ada..fbfd057d 100644
--- a/llama.cpp/ggml-vector-amd-fma.c
+++ b/llama.cpp/ggml-vector-amd-fma.c
@@ -52,6 +52,7 @@
#define ggml_vec_max_f32 ggml_vec_max_f32_amd_fma
#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_fma
#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_fma
+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_fma
#define GGML_VECTOR
#include "ggml-vector.inc"
#endif // __x86_64__
diff --git a/llama.cpp/ggml-vector-arm80.c b/llama.cpp/ggml-vector-arm80.c
index 64d51d24..bc4595a1 100644
--- a/llama.cpp/ggml-vector-arm80.c
+++ b/llama.cpp/ggml-vector-arm80.c
@@ -52,6 +52,7 @@
#define ggml_vec_max_f32 ggml_vec_max_f32_arm80
#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_arm80
#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_arm80
+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_arm80
#define GGML_VECTOR
#include "ggml-vector.inc"
#endif // __aarch64__
diff --git a/llama.cpp/ggml-vector-arm82.c b/llama.cpp/ggml-vector-arm82.c
index fa07a23f..20fb8d44 100644
--- a/llama.cpp/ggml-vector-arm82.c
+++ b/llama.cpp/ggml-vector-arm82.c
@@ -52,6 +52,7 @@
#define ggml_vec_max_f32 ggml_vec_max_f32_arm82
#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_arm82
#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_arm82
+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_arm82
#define GGML_VECTOR
#include "ggml-vector.inc"
#endif // __aarch64__
diff --git a/llama.cpp/ggml-vector.cpp b/llama.cpp/ggml-vector.cpp
index a6aed0cb..3a9a5b8e 100644
--- a/llama.cpp/ggml-vector.cpp
+++ b/llama.cpp/ggml-vector.cpp
@@ -480,6 +480,15 @@ extern "C" void ggml_vec_argmax_f32_amd_avx(const int n, int * s, const float *
extern "C" void ggml_vec_argmax_f32_arm82(const int n, int * s, const float * x);
extern "C" void ggml_vec_argmax_f32_arm80(const int n, int * s, const float * x);
+extern "C" float ggml_vec_soft_max_f32_amd_avx512bf16(const int n, float * y, const float * x, float max);
+extern "C" float ggml_vec_soft_max_f32_amd_avx512(const int n, float * y, const float * x, float max);
+extern "C" float ggml_vec_soft_max_f32_amd_avx2(const int n, float * y, const float * x, float max);
+extern "C" float ggml_vec_soft_max_f32_amd_f16c(const int n, float * y, const float * x, float max);
+extern "C" float ggml_vec_soft_max_f32_amd_fma(const int n, float * y, const float * x, float max);
+extern "C" float ggml_vec_soft_max_f32_amd_avx(const int n, float * y, const float * x, float max);
+extern "C" float ggml_vec_soft_max_f32_arm82(const int n, float * y, const float * x, float max);
+extern "C" float ggml_vec_soft_max_f32_arm80(const int n, float * y, const float * x, float max);
+
static const struct VectorFuncs {
typeof(ggml_fp16_to_fp32_row) *ptr_ggml_fp16_to_fp32_row;
typeof(ggml_fp32_to_fp16_row) *ptr_ggml_fp32_to_fp16_row;
@@ -534,6 +543,7 @@ static const struct VectorFuncs {
typeof(ggml_vec_max_f32) *ptr_ggml_vec_max_f32;
typeof(ggml_vec_norm_inv_f32) *ptr_ggml_vec_norm_inv_f32;
typeof(ggml_vec_argmax_f32) *ptr_ggml_vec_argmax_f32;
+ typeof(ggml_vec_soft_max_f32) *ptr_ggml_vec_soft_max_f32;
VectorFuncs() {
#ifdef __x86_64__
@@ -591,6 +601,7 @@ static const struct VectorFuncs {
ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx512bf16;
ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx512bf16;
ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx512bf16;
+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_avx512bf16;
return;
}
#endif
@@ -649,6 +660,7 @@ static const struct VectorFuncs {
ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx512;
ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx512;
ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx512;
+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_avx512;
return;
}
#endif
@@ -707,6 +719,7 @@ static const struct VectorFuncs {
ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx2;
ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx2;
ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx2;
+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_avx2;
return;
}
#endif
@@ -765,6 +778,7 @@ static const struct VectorFuncs {
ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_f16c;
ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_f16c;
ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_f16c;
+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_f16c;
return;
}
#endif
@@ -823,6 +837,7 @@ static const struct VectorFuncs {
ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_fma;
ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_fma;
ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_fma;
+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_fma;
return;
}
#endif
@@ -881,6 +896,7 @@ static const struct VectorFuncs {
ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx;
ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx;
ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx;
+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_avx;
return;
}
#endif
@@ -939,6 +955,7 @@ static const struct VectorFuncs {
ptr_ggml_vec_max_f32 = ggml_vec_max_f32_arm82;
ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_arm82;
ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_arm82;
+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_arm82;
return;
}
#endif
@@ -997,6 +1014,7 @@ static const struct VectorFuncs {
ptr_ggml_vec_max_f32 = ggml_vec_max_f32_arm80;
ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_arm80;
ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_arm80;
+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_arm80;
return;
}
#endif
@@ -1215,3 +1233,7 @@ void ggml_vec_argmax_f32(const int n, int * s, const float * x) {
return funcs.ptr_ggml_vec_argmax_f32(n, s, x);
}
+float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {
+ return funcs.ptr_ggml_vec_soft_max_f32(n, y, x, max);
+}
+
diff --git a/llama.cpp/ggml-vector.h b/llama.cpp/ggml-vector.h
index cdaff979..c6aa5e04 100644
--- a/llama.cpp/ggml-vector.h
+++ b/llama.cpp/ggml-vector.h
@@ -61,6 +61,7 @@ void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x);
void ggml_vec_max_f32(const int n, float * s, const float * x);
void ggml_vec_norm_inv_f32(const int n, float * s, const float * x);
void ggml_vec_argmax_f32(const int n, int * s, const float * x);
+float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max);
#ifdef __cplusplus
}
diff --git a/llama.cpp/ggml-vector.inc b/llama.cpp/ggml-vector.inc
index 244c63c2..62614a77 100644
--- a/llama.cpp/ggml-vector.inc
+++ b/llama.cpp/ggml-vector.inc
@@ -3,6 +3,8 @@
#include "ggml-vector.h"
#include "ggml-impl.h"
+#include "llamafile/expf.h"
+#include "llamafile/llamafile.h"
#include <stdatomic.h>
void ggml_once(atomic_uint *, void (*)(void));
@@ -1184,7 +1186,33 @@ void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {
}
void ggml_vec_silu_f32(const int n, float * y, const float * x) {
- for (int i = 0; i < n; ++i) {
+ int i = 0;
+
+#if defined(__AVX512F__)
+ if (!FLAG_trap) {
+ for (; i + 15 < n; i += 16) {
+ _mm512_storeu_ps(y + i, llamafile_silu_avx512(_mm512_loadu_ps(x + i)));
+ }
+ }
+#endif
+
+#if defined(__AVX2__) && defined(__FMA__)
+ if (!FLAG_trap) {
+ for (; i + 7 < n; i += 8) {
+ _mm256_storeu_ps(y + i, llamafile_silu_avx2fma(_mm256_loadu_ps(x + i)));
+ }
+ }
+#endif
+
+#if defined(__ARM_NEON)
+ if (!FLAG_trap) {
+ for (; i + 3 < n; i += 4) {
+ vst1q_f32(y + i, llamafile_silu_neon(vld1q_f32(x + i)));
+ }
+ }
+#endif
+
+ for (; i < n; ++i) {
y[i] = ggml_silu_f32(x[i]);
}
}
@@ -1254,3 +1282,49 @@ void ggml_vec_argmax_f32(const int n, int * s, const float * x) {
}
*s = idx;
}
+
+float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {
+ int i = 0;
+ ggml_float sum = 0;
+
+#if defined(__AVX512F__)
+ if (!FLAG_trap) {
+ for (; i + 15 < n; i += 16) {
+ __m512 val = llamafile_expf_avx512(_mm512_sub_ps(_mm512_loadu_ps(x + i),
+ _mm512_set1_ps(max)));
+ _mm512_storeu_ps(y + i, val);
+ sum += _mm512_reduce_add_ps(val);
+ }
+ }
+#endif
+
+#if defined(__AVX2__) && defined(__FMA__)
+ if (!FLAG_trap) {
+ for (; i + 7 < n; i += 8) {
+ __m256 val = llamafile_expf_avx2fma(_mm256_sub_ps(_mm256_loadu_ps(x + i),
+ _mm256_set1_ps(max)));
+ _mm256_storeu_ps(y + i, val);
+ __m128 v2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),
+ _mm256_castps256_ps128(val));
+ v2 = _mm_add_ps(v2, _mm_movehl_ps(v2, v2));
+ v2 = _mm_add_ss(v2, _mm_movehdup_ps(v2));
+ sum += _mm_cvtss_f32(v2);
+ }
+ }
+#endif
+
+#ifdef __ARM_NEON
+ for (; i + 3 < n; i += 4) {
+ float32x4_t val = llamafile_expf_neon(vsubq_f32(vld1q_f32(x + i), vdupq_n_f32(max)));
+ vst1q_f32(y + i, val);
+ sum += vaddvq_f32(val);
+ }
+#endif
+
+ for (; i < n; ++i) {
+ float val = expf(x[i] - max);
+ sum += (ggml_float)val;
+ y[i] = val;
+ }
+ return sum;
+}
diff --git a/llama.cpp/ggml-vector.py b/llama.cpp/ggml-vector.py
index 528bf60d..31f20426 100644
--- a/llama.cpp/ggml-vector.py
+++ b/llama.cpp/ggml-vector.py
@@ -3,7 +3,7 @@ import re
with open('llama.cpp/ggml-vector.h') as f:
prototypes = f.read()
prototypes = [line.replace(';', '') for line in prototypes.split('\n')
- if line.endswith(';') and not line.startswith('//')]
+ if line.endswith(';') and not line.startswith('//') and '(' in line]
FUNCS = [(re.search(r'(?<= )\w+', proto).group(0), proto)
for proto in prototypes]
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 5eaf8852..338f80be 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -11867,14 +11867,7 @@ static void ggml_compute_forward_soft_max_f32(
float max = -INFINITY;
ggml_vec_max_f32(nc, &max, wp);
- ggml_float sum = 0.0;
-
- for (int i = 0; i < nc; i++) {
- const float val = expf(wp[i] - max);
- sum += (ggml_float)val;
- dp[i] = val;
- }
-
+ float sum = ggml_vec_soft_max_f32(nc, dp, wp, max);
assert(sum > 0.0);
sum = 1.0/sum;
diff --git a/llamafile/debug.cpp b/llamafile/debug.cpp
index 15512354..c68788b0 100644
--- a/llamafile/debug.cpp
+++ b/llamafile/debug.cpp
@@ -30,15 +30,13 @@
#include "llama.cpp/ggml.h"
-static int get_config(void) {
- int config = FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW;
- // TODO(jart): enable if quantization isn't being used
- // or possibly suppress exceptions in quants
- // config |= FE_UNDERFLOW;
- return config;
-}
+#define TRAPS (FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW)
+#define UNDERFLOW_ALARM "\e[s\e[H\e[7;1;49;31mUNDERFLOW\e[0m\e[u"
+#define UNDERFLOW_RESET "\e[s\e[H \e[u"
+#define UNDERFLOW_DELAY 2
bool FLAG_trap;
+static atomic_llong g_underflowed;
static thread_local int g_enabled;
thread_local int llamafile_debug_op_index;
const struct ggml_cgraph *llamafile_debug_graph;
@@ -46,15 +44,34 @@ const struct ggml_cgraph *llamafile_debug_graph;
static struct TerminalBuddy {
TerminalBuddy() {
active = !tcgetattr(0, &state);
+ is_terminal = active && isatty(2);
}
void restore() {
if (active)
tcsetattr(0, TCSANOW, &state);
}
bool active;
+ bool is_terminal;
struct termios state;
} g_terminal_buddy;
+static long long millis(void) {
+ return timespec_tomillis(timespec_real());
+}
+
+static inline void spinlock(atomic_uint *lock) {
+ int x;
+ for (;;) {
+ x = atomic_exchange_explicit(lock, 1, memory_order_acquire);
+ if (!x)
+ break;
+ }
+}
+
+static inline void spunlock(atomic_uint *lock) {
+ atomic_store_explicit(lock, 0, memory_order_release);
+}
+
static const char *describe_vertex(struct ggml_tensor *t) {
if (t->op == GGML_OP_UNARY)
return ggml_unary_op_name((enum ggml_unary_op)t->op_params[0]);
@@ -99,41 +116,70 @@ int fedisableexcept(int excepts) {
return 0;
}
-static inline void spinlock(atomic_uint *lock) {
- int x;
- for (;;) {
- x = atomic_exchange_explicit(lock, 1, memory_order_acquire);
- if (!x)
- break;
+// recover from trap so that execution may resume
+// without this the same signal will just keep getting raised
+static void recover(ucontext_t *ctx, int traps) {
+#ifdef __x86_64__
+ if (ctx->uc_mcontext.fpregs) {
+ ctx->uc_mcontext.fpregs->mxcsr |= traps << 7; // disable traps
+ ctx->uc_mcontext.fpregs->mxcsr &= ~traps; // clear cages
+ return;
}
-}
-
-static inline void spunlock(atomic_uint *lock) {
- atomic_store_explicit(lock, 0, memory_order_release);
+#elif defined(__aarch64__)
+ struct _aarch64_ctx *ac;
+ for (ac = (struct _aarch64_ctx *)ctx->uc_mcontext.__reserved; ac->magic;
+ ac = (struct _aarch64_ctx *)((char *)ac + ac->size)) {
+ if (ac->magic == FPSIMD_MAGIC) {
+ struct fpsimd_context *sm = (struct fpsimd_context *)ac;
+ sm->fpcr &= ~(traps << 8); // disable traps
+ sm->fpsr &= ~traps; // clear cages
+ return;
+ }
+ }
+#endif
+ // time to die
+ g_terminal_buddy.restore();
+ _exit(128 + SIGFPE);
}
static void on_sigfpe(int sig, siginfo_t *si, void *arg) {
+ ucontext_t *ctx = (ucontext_t *)arg;
+ int reason = si->si_code;
+
+ // underflows are something we expect to happen, particularly with
+ // quantization, so we'll display a check engine light whenever it
+ // happens, provided we're in teletypewriter mode.
+ if (reason == FPE_FLTUND) {
+ if (g_terminal_buddy.is_terminal) {
+ long long now = millis();
+ if ((now - atomic_exchange_explicit(&g_underflowed, now, memory_order_relaxed)) >
+ UNDERFLOW_DELAY) {
+ write(2, UNDERFLOW_ALARM, strlen(UNDERFLOW_ALARM));
+ }
+ }
+ recover(ctx, FE_UNDERFLOW);
+ return;
+ }
static atomic_uint lock;
spinlock(&lock);
const char *issue;
- ucontext_t *ctx = (ucontext_t *)arg;
- if (si->si_code == FPE_INTDIV)
+ if (reason == FPE_INTDIV)
issue = "integer divide by zero";
- else if (si->si_code == FPE_INTOVF)
+ else if (reason == FPE_INTOVF)
issue = "integer overflow";
- else if (si->si_code == FPE_FLTDIV)
+ else if (reason == FPE_FLTDIV)
issue = "floating point divide by zero";
- else if (si->si_code == FPE_FLTOVF)
+ else if (reason == FPE_FLTOVF)
issue = "floating point overflow";
- else if (si->si_code == FPE_FLTUND)
+ else if (reason == FPE_FLTUND)
issue = "floating point underflow";
- else if (si->si_code == FPE_FLTRES)
+ else if (reason == FPE_FLTRES)
issue = "floating point inexact";
- else if (si->si_code == FPE_FLTINV)
+ else if (reason == FPE_FLTINV)
issue = "invalid floating point operation";
- else if (si->si_code == FPE_FLTSUB)
+ else if (reason == FPE_FLTSUB)
issue = "subscript out of range";
else
issue = "sigfpe";
@@ -165,33 +211,8 @@ static void on_sigfpe(int sig, siginfo_t *si, void *arg) {
once = true;
}
- // recover from trap so that execution may resume
- // without this the same signal will just keep getting raised
- int traps = get_config();
-#ifdef __x86_64__
- if (ctx->uc_mcontext.fpregs) {
- ctx->uc_mcontext.fpregs->mxcsr |= traps << 7; // disable traps
- ctx->uc_mcontext.fpregs->mxcsr &= ~traps; // clear cages
- spunlock(&lock);
- return;
- }
-#elif defined(__aarch64__)
- struct _aarch64_ctx *ac;
- for (ac = (struct _aarch64_ctx *)ctx->uc_mcontext.__reserved; ac->magic;
- ac = (struct _aarch64_ctx *)((char *)ac + ac->size)) {
- if (ac->magic == FPSIMD_MAGIC) {
- struct fpsimd_context *sm = (struct fpsimd_context *)ac;
- sm->fpcr &= ~(traps << 8); // disable traps
- sm->fpsr &= ~traps; // clear cages
- spunlock(&lock);
- return;
- }
- }
-#endif
-
- // time to die
- g_terminal_buddy.restore();
- _exit(128 + SIGFPE);
+ recover(ctx, TRAPS);
+ spunlock(&lock);
}
static void setup_sigfpe(void) {
@@ -209,14 +230,13 @@ int llamafile_trapping_enabled(int delta) {
static atomic_uint once;
bool was_enabled = g_enabled > 0;
bool is_enabled = (g_enabled += delta) > 0;
- int config = get_config();
feclearexcept(FE_ALL_EXCEPT);
if (is_enabled && !was_enabled) {
cosmo_once(&once, setup_sigfpe);
- feenableexcept(config);
+ feenableexcept(TRAPS);
}
if (!is_enabled && was_enabled) {
- fedisableexcept(config);
+ fedisableexcept(TRAPS);
}
return g_enabled;
}
@@ -224,6 +244,16 @@ int llamafile_trapping_enabled(int delta) {
void llamafile_trapping_restore(void) {
if (g_enabled > 0) {
feclearexcept(FE_ALL_EXCEPT);
- feenableexcept(get_config());
+ feenableexcept(TRAPS);
+ long long last;
+ if (g_terminal_buddy.is_terminal &&
+ (last = atomic_load_explicit(&g_underflowed, memory_order_relaxed))) {
+ long long now = millis();
+ if (now - last > UNDERFLOW_DELAY &&
+ now - atomic_exchange_explicit(&g_underflowed, 0, memory_order_relaxed) >
+ UNDERFLOW_DELAY) {
+ write(2, UNDERFLOW_RESET, strlen(UNDERFLOW_RESET));
+ }
+ }
}
}
diff --git a/llamafile/expf.h b/llamafile/expf.h
new file mode 100644
index 00000000..20082954
--- /dev/null
+++ b/llamafile/expf.h
@@ -0,0 +1,146 @@
+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2023 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#ifdef __AVX512F__
+#include <immintrin.h>
+
+// computes expf() for each element in vector.
+//
+// the maximum error is 1.45358 +0.5 ulp. the only difference between
+// this function and expf(), is that there's currently no support for
+// subnormals. input values are clamped to range: [-87.6831, 88.3763]
+// whereas expf() allows inputs as low as -103.972. therefore numbers
+// will be flushed to zero sooner than they otherwise would with this
+// function. nearest rounding mode is always used. exception trapping
+// isn't supported although this function does a good job avoiding it
+//
+static inline __m512 llamafile_expf_avx512(__m512 x) {
+ __m512 a, b, c, d, e, f, g;
+ __m512 will_turn_into_inf = _mm512_set1_ps(0x1.62e44p+6f);
+ __m512 max_before_overflow = _mm512_set1_ps(0x1.61814cp+6f);
+ __m512 min_before_underflow = _mm512_set1_ps(-0x1.5ebb86p+6f);
+ x = _mm512_mask_blend_ps(_mm512_cmp_ps_mask(x, max_before_overflow, _CMP_GE_OQ), x,
+ will_turn_into_inf);
+ x = _mm512_mask_blend_ps(_mm512_cmp_ps_mask(x, min_before_underflow, _CMP_LE_OQ), x,
+ min_before_underflow);
+ a = _mm512_fmadd_round_ps(_mm512_set1_ps(0x1.715476p+0f), x, _mm512_set1_ps(0x1.8p23f),
+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
+ b = _mm512_sub_ps(a, _mm512_set1_ps(0x1.8p23f));
+ c = _mm512_fnmadd_round_ps(_mm512_set1_ps(0x1.62e4p-1f), b, x,
+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
+ c = _mm512_fnmadd_round_ps(_mm512_set1_ps(0x1.7f7d1cp-20f), b, c,
+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
+ d = _mm512_castsi512_ps(_mm512_add_epi32(_mm512_slli_epi32(_mm512_castps_si512(a), 23),
+ _mm512_set1_epi32(0x3f800000u)));
+ e = _mm512_mul_round_ps(c, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
+ f = _mm512_fmadd_round_ps(_mm512_set1_ps(0x1.0e4020p-7f), c, _mm512_set1_ps(0x1.573e2ep-5f),
+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
+ g = _mm512_fmadd_round_ps(_mm512_set1_ps(0x1.555e66p-3f), c, _mm512_set1_ps(0x1.fffdb6p-2f),
+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
+ g = _mm512_fmadd_round_ps(f, e, g, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
+ f = _mm512_mul_round_ps(_mm512_set1_ps(0x1.ffffecp-1f), c,
+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
+ return _mm512_fmadd_round_ps(
+ _mm512_fmadd_round_ps(g, e, f, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC), d, d,
+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
+}
+
+// computes silu x/(1+exp(-x)) in single precision
+static inline __m512 llamafile_silu_avx512(__m512 x) {
+ __m512 one = _mm512_set1_ps(1);
+ __m512 zero = _mm512_setzero_ps();
+ __m512 neg_x = _mm512_sub_ps(zero, x);
+ __m512 exp_neg_x = llamafile_expf_avx512(neg_x);
+ __m512 one_plus_exp_neg_x = _mm512_add_ps(one, exp_neg_x);
+ return _mm512_div_ps(x, one_plus_exp_neg_x);
+}
+
+#endif // __AVX512F__
+
+#if defined(__AVX2__) && defined(__FMA__)
+#include <immintrin.h>
+
+// computes expf() for each element in vector.
+//
+// the maximum error is 1.45358 +0.5 ulp. the only difference between
+// this function and expf(), is that there's currently no support for
+// subnormals. input values are clamped to range: [-87.6831, 88.3763]
+// whereas expf() allows inputs as low as -103.972. therefore numbers
+// will be flushed to zero sooner than they otherwise would with this
+// function. exception trapping isnt supported although this function
+// does a good job avoiding it.
+//
+static inline __m256 llamafile_expf_avx2fma(__m256 x) {
+ __m256 a, b, c, d, e, f, g;
+ __m256 will_turn_into_inf = _mm256_set1_ps(0x1.62e44p+6f);
+ __m256 max_before_overflow = _mm256_set1_ps(0x1.61814cp+6f);
+ __m256 min_before_underflow = _mm256_set1_ps(-0x1.5ebb86p+6f);
+ __m256 min_mask = _mm256_cmp_ps(x, min_before_underflow, _CMP_LE_OQ);
+ __m256 max_mask = _mm256_cmp_ps(x, max_before_overflow, _CMP_GE_OQ);
+ x = _mm256_or_ps(_mm256_and_ps(min_mask, min_before_underflow), _mm256_andnot_ps(min_mask, x));
+ x = _mm256_or_ps(_mm256_and_ps(max_mask, will_turn_into_inf), _mm256_andnot_ps(max_mask, x));
+ a = _mm256_fmadd_ps(_mm256_set1_ps(0x1.715476p+0f), x, _mm256_set1_ps(0x1.8p23f));
+ b = _mm256_sub_ps(a, _mm256_set1_ps(0x1.8p23f));
+ c = _mm256_fnmadd_ps(_mm256_set1_ps(0x1.62e4p-1f), b, x);
+ c = _mm256_fnmadd_ps(_mm256_set1_ps(0x1.7f7d1cp-20f), b, c);
+ d = _mm256_castsi256_ps(_mm256_add_epi32(_mm256_slli_epi32(_mm256_castps_si256(a), 23),
+ _mm256_set1_epi32(0x3f800000u)));
+ e = _mm256_mul_ps(c, c);
+ f = _mm256_fmadd_ps(_mm256_set1_ps(0x1.0e4020p-7f), c, _mm256_set1_ps(0x1.573e2ep-5f));
+ g = _mm256_fmadd_ps(_mm256_set1_ps(0x1.555e66p-3f), c, _mm256_set1_ps(0x1.fffdb6p-2f));
+ g = _mm256_fmadd_ps(f, e, g);
+ f = _mm256_mul_ps(_mm256_set1_ps(0x1.ffffecp-1f), c);
+ return _mm256_fmadd_ps(_mm256_fmadd_ps(g, e, f), d, d);
+}
+
+// computes silu x/(1+exp(-x)) in single precision
+static inline __m256 llamafile_silu_avx2fma(__m256 x) {
+ __m256 one = _mm256_set1_ps(1);
+ __m256 zero = _mm256_setzero_ps();
+ __m256 neg_x = _mm256_sub_ps(zero, x);
+ __m256 exp_neg_x = llamafile_expf_avx2fma(neg_x);
+ __m256 one_plus_exp_neg_x = _mm256_add_ps(one, exp_neg_x);
+ return _mm256_div_ps(x, one_plus_exp_neg_x);
+}
+
+#endif // __AVX2__
+
+#ifdef __ARM_NEON
+#include <arm_neon.h>
+
+float32x4_t v_expf(float32x4_t);
+
+static inline float32x4_t llamafile_expf_neon(float32x4_t x) {
+ return v_expf(x);
+}
+
+static inline float32x4_t llamafile_silu_neon(float32x4_t x) {
+ float32x4_t one = vdupq_n_f32(1.0f);
+ float32x4_t zero = vdupq_n_f32(0.0f);
+ float32x4_t neg_x = vsubq_f32(zero, x);
+ float32x4_t exp_neg_x = llamafile_expf_neon(neg_x);
+ float32x4_t one_plus_exp_neg_x = vaddq_f32(one, exp_neg_x);
+ return vdivq_f32(x, one_plus_exp_neg_x);
+}
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/llamafile/v_expf.c b/llamafile/v_expf.c
new file mode 100644
index 00000000..4127df92
--- /dev/null
+++ b/llamafile/v_expf.c
@@ -0,0 +1,145 @@
+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 1999-2022 Arm Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __aarch64__
+
+#include <arm_neon.h>
+
+#define V4(X) {X, X, X, X}
+#define NOINLINE __attribute__((__noinline__))
+#define VPCS_ATTR __attribute__((aarch64_vector_pcs))
+#define likely(x) __builtin_expect(!!(x), 1)
+#define unlikely(x) __builtin_expect(x, 0)
+
+static inline uint32x4_t v_u32(uint32_t x) {
+ return (uint32x4_t)V4(x);
+}
+
+/* true if any elements of a vector compare result is non-zero. */
+static inline int v_any_u32(uint32x4_t x) {
+ /* assume elements in x are either 0 or -1u. */
+ return vpaddd_u64(vreinterpretq_u64_u32(x)) != 0;
+}
+
+static const struct data {
+ float32x4_t poly[5];
+ float32x4_t shift, inv_ln2, ln2_hi, ln2_lo;
+ uint32x4_t exponent_bias;
+#if !WANT_SIMD_EXCEPT
+ float32x4_t special_bound, scale_thresh;
+#endif
+} data = {
+ /* maxerr: 1.45358 +0.5 ulp. */
+ .poly = {V4(0x1.0e4020p-7f), V4(0x1.573e2ep-5f), V4(0x1.555e66p-3f), V4(0x1.fffdb6p-2f),
+ V4(0x1.ffffecp-1f)},
+ .shift = V4(0x1.8p23f),
+ .inv_ln2 = V4(0x1.715476p+0f),
+ .ln2_hi = V4(0x1.62e4p-1f),
+ .ln2_lo = V4(0x1.7f7d1cp-20f),
+ .exponent_bias = V4(0x3f800000),
+#if !WANT_SIMD_EXCEPT
+ .special_bound = V4(126.0f),
+ .scale_thresh = V4(192.0f),
+#endif
+};
+
+#define C(i) d->poly[i]
+
+#if WANT_SIMD_EXCEPT
+
+#define TinyBound v_u32(0x20000000) /* asuint (0x1p-63). */
+#define BigBound v_u32(0x42800000) /* asuint (0x1p6). */
+#define SpecialBound v_u32(0x22800000) /* BigBound - TinyBound. */
+
+static float32x4_t VPCS_ATTR NOINLINE special_case(float32x4_t x, float32x4_t y, uint32x4_t cmp) {
+ /* If fenv exceptions are to be triggered correctly, fall back to the scalar
+ routine to special lanes. */
+ return v_call_f32(expf, x, y, cmp);
+}
+
+#else
+
+#define SpecialOffset v_u32(0x82000000)
+#define SpecialBias v_u32(0x7f000000)
+
+static float32x4_t VPCS_ATTR NOINLINE special_case(float32x4_t poly, float32x4_t n, uint32x4_t e,
+ uint32x4_t cmp1, float32x4_t scale,
+ const struct data *d) {
+ /* 2^n may overflow, break it up into s1*s2. */
+ uint32x4_t b = vandq_u32(vclezq_f32(n), SpecialOffset);
+ float32x4_t s1 = vreinterpretq_f32_u32(vaddq_u32(b, SpecialBias));
+ float32x4_t s2 = vreinterpretq_f32_u32(vsubq_u32(e, b));
+ uint32x4_t cmp2 = vcagtq_f32(n, d->scale_thresh);
+ float32x4_t r2 = vmulq_f32(s1, s1);
+ float32x4_t r1 = vmulq_f32(vfmaq_f32(s2, poly, s2), s1);
+ /* Similar to r1 but avoids double rounding in the subnormal range. */
+ float32x4_t r0 = vfmaq_f32(scale, poly, scale);
+ float32x4_t r = vbslq_f32(cmp1, r1, r0);
+ return vbslq_f32(cmp2, r2, r);
+}
+
+#endif
+
+float32x4_t v_expf(float32x4_t x) {
+ const struct data *d = &data;
+ __asm__("" : "+r"(d));
+ float32x4_t n, r, r2, scale, p, q, poly, z;
+ uint32x4_t cmp, e;
+
+#if WANT_SIMD_EXCEPT
+ /* asuint(x) - TinyBound >= BigBound - TinyBound. */
+ cmp = vcgeq_u32(vsubq_u32(vandq_u32(vreinterpretq_u32_f32(x), v_u32(0x7fffffff)), TinyBound),
+ SpecialBound);
+ float32x4_t xm = x;
+ /* If any lanes are special, mask them with 1 and retain a copy of x to allow
+ special case handler to fix special lanes later. This is only necessary if
+ fenv exceptions are to be triggered correctly. */
+ if (unlikely(v_any_u32(cmp)))
+ x = vbslq_f32(cmp, v_f32(1), x);
+#endif
+
+ /* exp(x) = 2^n (1 + poly(r)), with 1 + poly(r) in [1/sqrt(2),sqrt(2)]
+ x = ln2*n + r, with r in [-ln2/2, ln2/2]. */
+ z = vfmaq_f32(d->shift, x, d->inv_ln2);
+ n = vsubq_f32(z, d->shift);
+ r = vfmsq_f32(x, n, d->ln2_hi);
+ r = vfmsq_f32(r, n, d->ln2_lo);
+ e = vshlq_n_u32(vreinterpretq_u32_f32(z), 23);
+ scale = vreinterpretq_f32_u32(vaddq_u32(e, d->exponent_bias));
+
+#if !WANT_SIMD_EXCEPT
+ cmp = vcagtq_f32(n, d->special_bound);
+#endif
+
+ r2 = vmulq_f32(r, r);
+ p = vfmaq_f32(C(1), C(0), r);
+ q = vfmaq_f32(C(3), C(2), r);
+ q = vfmaq_f32(q, p, r2);
+ p = vmulq_f32(C(4), r);
+ poly = vfmaq_f32(p, q, r2);
+
+ if (unlikely(v_any_u32(cmp)))
+#if WANT_SIMD_EXCEPT
+ return special_case(xm, vfmaq_f32(scale, poly, scale), cmp);
+#else
+ return special_case(poly, n, e, cmp, scale, d);
+#endif
+
+ return vfmaq_f32(scale, poly, scale);
+}
+
+#endif // __aarch64__