Skip to content

Distribute and run LLMs with a single file.

Implement the real trick to GeLU with proof

Justine Tunneyb5748f36727c
diff --git a/llama.cpp/ggml-vector.inc b/llama.cpp/ggml-vector.inc
index 04be033a..cb40b9d6 100644
--- a/llama.cpp/ggml-vector.inc
+++ b/llama.cpp/ggml-vector.inc
@@ -14,8 +14,6 @@
#error "ref: https://github.com/ggerganov/llama.cpp/pull/7154#issuecomment-2143844461"
#endif
-/* #define GGML_GELU_FP16 */
-
#define UNUSED GGML_UNUSED
#define GGML_SOFT_MAX_UNROLL 4
@@ -1302,27 +1300,6 @@ inline static float ggml_gelu_f32(float x) {
return .5f*x*(1.f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));
}
-#ifdef GGML_GELU_FP16
-
-// precomputed gelu table in brain16 (128 KB)
-static atomic_uint ggml_table_gelu_once;
-static ggml_fp16_t ggml_table_gelu_f16[1 << 16];
-
-static void ggml_gelu_init(void) {
- llamafile_trapping_enabled(-1);
- for (int i = 0; i < 65536; ++i) {
- union {
- uint16_t i;
- ggml_fp16_t f;
- } u = {i};
- ggml_table_gelu_f16[i] = GGML_COMPUTE_FP32_TO_FP16(
- ggml_gelu_f32(GGML_COMPUTE_FP16_TO_FP32(u.f)));
- }
- llamafile_trapping_enabled(+1);
-}
-
-#endif // GGML_GELU_FP16
-
#if defined(__ARM_NEON) && defined(__aarch64__)
/* Approximation for single-precision vector tanh (2.58 ULP)
@@ -1337,23 +1314,18 @@ ggml_vtanhf(float32x4_t x)
const uint32x4_t iax = vreinterpretq_u32_f32(ax);
const uint32x4_t sign = veorq_u32(ix, iax);
const uint32x4_t is_boring = vcgtq_u32(iax, vdupq_n_u32(0x41102cb3));
- const float32x4_t boring =
- vreinterpretq_f32_u32(vorrq_u32(sign, vdupq_n_u32(0x3f800000)));
+ const float32x4_t boring = vreinterpretq_f32_u32(vorrq_u32(sign, vdupq_n_u32(0x3f800000)));
const uint32x4_t special = vcgtq_u32(iax, vdupq_n_u32(0x7f800000));
const float32x4_t ex = vmulq_n_f32(x, 2);
const float32x4_t e = { 0x1.715476p+0f, 0x1.62e4p-1f, 0x1.7f7d1cp-20f };
- const float32x4_t j =
- vsubq_f32(vfmaq_laneq_f32(vdupq_n_f32(0x1.8p23f), ex, e, 0),
- vdupq_n_f32(0x1.8p23f));
+ const float32x4_t j = vsubq_f32(vfmaq_laneq_f32(vdupq_n_f32(0x1.8p23f), ex, e, 0), vdupq_n_f32(0x1.8p23f));
const int32x4_t i = vcvtq_s32_f32(j);
const float32x4_t f = vfmsq_laneq_f32(ex, j, e, 1);
const float32x4_t f1 = vfmsq_laneq_f32(f, j, e, 2);
const float32x4_t f2 = vmulq_f32(f1, f1);
const float32x4_t f4 = vmulq_f32(f2, f2);
- const float32x4_t p01 =
- vfmaq_f32(vdupq_n_f32(0x1.fffffep-2), vdupq_n_f32(0x1.5554aep-3), f1);
- const float32x4_t p23 =
- vfmaq_f32(vdupq_n_f32(0x1.555736p-5), vdupq_n_f32(0x1.12287cp-7), f1);
+ const float32x4_t p01 = vfmaq_f32(vdupq_n_f32(0x1.fffffep-2), vdupq_n_f32(0x1.5554aep-3), f1);
+ const float32x4_t p23 = vfmaq_f32(vdupq_n_f32(0x1.555736p-5), vdupq_n_f32(0x1.12287cp-7), f1);
const float32x4_t p03 = vfmaq_f32(p01, p23, f2);
const float32x4_t p = vfmaq_f32(p03, vdupq_n_f32(0x1.6b55a2p-10), f4);
const float32x4_t p2 = vfmaq_f32(f1, f2, p);
@@ -1380,8 +1352,7 @@ ggml_vgeluf(float32x4_t x)
const float32x4_t x_squared = vmulq_f32(x, x);
const float32x4_t ax2 = vmulq_f32(coef_a, x_squared);
const float32x4_t one_plus_ax2 = vaddq_f32(one, ax2);
- const float32x4_t inner =
- vmulq_f32(vmulq_f32(sqrt_2_over_pi, x), one_plus_ax2);
+ const float32x4_t inner = vmulq_f32(vmulq_f32(sqrt_2_over_pi, x), one_plus_ax2);
const float32x4_t tanh_inner = ggml_vtanhf(inner);
const float32x4_t one_plus_tanh = vaddq_f32(one, tanh_inner);
return vmulq_f32(vmulq_f32(half, x), one_plus_tanh);
@@ -1408,27 +1379,22 @@ ggml_vtanhf(__m512 x)
const __m512 two = _mm512_set1_ps(2);
const __m512 ax = _mm512_abs_ps(x);
const __m512 sign = _mm512_and_ps(x, sign_mask);
- const __mmask16 is_boring =
- _mm512_cmp_ps_mask(ax, _mm512_set1_ps(0x1.205966p+3), _CMP_GT_OQ);
+ const __mmask16 is_boring = _mm512_cmp_ps_mask(ax, _mm512_set1_ps(0x1.205966p+3), _CMP_GT_OQ);
const __m512 boring = _mm512_or_ps(sign, one);
const __m512 ex = _mm512_mul_ps(x, two);
- const __m512 j = _mm512_fmadd_ps(
- ex, _mm512_set1_ps(0x1.715476p+0f), _mm512_set1_ps(0x1.8p23f));
+ const __m512 j = _mm512_fmadd_ps( ex, _mm512_set1_ps(0x1.715476p+0f), _mm512_set1_ps(0x1.8p23f));
const __m512 jj = _mm512_sub_ps(j, _mm512_set1_ps(0x1.8p23f));
const __m512i i = _mm512_cvttps_epi32(jj);
const __m512 f = _mm512_fnmadd_ps(_mm512_set1_ps(0x1.62e4p-1f), jj, ex);
const __m512 f1 = _mm512_fnmadd_ps(_mm512_set1_ps(0x1.7f7d1cp-20f), jj, f);
const __m512 f2 = _mm512_mul_ps(f1, f1);
const __m512 f4 = _mm512_mul_ps(f2, f2);
- const __m512 p01 = _mm512_fmadd_ps(
- f1, _mm512_set1_ps(0x1.5554aep-3), _mm512_set1_ps(0x1.fffffep-2));
- const __m512 p23 = _mm512_fmadd_ps(
- f1, _mm512_set1_ps(0x1.12287cp-7), _mm512_set1_ps(0x1.555736p-5));
+ const __m512 p01 = _mm512_fmadd_ps( f1, _mm512_set1_ps(0x1.5554aep-3), _mm512_set1_ps(0x1.fffffep-2));
+ const __m512 p23 = _mm512_fmadd_ps( f1, _mm512_set1_ps(0x1.12287cp-7), _mm512_set1_ps(0x1.555736p-5));
const __m512 p03 = _mm512_fmadd_ps(f2, p23, p01);
const __m512 p = _mm512_fmadd_ps(f4, _mm512_set1_ps(0x1.6b55a2p-10), p03);
const __m512 p2 = _mm512_fmadd_ps(f2, p, f1);
- const __m512i u =
- _mm512_add_epi32(_mm512_slli_epi32(i, 23), _mm512_set1_epi32(0x3f800000));
+ const __m512i u = _mm512_add_epi32(_mm512_slli_epi32(i, 23), _mm512_set1_epi32(0x3f800000));
const __m512 t = _mm512_castsi512_ps(u);
const __m512 q = _mm512_fmadd_ps(p2, t, _mm512_sub_ps(t, one));
const __m512 y = _mm512_div_ps(q, _mm512_add_ps(q, two));
@@ -1445,8 +1411,7 @@ ggml_vgeluf(__m512 x)
const __m512 x_squared = _mm512_mul_ps(x, x);
const __m512 ax2 = _mm512_mul_ps(coef_a, x_squared);
const __m512 one_plus_ax2 = _mm512_add_ps(one, ax2);
- const __m512 inner =
- _mm512_mul_ps(_mm512_mul_ps(sqrt_2_over_pi, x), one_plus_ax2);
+ const __m512 inner = _mm512_mul_ps(_mm512_mul_ps(sqrt_2_over_pi, x), one_plus_ax2);
const __m512 tanh_inner = ggml_vtanhf(inner);
const __m512 one_plus_tanh = _mm512_add_ps(one, tanh_inner);
return _mm512_mul_ps(_mm512_mul_ps(half, x), one_plus_tanh);
@@ -1489,32 +1454,26 @@ ggml_vtanhf(__m256 x)
const __m256 two = _mm256_set1_ps(2);
const __m256 ax = _mm256_and_ps(x, abs_mask);
const __m256 sign = _mm256_and_ps(x, _mm256_set1_ps(-0.f));
- const __m256 is_boring =
- _mm256_cmp_ps(ax, _mm256_set1_ps(0x1.205966p+3), _CMP_GT_OQ);
+ const __m256 is_boring = _mm256_cmp_ps(ax, _mm256_set1_ps(0x1.205966p+3), _CMP_GT_OQ);
const __m256 boring = _mm256_or_ps(sign, one);
const __m256 ex = _mm256_mul_ps(x, two);
- const __m256 j =
- MADD256(ex, _mm256_set1_ps(0x1.715476p+0f), _mm256_set1_ps(0x1.8p23f));
+ const __m256 j = MADD256(ex, _mm256_set1_ps(0x1.715476p+0f), _mm256_set1_ps(0x1.8p23f));
const __m256 jj = _mm256_sub_ps(j, _mm256_set1_ps(0x1.8p23f));
const __m256i i = _mm256_cvttps_epi32(jj);
const __m256 f = NMADD256(_mm256_set1_ps(0x1.62e4p-1f), jj, ex);
const __m256 f1 = NMADD256(_mm256_set1_ps(0x1.7f7d1cp-20f), jj, f);
const __m256 f2 = _mm256_mul_ps(f1, f1);
const __m256 f4 = _mm256_mul_ps(f2, f2);
- const __m256 p01 =
- MADD256(f1, _mm256_set1_ps(0x1.5554aep-3), _mm256_set1_ps(0x1.fffffep-2));
- const __m256 p23 =
- MADD256(f1, _mm256_set1_ps(0x1.12287cp-7), _mm256_set1_ps(0x1.555736p-5));
+ const __m256 p01 = MADD256(f1, _mm256_set1_ps(0x1.5554aep-3), _mm256_set1_ps(0x1.fffffep-2));
+ const __m256 p23 = MADD256(f1, _mm256_set1_ps(0x1.12287cp-7), _mm256_set1_ps(0x1.555736p-5));
const __m256 p03 = MADD256(f2, p23, p01);
const __m256 p = MADD256(f4, _mm256_set1_ps(0x1.6b55a2p-10), p03);
const __m256 p2 = MADD256(f2, p, f1);
- const __m256i u =
- _mm256_add_epi32(_mm256_slli_epi32(i, 23), _mm256_set1_epi32(0x3f800000));
+ const __m256i u = _mm256_add_epi32(_mm256_slli_epi32(i, 23), _mm256_set1_epi32(0x3f800000));
const __m256 t = _mm256_castsi256_ps(u);
const __m256 q = MADD256(p2, t, _mm256_sub_ps(t, one));
const __m256 y = _mm256_div_ps(q, _mm256_add_ps(q, two));
- return _mm256_or_ps(_mm256_and_ps(is_boring, boring),
- _mm256_andnot_ps(is_boring, y));
+ return _mm256_or_ps(_mm256_and_ps(is_boring, boring), _mm256_andnot_ps(is_boring, y));
}
inline static __m256
@@ -1527,8 +1486,7 @@ ggml_vgeluf(__m256 x)
const __m256 x_squared = _mm256_mul_ps(x, x);
const __m256 ax2 = _mm256_mul_ps(coef_a, x_squared);
const __m256 one_plus_ax2 = _mm256_add_ps(one, ax2);
- const __m256 inner =
- _mm256_mul_ps(_mm256_mul_ps(sqrt_2_over_pi, x), one_plus_ax2);
+ const __m256 inner = _mm256_mul_ps(_mm256_mul_ps(sqrt_2_over_pi, x), one_plus_ax2);
const __m256 tanh_inner = ggml_vtanhf(inner);
const __m256 one_plus_tanh = _mm256_add_ps(one, tanh_inner);
return _mm256_mul_ps(_mm256_mul_ps(half, x), one_plus_tanh);
@@ -1574,28 +1532,23 @@ ggml_vtanhf(__m128 x)
const __m128 is_boring = _mm_cmpgt_ps(ax, _mm_set1_ps(0x1.205966p+3));
const __m128 boring = _mm_or_ps(sign, one);
const __m128 ex = _mm_mul_ps(x, two);
- const __m128 j =
- MADD128(ex, _mm_set1_ps(0x1.715476p+0f), _mm_set1_ps(0x1.8p23f));
+ const __m128 j = MADD128(ex, _mm_set1_ps(0x1.715476p+0f), _mm_set1_ps(0x1.8p23f));
const __m128 jj = _mm_sub_ps(j, _mm_set1_ps(0x1.8p23f));
const __m128i i = _mm_cvttps_epi32(jj);
const __m128 f = NMADD128(_mm_set1_ps(0x1.62e4p-1f), jj, ex);
const __m128 f1 = NMADD128(_mm_set1_ps(0x1.7f7d1cp-20f), jj, f);
const __m128 f2 = _mm_mul_ps(f1, f1);
const __m128 f4 = _mm_mul_ps(f2, f2);
- const __m128 p01 =
- MADD128(f1, _mm_set1_ps(0x1.5554aep-3), _mm_set1_ps(0x1.fffffep-2));
- const __m128 p23 =
- MADD128(f1, _mm_set1_ps(0x1.12287cp-7), _mm_set1_ps(0x1.555736p-5));
+ const __m128 p01 = MADD128(f1, _mm_set1_ps(0x1.5554aep-3), _mm_set1_ps(0x1.fffffep-2));
+ const __m128 p23 = MADD128(f1, _mm_set1_ps(0x1.12287cp-7), _mm_set1_ps(0x1.555736p-5));
const __m128 p03 = MADD128(f2, p23, p01);
const __m128 p = MADD128(f4, _mm_set1_ps(0x1.6b55a2p-10), p03);
const __m128 p2 = MADD128(f2, p, f1);
- const __m128i u =
- _mm_add_epi32(_mm_slli_epi32(i, 23), _mm_set1_epi32(0x3f800000));
+ const __m128i u = _mm_add_epi32(_mm_slli_epi32(i, 23), _mm_set1_epi32(0x3f800000));
const __m128 t = _mm_castsi128_ps(u);
const __m128 q = MADD128(p2, t, _mm_sub_ps(t, one));
const __m128 y = _mm_div_ps(q, _mm_add_ps(q, two));
- return _mm_or_ps(_mm_and_ps(is_boring, boring),
- _mm_andnot_ps(is_boring, y));
+ return _mm_or_ps(_mm_and_ps(is_boring, boring), _mm_andnot_ps(is_boring, y));
}
inline static __m128
@@ -1608,8 +1561,7 @@ ggml_vgeluf(__m128 x)
const __m128 x_squared = _mm_mul_ps(x, x);
const __m128 ax2 = _mm_mul_ps(coef_a, x_squared);
const __m128 one_plus_ax2 = _mm_add_ps(one, ax2);
- const __m128 inner =
- _mm_mul_ps(_mm_mul_ps(sqrt_2_over_pi, x), one_plus_ax2);
+ const __m128 inner = _mm_mul_ps(_mm_mul_ps(sqrt_2_over_pi, x), one_plus_ax2);
const __m128 tanh_inner = ggml_vtanhf(inner);
const __m128 one_plus_tanh = _mm_add_ps(one, tanh_inner);
return _mm_mul_ps(_mm_mul_ps(half, x), one_plus_tanh);
@@ -1619,51 +1571,78 @@ ggml_vgeluf(__m128 x)
void ggml_vec_gelu_f32(const int n, float * y, const float * x) {
int i = 0;
-
- if (!FLAG_trap) { // [jart] preserve this line
+ if (FLAG_trap) {
+ for (; i < n; ++i) {
+ y[i] = ggml_gelu_f32(x[i]);
+ }
+ return;
+ }
#if defined(__ARM_NEON) && defined(__aarch64__)
for (; i + 3 < n; i += 4) {
vst1q_f32(y + i, ggml_vgeluf(vld1q_f32(x + i)));
}
+ if (i < n) {
+ float temp_x[4] = {0};
+ float temp_y[4] = {0};
+ int rem = n - i;
+ for (int j = 0; j < rem; j++) {
+ temp_x[j] = x[i + j];
+ }
+ float32x4_t x_vec = vld1q_f32(temp_x);
+ float32x4_t y_vec = ggml_vgeluf(x_vec);
+ vst1q_f32(temp_y, y_vec);
+ for (int j = 0; j < rem; j++) {
+ y[i + j] = temp_y[j];
+ }
+ }
#elif defined(__AVX512F__) && defined(__AVX512DQ__)
for (; i + 15 < n; i += 16) {
_mm512_storeu_ps(y + i, ggml_vgeluf(_mm512_loadu_ps(x + i)));
}
+ if (i < n) {
+ __mmask16 mask = _cvtu32_mask16((1U << (n - i)) - 1);
+ __m512 x_vec = _mm512_maskz_loadu_ps(mask, x + i);
+ __m512 y_vec = ggml_vgeluf(x_vec);
+ _mm512_mask_storeu_ps(y + i, mask, y_vec);
+ }
+ return;
#elif defined(__AVX2__)
for (; i + 7 < n; i += 8) {
_mm256_storeu_ps(y + i, ggml_vgeluf(_mm256_loadu_ps(x + i)));
}
+ if (i < n) {
+ __m256i mask = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
+ mask = _mm256_cmpgt_epi32(_mm256_set1_epi32(n - i), mask);
+ __m256 x_vec = _mm256_maskload_ps(x + i, mask);
+ __m256 y_vec = ggml_vgeluf(x_vec);
+ _mm256_maskstore_ps(y + i, mask, y_vec);
+ }
#elif defined(__SSE2__)
for (; i + 3 < n; i += 4) {
_mm_storeu_ps(y + i, ggml_vgeluf(_mm_loadu_ps(x + i)));
}
-#endif
- } // [jart] preserve this line
-
- if (i == n)
- return;
-#ifdef GGML_GELU_FP16
- cosmo_once(&ggml_table_gelu_once, ggml_gelu_init);
-#endif
- for (; i < n; ++i) {
-#ifdef GGML_GELU_FP16
- if (x[i] <= -10.0f) {
- y[i] = 0.0f;
- } else if (x[i] >= 10.0f) {
- y[i] = x[i];
- } else {
- uint16_t t;
- ggml_fp16_t fp16 = GGML_FP32_TO_FP16(x[i]);
- memcpy(&t, &fp16, sizeof(uint16_t));
- y[i] = GGML_FP16_TO_FP32(ggml_table_gelu_f16[t]);
+ if (i < n) {
+ float temp_x[4] = {0};
+ float temp_y[4] = {0};
+ int rem = n - i;
+ for (int j = 0; j < rem; j++) {
+ temp_x[j] = x[i + j];
}
+ __m128 x_vec = _mm_loadu_ps(temp_x);
+ __m128 y_vec = ggml_vgeluf(x_vec);
+ _mm_storeu_ps(temp_y, y_vec);
+ for (int j = 0; j < rem; j++) {
+ y[i + j] = temp_y[j];
+ }
+ }
#else
- // computes canonical gelu approximation with ~32 bit accuracy
+ for (; i < n; ++i) {
y[i] = ggml_gelu_f32(x[i]);
-#endif
}
+#endif
}
+
////////////////////////////////////////////////////////////////////////////////
// Quick GeLU
@@ -1690,7 +1669,7 @@ inline static float ggml_silu_f32(float x) {
// the maximum error is 1.45358 plus 0.5 ulps
// numbers above 88.38 will flush to infinity
// numbers beneath -103.97 will flush to zero
-inline static float32x4_t ggml_v_expf(float32x4_t x) {
+inline static float32x4_t ggml_vexpf(float32x4_t x) {
const float32x4_t r = vdupq_n_f32(0x1.8p23f);
const float32x4_t z = vfmaq_f32(r, x, vdupq_n_f32(0x1.715476p+0f));
const float32x4_t n = vsubq_f32(z, r);
@@ -1714,11 +1693,11 @@ inline static float32x4_t ggml_v_expf(float32x4_t x) {
}
// computes silu x/(1+exp(-x)) in single precision vector
-inline static float32x4_t ggml_v_silu(float32x4_t x) {
+inline static float32x4_t ggml_vsiluf(float32x4_t x) {
const float32x4_t one = vdupq_n_f32(1.0f);
const float32x4_t zero = vdupq_n_f32(0.0f);
const float32x4_t neg_x = vsubq_f32(zero, x);
- const float32x4_t exp_neg_x = ggml_v_expf(neg_x);
+ const float32x4_t exp_neg_x = ggml_vexpf(neg_x);
const float32x4_t one_plus_exp_neg_x = vaddq_f32(one, exp_neg_x);
return vdivq_f32(x, one_plus_exp_neg_x);
}
@@ -1729,7 +1708,7 @@ inline static float32x4_t ggml_v_silu(float32x4_t x) {
// the maximum error is 1.45358 plus 0.5 ulps
// numbers above 88.38 will flush to infinity
// numbers beneath -103.97 will flush to zero
-inline static __m512 ggml_v_expf(__m512 x) {
+inline static __m512 ggml_vexpf(__m512 x) {
const __m512 r = _mm512_set1_ps(0x1.8p23f);
const __m512 z = _mm512_fmadd_ps(x, _mm512_set1_ps(0x1.715476p+0f), r);
const __m512 n = _mm512_sub_ps(z, r);
@@ -1757,11 +1736,11 @@ inline static __m512 ggml_v_expf(__m512 x) {
}
// computes silu x/(1+exp(-x)) in single precision vector
-inline static __m512 ggml_v_silu(__m512 x) {
+inline static __m512 ggml_vsiluf(__m512 x) {
const __m512 one = _mm512_set1_ps(1);
const __m512 zero = _mm512_setzero_ps();
const __m512 neg_x = _mm512_sub_ps(zero, x);
- const __m512 exp_neg_x = ggml_v_expf(neg_x);
+ const __m512 exp_neg_x = ggml_vexpf(neg_x);
const __m512 one_plus_exp_neg_x = _mm512_add_ps(one, exp_neg_x);
return _mm512_div_ps(x, one_plus_exp_neg_x);
}
@@ -1772,7 +1751,7 @@ inline static __m512 ggml_v_silu(__m512 x) {
// the maximum error is 1.45358 plus 0.5 ulps
// numbers above 88.38 will flush to infinity
// numbers beneath -103.97 will flush to zero
-inline static __m256 ggml_v_expf(__m256 x) {
+inline static __m256 ggml_vexpf(__m256 x) {
const __m256 r = _mm256_set1_ps(0x1.8p23f);
const __m256 z = _mm256_fmadd_ps(x, _mm256_set1_ps(0x1.715476p+0f), r);
const __m256 n = _mm256_sub_ps(z, r);
@@ -1812,11 +1791,11 @@ inline static __m256 ggml_v_expf(__m256 x) {
}
// computes silu x/(1+exp(-x)) in single precision vector
-inline static __m256 ggml_v_silu(__m256 x) {
+inline static __m256 ggml_vsiluf(__m256 x) {
const __m256 one = _mm256_set1_ps(1);
const __m256 zero = _mm256_setzero_ps();
const __m256 neg_x = _mm256_sub_ps(zero, x);
- const __m256 exp_neg_x = ggml_v_expf(neg_x);
+ const __m256 exp_neg_x = ggml_vexpf(neg_x);
const __m256 one_plus_exp_neg_x = _mm256_add_ps(one, exp_neg_x);
return _mm256_div_ps(x, one_plus_exp_neg_x);
}
@@ -1835,7 +1814,7 @@ inline static __m256 ggml_v_silu(__m256 x) {
// the maximum error is 1.45358 plus 0.5 ulps
// numbers above 88.38 will flush to infinity
// numbers beneath -103.97 will flush to zero
-inline static __m128 ggml_v_expf(__m128 x) {
+inline static __m128 ggml_vexpf(__m128 x) {
const __m128 r = _mm_set1_ps(0x1.8p23f);
const __m128 z = MADD128(x, _mm_set1_ps(0x1.715476p+0f), r);
const __m128 n = _mm_sub_ps(z, r);
@@ -1866,11 +1845,11 @@ inline static __m128 ggml_v_expf(__m128 x) {
}
// computes silu x/(1+exp(-x)) in single precision vector
-inline static __m128 ggml_v_silu(__m128 x) {
+inline static __m128 ggml_vsiluf(__m128 x) {
const __m128 one = _mm_set1_ps(1);
const __m128 zero = _mm_setzero_ps();
const __m128 neg_x = _mm_sub_ps(zero, x);
- const __m128 exp_neg_x = ggml_v_expf(neg_x);
+ const __m128 exp_neg_x = ggml_vexpf(neg_x);
const __m128 one_plus_exp_neg_x = _mm_add_ps(one, exp_neg_x);
return _mm_div_ps(x, one_plus_exp_neg_x);
}
@@ -1879,28 +1858,75 @@ inline static __m128 ggml_v_silu(__m128 x) {
void ggml_vec_silu_f32(const int n, float * y, const float * x) {
int i = 0;
- if (!FLAG_trap) { // [jart] preserve this line
-#if defined(__AVX512F__) && defined(__AVX512DQ__)
+ if (FLAG_trap) {
+ for (; i < n; ++i) {
+ y[i] = ggml_silu_f32(x[i]);
+ }
+ return;
+ }
+#if defined(__ARM_NEON) && defined(__aarch64__)
+ for (; i + 3 < n; i += 4) {
+ vst1q_f32(y + i, ggml_vsiluf(vld1q_f32(x + i)));
+ }
+ if (i < n) {
+ float temp_x[4] = {0};
+ float temp_y[4] = {0};
+ int rem = n - i;
+ for (int j = 0; j < rem; j++) {
+ temp_x[j] = x[i + j];
+ }
+ float32x4_t x_vec = vld1q_f32(temp_x);
+ float32x4_t y_vec = ggml_vsiluf(x_vec);
+ vst1q_f32(temp_y, y_vec);
+ for (int j = 0; j < rem; j++) {
+ y[i + j] = temp_y[j];
+ }
+ }
+#elif defined(__AVX512F__) && defined(__AVX512DQ__)
for (; i + 15 < n; i += 16) {
- _mm512_storeu_ps(y + i, ggml_v_silu(_mm512_loadu_ps(x + i)));
+ _mm512_storeu_ps(y + i, ggml_vsiluf(_mm512_loadu_ps(x + i)));
}
-#elif defined(__AVX2__) && defined(__FMA__)
+ if (i < n) {
+ __mmask16 mask = _cvtu32_mask16((1U << (n - i)) - 1);
+ __m512 x_vec = _mm512_maskz_loadu_ps(mask, x + i);
+ __m512 y_vec = ggml_vsiluf(x_vec);
+ _mm512_mask_storeu_ps(y + i, mask, y_vec);
+ }
+ return;
+#elif defined(__AVX2__)
for (; i + 7 < n; i += 8) {
- _mm256_storeu_ps(y + i, ggml_v_silu(_mm256_loadu_ps(x + i)));
+ _mm256_storeu_ps(y + i, ggml_vsiluf(_mm256_loadu_ps(x + i)));
}
-#elif defined(__SSE2__)
- for (; i + 3 < n; i += 4) {
- _mm_storeu_ps(y + i, ggml_v_silu(_mm_loadu_ps(x + i)));
+ if (i < n) {
+ __m256i mask = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
+ mask = _mm256_cmpgt_epi32(_mm256_set1_epi32(n - i), mask);
+ __m256 x_vec = _mm256_maskload_ps(x + i, mask);
+ __m256 y_vec = ggml_vsiluf(x_vec);
+ _mm256_maskstore_ps(y + i, mask, y_vec);
}
-#elif defined(__ARM_NEON) && defined(__aarch64__)
+#elif defined(__SSE2__)
for (; i + 3 < n; i += 4) {
- vst1q_f32(y + i, ggml_v_silu(vld1q_f32(x + i)));
+ _mm_storeu_ps(y + i, ggml_vsiluf(_mm_loadu_ps(x + i)));
+ }
+ if (i < n) {
+ float temp_x[4] = {0};
+ float temp_y[4] = {0};
+ int rem = n - i;
+ for (int j = 0; j < rem; j++) {
+ temp_x[j] = x[i + j];
+ }
+ __m128 x_vec = _mm_loadu_ps(temp_x);
+ __m128 y_vec = ggml_vsiluf(x_vec);
+ _mm_storeu_ps(temp_y, y_vec);
+ for (int j = 0; j < rem; j++) {
+ y[i + j] = temp_y[j];
+ }
}
-#endif
- } // [jart] preserve this line
+#else
for (; i < n; ++i) {
y[i] = ggml_silu_f32(x[i]);
}
+#endif
}
ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {
@@ -1909,15 +1935,15 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float
if (!FLAG_trap) { // [jart] preserve this line
#if defined(__AVX512F__) && defined(__AVX512DQ__)
for (; i + 15 < n; i += 16) {
- __m512 val = ggml_v_expf(_mm512_sub_ps(_mm512_loadu_ps(x + i),
- _mm512_set1_ps(max)));
+ __m512 val = ggml_vexpf(_mm512_sub_ps(_mm512_loadu_ps(x + i),
+ _mm512_set1_ps(max)));
if (y) _mm512_storeu_ps(y + i, val);
sum += (ggml_float)_mm512_reduce_add_ps(val);
}
#elif defined(__AVX2__) && defined(__FMA__)
for (; i + 7 < n; i += 8) {
- __m256 val = ggml_v_expf(_mm256_sub_ps(_mm256_loadu_ps(x + i),
- _mm256_set1_ps(max)));
+ __m256 val = ggml_vexpf(_mm256_sub_ps(_mm256_loadu_ps(x + i),
+ _mm256_set1_ps(max)));
if (y) _mm256_storeu_ps(y + i, val);
__m128 val2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),
_mm256_castps256_ps128(val));
@@ -1927,8 +1953,8 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float
}
#elif defined(__SSE2__)
for (; i + 3 < n; i += 4) {
- __m128 val = ggml_v_expf(_mm_sub_ps(_mm_loadu_ps(x + i),
- _mm_set1_ps(max)));
+ __m128 val = ggml_vexpf(_mm_sub_ps(_mm_loadu_ps(x + i),
+ _mm_set1_ps(max)));
if (y) _mm_storeu_ps(y + i, val);
#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
val = _mm_add_ps(val, _mm_movehl_ps(val, val));
@@ -1943,8 +1969,8 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float
}
#elif defined(__ARM_NEON) && defined(__aarch64__)
for (; i + 3 < n; i += 4) {
- float32x4_t val = ggml_v_expf(vsubq_f32(vld1q_f32(x + i),
- vdupq_n_f32(max)));
+ float32x4_t val = ggml_vexpf(vsubq_f32(vld1q_f32(x + i),
+ vdupq_n_f32(max)));
if (y) vst1q_f32(y + i, val);
sum += (ggml_float)vaddvq_f32(val);
}
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index c972d44c..c8e6a4fa 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -56,6 +56,7 @@ o/$(MODE)/llamafile: \
o/$(MODE)/llamafile/pool_cancel_test.runs \
o/$(MODE)/llamafile/parse_cidr_test.runs \
o/$(MODE)/llamafile/thread_test.runs \
+ o/$(MODE)/llamafile/vmathf_test.runs \
################################################################################
# microarchitectures
@@ -147,6 +148,10 @@ o/$(MODE)/llamafile/tinyblas_cpu_sgemm_arm82.o: \
################################################################################
# testing
+o/$(MODE)/llamafile/vmathf_test: \
+ o/$(MODE)/llamafile/vmathf_test.o \
+ o/$(MODE)/llama.cpp/llama.cpp.a \
+
o/$(MODE)/llamafile/parse_cidr_test: \
o/$(MODE)/llamafile/parse_cidr_test.o \
o/$(MODE)/llamafile/parse_cidr.o \
diff --git a/llamafile/numba.h b/llamafile/numba.h
index 576f9f20..5d75f991 100644
--- a/llamafile/numba.h
+++ b/llamafile/numba.h
@@ -23,7 +23,7 @@ inline float float01(unsigned x) { // (0,1)
return 1.f / 8388608 * ((x >> 9) + .5f);
}
-inline float numba(void) { // (-10,10)
+inline float numba(void) { // (-1,1)
return float01(rand32()) * 2.f - 1.f;
}
diff --git a/llamafile/vmathf_test.cpp b/llamafile/vmathf_test.cpp
new file mode 100644
index 00000000..9c5eeb42
--- /dev/null
+++ b/llamafile/vmathf_test.cpp
@@ -0,0 +1,177 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "float.h"
+#include "numba.h"
+
+#include <assert.h>
+#include <cosmo.h>
+#include <math.h>
+#include <sys/auxv.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "llama.cpp/ggml-vector.h"
+
+extern "C" void ggml_vec_gelu_f32_amd_avx512bf16(const int n, float *y, const float *x);
+extern "C" void ggml_vec_gelu_f32_amd_avx512vl(const int n, float *y, const float *x);
+extern "C" void ggml_vec_gelu_f32_amd_avx512(const int n, float *y, const float *x);
+extern "C" void ggml_vec_gelu_f32_amd_avx2(const int n, float *y, const float *x);
+extern "C" void ggml_vec_gelu_f32_amd_f16c(const int n, float *y, const float *x);
+extern "C" void ggml_vec_gelu_f32_amd_fma(const int n, float *y, const float *x);
+extern "C" void ggml_vec_gelu_f32_amd_avx(const int n, float *y, const float *x);
+extern "C" void ggml_vec_gelu_f32_amd_ssse3(const int n, float *y, const float *x);
+extern "C" void ggml_vec_gelu_f32_amd_k8(const int n, float *y, const float *x);
+extern "C" void ggml_vec_gelu_f32_arm82(const int n, float *y, const float *x);
+extern "C" void ggml_vec_gelu_f32_arm80(const int n, float *y, const float *x);
+
+extern "C" void ggml_vec_silu_f32_amd_avx512bf16(const int n, float *y, const float *x);
+extern "C" void ggml_vec_silu_f32_amd_avx512vl(const int n, float *y, const float *x);
+extern "C" void ggml_vec_silu_f32_amd_avx512(const int n, float *y, const float *x);
+extern "C" void ggml_vec_silu_f32_amd_avx2(const int n, float *y, const float *x);
+extern "C" void ggml_vec_silu_f32_amd_f16c(const int n, float *y, const float *x);
+extern "C" void ggml_vec_silu_f32_amd_fma(const int n, float *y, const float *x);
+extern "C" void ggml_vec_silu_f32_amd_avx(const int n, float *y, const float *x);
+extern "C" void ggml_vec_silu_f32_amd_ssse3(const int n, float *y, const float *x);
+extern "C" void ggml_vec_silu_f32_amd_k8(const int n, float *y, const float *x);
+extern "C" void ggml_vec_silu_f32_arm82(const int n, float *y, const float *x);
+extern "C" void ggml_vec_silu_f32_arm80(const int n, float *y, const float *x);
+
+#define N 256
+
+float mathf(void vmathf(int, float *, const float *), float x) {
+ float A[1] = {x};
+ float B[1];
+ vmathf(1, B, A);
+ return B[0];
+}
+
+void test_vmathf(void vmathf(int, float *, const float *)) {
+
+ // create page protected mappings that let us verify no memory
+ // access overlaps the end of an array.
+
+ int pagesz = sysconf(_SC_PAGESIZE);
+ int need = N + sizeof(float);
+ int greed = (need + pagesz - 1) & -pagesz;
+
+ char *map1 =
+ (char *)mmap(0, greed + pagesz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+ npassert(map1 != MAP_FAILED);
+ npassert(!mprotect(map1 + greed, pagesz, PROT_NONE));
+
+ char *map2 =
+ (char *)mmap(0, greed + pagesz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+ npassert(map2 != MAP_FAILED);
+ npassert(!mprotect(map2 + greed, pagesz, PROT_NONE));
+
+ // verify the vectorized operation is internally consistent, i.e.
+ // checking the single-element version of itself yields identical
+ // values as the N-element version of itself.
+
+ float *AE = (float *)(map1 + greed);
+ float *BE = (float *)(map2 + greed);
+ for (int n = 0; n < N; ++n) {
+ float *A = AE - n;
+ float *B = BE - n;
+ for (int i = 0; i < n; ++i) {
+ if (rand32() % 2 == 0) {
+ A[i] = numba();
+ } else {
+ A[i] = flt::tofloat(rand32());
+ }
+ }
+ vmathf(n, B, A);
+ for (int i = 0; i < n; ++i) {
+ float x = B[i];
+ float y = mathf(vmathf, A[i]);
+ npassert(flt::toint(x) == flt::toint(y));
+ }
+ }
+
+ npassert(!munmap(map2, greed + pagesz));
+ npassert(!munmap(map1, greed + pagesz));
+}
+
+int main(int argc, char *argv[]) {
+ ShowCrashReports();
+
+ test_vmathf(ggml_vec_gelu_f32);
+ test_vmathf(ggml_vec_silu_f32);
+
+#ifdef __x86_64__
+
+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2) && X86_HAVE(AVX512F) &&
+ X86_HAVE(AVX512BW) && X86_HAVE(AVX512DQ) && X86_HAVE(AVX512VL) && X86_HAVE(AVX512_BF16)) {
+ test_vmathf(ggml_vec_gelu_f32_amd_avx512bf16);
+ test_vmathf(ggml_vec_silu_f32_amd_avx512bf16);
+ }
+
+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2) && X86_HAVE(AVX512F) &&
+ X86_HAVE(AVX512BW) && X86_HAVE(AVX512DQ) && X86_HAVE(AVX512VL)) {
+ test_vmathf(ggml_vec_gelu_f32_amd_avx512vl);
+ test_vmathf(ggml_vec_silu_f32_amd_avx512vl);
+ }
+
+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2) && X86_HAVE(AVX512F)) {
+ test_vmathf(ggml_vec_gelu_f32_amd_avx512);
+ test_vmathf(ggml_vec_silu_f32_amd_avx512);
+ }
+
+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2)) {
+ test_vmathf(ggml_vec_gelu_f32_amd_avx2);
+ test_vmathf(ggml_vec_silu_f32_amd_avx2);
+ }
+
+ if (X86_HAVE(AVX) && X86_HAVE(F16C)) {
+ test_vmathf(ggml_vec_gelu_f32_amd_f16c);
+ test_vmathf(ggml_vec_silu_f32_amd_f16c);
+ }
+
+ if (X86_HAVE(AVX) && X86_HAVE(FMA)) {
+ test_vmathf(ggml_vec_gelu_f32_amd_fma);
+ test_vmathf(ggml_vec_silu_f32_amd_fma);
+ }
+
+ if (X86_HAVE(AVX)) {
+ test_vmathf(ggml_vec_gelu_f32_amd_avx);
+ test_vmathf(ggml_vec_silu_f32_amd_avx);
+ }
+
+ if (X86_HAVE(SSSE3)) {
+ test_vmathf(ggml_vec_gelu_f32_amd_ssse3);
+ test_vmathf(ggml_vec_silu_f32_amd_ssse3);
+ }
+
+ test_vmathf(ggml_vec_gelu_f32_amd_k8);
+ test_vmathf(ggml_vec_silu_f32_amd_k8);
+
+#elif defined(__aarch64__)
+
+ if ((getauxval(AT_HWCAP) & HWCAP_FPHP) && (getauxval(AT_HWCAP) & HWCAP_ASIMDHP)) {
+ test_vmathf(ggml_vec_gelu_f32_arm82);
+ test_vmathf(ggml_vec_silu_f32_arm82);
+ }
+
+ test_vmathf(ggml_vec_gelu_f32_arm80);
+ test_vmathf(ggml_vec_silu_f32_arm80);
+
+#endif
+
+ CheckForMemoryLeaks();
+ return 0;
+}