Skip to content

Distribute and run LLMs with a single file.

Make whisperfile 12% faster in GPU mode

Justine Tunneyb3bdc62cccbd
diff --git a/llama.cpp/ggml-vector.cpp b/llama.cpp/ggml-vector.cpp
index 60d9d034..03145b0f 100644
--- a/llama.cpp/ggml-vector.cpp
+++ b/llama.cpp/ggml-vector.cpp
@@ -1,7 +1,6 @@
#include <cosmo.h>
#include <sys/auxv.h>
#include <libc/sysv/consts/hwcap.h>
-#define restrict
#include "ggml-vector.h"
extern "C" void ggml_fp16_to_fp32_row_amd_avx512bf16(const ggml_fp16_t * x, float * y, int64_t n);
@@ -48,60 +47,60 @@ extern "C" void ggml_fp32_to_bf16_row_amd_k8(const float * x, ggml_bf16_t * y, i
extern "C" void ggml_fp32_to_bf16_row_arm82(const float * x, ggml_bf16_t * y, int64_t n);
extern "C" void ggml_fp32_to_bf16_row_arm80(const float * x, ggml_bf16_t * y, int64_t n);
-extern "C" void ggml_vec_dot_f32_amd_avx512bf16(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f32_amd_avx512(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f32_amd_avx2(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f32_amd_f16c(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f32_amd_fma(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f32_amd_avx(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f32_amd_ssse3(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f32_amd_k8(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f32_arm82(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f32_arm80(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
-
-extern "C" void ggml_vec_dot_f16_amd_avx512bf16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f16_amd_avx512(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f16_amd_avx2(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f16_amd_f16c(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f16_amd_fma(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f16_amd_avx(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f16_amd_ssse3(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f16_amd_k8(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f16_arm82(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_f16_arm80(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
-
-extern "C" void ggml_vec_dot_bf16_amd_avx512bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_bf16_amd_avx512(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_bf16_amd_avx2(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_bf16_amd_f16c(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_bf16_amd_fma(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_bf16_amd_avx(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_bf16_amd_ssse3(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_bf16_amd_k8(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_bf16_arm82(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
-extern "C" void ggml_vec_dot_bf16_arm80(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
-
-extern "C" void ggml_vec_dot_f16_unroll_amd_avx512bf16(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
-extern "C" void ggml_vec_dot_f16_unroll_amd_avx512(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
-extern "C" void ggml_vec_dot_f16_unroll_amd_avx2(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
-extern "C" void ggml_vec_dot_f16_unroll_amd_f16c(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
-extern "C" void ggml_vec_dot_f16_unroll_amd_fma(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
-extern "C" void ggml_vec_dot_f16_unroll_amd_avx(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
-extern "C" void ggml_vec_dot_f16_unroll_amd_ssse3(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
-extern "C" void ggml_vec_dot_f16_unroll_amd_k8(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
-extern "C" void ggml_vec_dot_f16_unroll_arm82(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
-extern "C" void ggml_vec_dot_f16_unroll_arm80(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
-
-extern "C" void ggml_vec_mad_f32_unroll_amd_avx512bf16(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
-extern "C" void ggml_vec_mad_f32_unroll_amd_avx512(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
-extern "C" void ggml_vec_mad_f32_unroll_amd_avx2(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
-extern "C" void ggml_vec_mad_f32_unroll_amd_f16c(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
-extern "C" void ggml_vec_mad_f32_unroll_amd_fma(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
-extern "C" void ggml_vec_mad_f32_unroll_amd_avx(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
-extern "C" void ggml_vec_mad_f32_unroll_amd_ssse3(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
-extern "C" void ggml_vec_mad_f32_unroll_amd_k8(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
-extern "C" void ggml_vec_mad_f32_unroll_arm82(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
-extern "C" void ggml_vec_mad_f32_unroll_arm80(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
+extern "C" void ggml_vec_dot_f32_amd_avx512bf16(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_amd_avx512(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_amd_avx2(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_amd_f16c(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_amd_fma(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_amd_avx(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_amd_ssse3(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_amd_k8(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_arm82(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_arm80(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_f16_amd_avx512bf16(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_amd_avx512(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_amd_avx2(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_amd_f16c(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_amd_fma(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_amd_avx(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_amd_ssse3(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_amd_k8(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_arm82(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_arm80(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_bf16_amd_avx512bf16(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_amd_avx512(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_amd_avx2(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_amd_f16c(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_amd_fma(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_amd_avx(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_amd_ssse3(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_amd_k8(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_arm82(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_arm80(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_f16_unroll_amd_avx512bf16(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
+extern "C" void ggml_vec_dot_f16_unroll_amd_avx512(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
+extern "C" void ggml_vec_dot_f16_unroll_amd_avx2(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
+extern "C" void ggml_vec_dot_f16_unroll_amd_f16c(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
+extern "C" void ggml_vec_dot_f16_unroll_amd_fma(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
+extern "C" void ggml_vec_dot_f16_unroll_amd_avx(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
+extern "C" void ggml_vec_dot_f16_unroll_amd_ssse3(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
+extern "C" void ggml_vec_dot_f16_unroll_amd_k8(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
+extern "C" void ggml_vec_dot_f16_unroll_arm82(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
+extern "C" void ggml_vec_dot_f16_unroll_arm80(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
+
+extern "C" void ggml_vec_mad_f32_unroll_amd_avx512bf16(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
+extern "C" void ggml_vec_mad_f32_unroll_amd_avx512(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
+extern "C" void ggml_vec_mad_f32_unroll_amd_avx2(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
+extern "C" void ggml_vec_mad_f32_unroll_amd_f16c(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
+extern "C" void ggml_vec_mad_f32_unroll_amd_fma(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
+extern "C" void ggml_vec_mad_f32_unroll_amd_avx(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
+extern "C" void ggml_vec_mad_f32_unroll_amd_ssse3(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
+extern "C" void ggml_vec_mad_f32_unroll_amd_k8(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
+extern "C" void ggml_vec_mad_f32_unroll_arm82(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
+extern "C" void ggml_vec_mad_f32_unroll_arm80(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
extern "C" void ggml_vec_set_i8_amd_avx512bf16(const int n, int8_t * x, const int8_t v);
extern "C" void ggml_vec_set_i8_amd_avx512(const int n, int8_t * x, const int8_t v);
@@ -290,27 +289,27 @@ extern "C" void ggml_vec_scale_f16_amd_k8(const int n, ggml_fp16_t * y, const fl
extern "C" void ggml_vec_scale_f16_arm82(const int n, ggml_fp16_t * y, const float v);
extern "C" void ggml_vec_scale_f16_arm80(const int n, ggml_fp16_t * y, const float v);
-extern "C" void ggml_vec_mad_f32_amd_avx512bf16(const int n, float * restrict y, const float * restrict x, const float v);
-extern "C" void ggml_vec_mad_f32_amd_avx512(const int n, float * restrict y, const float * restrict x, const float v);
-extern "C" void ggml_vec_mad_f32_amd_avx2(const int n, float * restrict y, const float * restrict x, const float v);
-extern "C" void ggml_vec_mad_f32_amd_f16c(const int n, float * restrict y, const float * restrict x, const float v);
-extern "C" void ggml_vec_mad_f32_amd_fma(const int n, float * restrict y, const float * restrict x, const float v);
-extern "C" void ggml_vec_mad_f32_amd_avx(const int n, float * restrict y, const float * restrict x, const float v);
-extern "C" void ggml_vec_mad_f32_amd_ssse3(const int n, float * restrict y, const float * restrict x, const float v);
-extern "C" void ggml_vec_mad_f32_amd_k8(const int n, float * restrict y, const float * restrict x, const float v);
-extern "C" void ggml_vec_mad_f32_arm82(const int n, float * restrict y, const float * restrict x, const float v);
-extern "C" void ggml_vec_mad_f32_arm80(const int n, float * restrict y, const float * restrict x, const float v);
-
-extern "C" void ggml_vec_mad_f16_amd_avx512bf16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
-extern "C" void ggml_vec_mad_f16_amd_avx512(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
-extern "C" void ggml_vec_mad_f16_amd_avx2(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
-extern "C" void ggml_vec_mad_f16_amd_f16c(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
-extern "C" void ggml_vec_mad_f16_amd_fma(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
-extern "C" void ggml_vec_mad_f16_amd_avx(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
-extern "C" void ggml_vec_mad_f16_amd_ssse3(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
-extern "C" void ggml_vec_mad_f16_amd_k8(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
-extern "C" void ggml_vec_mad_f16_arm82(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
-extern "C" void ggml_vec_mad_f16_arm80(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
+extern "C" void ggml_vec_mad_f32_amd_avx512bf16(const int n, float * y, const float * x, const float v);
+extern "C" void ggml_vec_mad_f32_amd_avx512(const int n, float * y, const float * x, const float v);
+extern "C" void ggml_vec_mad_f32_amd_avx2(const int n, float * y, const float * x, const float v);
+extern "C" void ggml_vec_mad_f32_amd_f16c(const int n, float * y, const float * x, const float v);
+extern "C" void ggml_vec_mad_f32_amd_fma(const int n, float * y, const float * x, const float v);
+extern "C" void ggml_vec_mad_f32_amd_avx(const int n, float * y, const float * x, const float v);
+extern "C" void ggml_vec_mad_f32_amd_ssse3(const int n, float * y, const float * x, const float v);
+extern "C" void ggml_vec_mad_f32_amd_k8(const int n, float * y, const float * x, const float v);
+extern "C" void ggml_vec_mad_f32_arm82(const int n, float * y, const float * x, const float v);
+extern "C" void ggml_vec_mad_f32_arm80(const int n, float * y, const float * x, const float v);
+
+extern "C" void ggml_vec_mad_f16_amd_avx512bf16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
+extern "C" void ggml_vec_mad_f16_amd_avx512(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
+extern "C" void ggml_vec_mad_f16_amd_avx2(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
+extern "C" void ggml_vec_mad_f16_amd_f16c(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
+extern "C" void ggml_vec_mad_f16_amd_fma(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
+extern "C" void ggml_vec_mad_f16_amd_avx(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
+extern "C" void ggml_vec_mad_f16_amd_ssse3(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
+extern "C" void ggml_vec_mad_f16_amd_k8(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
+extern "C" void ggml_vec_mad_f16_arm82(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
+extern "C" void ggml_vec_mad_f16_arm80(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
extern "C" void ggml_vec_norm_f32_amd_avx512bf16 (const int n, float * s, const float * x);
extern "C" void ggml_vec_norm_f32_amd_avx512 (const int n, float * s, const float * x);
@@ -1286,23 +1285,23 @@ void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {
return funcs.ptr_ggml_fp32_to_bf16_row(x, y, n);
}
-void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc) {
+void ggml_vec_dot_f32(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc) {
return funcs.ptr_ggml_vec_dot_f32(n, s, bs, x, bx, y, by, nrc);
}
-void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc) {
+void ggml_vec_dot_f16(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc) {
return funcs.ptr_ggml_vec_dot_f16(n, s, bs, x, bx, y, by, nrc);
}
-void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc) {
+void ggml_vec_dot_bf16(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc) {
return funcs.ptr_ggml_vec_dot_bf16(n, s, bs, x, bx, y, by, nrc);
}
-void ggml_vec_dot_f16_unroll(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y) {
+void ggml_vec_dot_f16_unroll(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y) {
return funcs.ptr_ggml_vec_dot_f16_unroll(n, xs, s, xv, y);
}
-void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv) {
+void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv) {
return funcs.ptr_ggml_vec_mad_f32_unroll(n, xs, vs, y, xv, vv);
}
@@ -1374,11 +1373,11 @@ void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {
return funcs.ptr_ggml_vec_scale_f16(n, y, v);
}
-void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v) {
+void ggml_vec_mad_f32(const int n, float * y, const float * x, const float v) {
return funcs.ptr_ggml_vec_mad_f32(n, y, x, v);
}
-void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v) {
+void ggml_vec_mad_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v) {
return funcs.ptr_ggml_vec_mad_f16(n, y, x, v);
}
diff --git a/llama.cpp/ggml-vector.h b/llama.cpp/ggml-vector.h
index 11215976..938d5c37 100644
--- a/llama.cpp/ggml-vector.h
+++ b/llama.cpp/ggml-vector.h
@@ -10,11 +10,11 @@ void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n);
void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n);
void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n);
void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n);
-void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
-void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
-void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
-void ggml_vec_dot_f16_unroll(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
-void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
+void ggml_vec_dot_f32(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
+void ggml_vec_dot_f16(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
+void ggml_vec_dot_bf16(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
+void ggml_vec_dot_f16_unroll(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
+void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v);
void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v);
void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v);
@@ -32,8 +32,8 @@ void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y)
void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y);
void ggml_vec_scale_f32(const int n, float * y, const float v);
void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v);
-void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v);
-void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
+void ggml_vec_mad_f32(const int n, float * y, const float * x, const float v);
+void ggml_vec_mad_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
void ggml_vec_norm_f32 (const int n, float * s, const float * x);
void ggml_vec_sqr_f32 (const int n, float * y, const float * x);
void ggml_vec_sqrt_f32 (const int n, float * y, const float * x);
diff --git a/llama.cpp/ggml-vector.inc b/llama.cpp/ggml-vector.inc
index ef87bd4e..74c64861 100644
--- a/llama.cpp/ggml-vector.inc
+++ b/llama.cpp/ggml-vector.inc
@@ -1582,14 +1582,14 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float
for (; i + 15 < n; i += 16) {
__m512 val = ggml_v_expf(_mm512_sub_ps(_mm512_loadu_ps(x + i),
_mm512_set1_ps(max)));
- _mm512_storeu_ps(y + i, val);
+ if (y) _mm512_storeu_ps(y + i, val);
sum += (ggml_float)_mm512_reduce_add_ps(val);
}
#elif defined(__AVX2__) && defined(__FMA__)
for (; i + 7 < n; i += 8) {
__m256 val = ggml_v_expf(_mm256_sub_ps(_mm256_loadu_ps(x + i),
_mm256_set1_ps(max)));
- _mm256_storeu_ps(y + i, val);
+ if (y) _mm256_storeu_ps(y + i, val);
__m128 val2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),
_mm256_castps256_ps128(val));
val2 = _mm_add_ps(val2, _mm_movehl_ps(val2, val2));
@@ -1600,7 +1600,7 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float
for (; i + 3 < n; i += 4) {
__m128 val = ggml_v_expf(_mm_sub_ps(_mm_loadu_ps(x + i),
_mm_set1_ps(max)));
- _mm_storeu_ps(y + i, val);
+ if (y) _mm_storeu_ps(y + i, val);
#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
val = _mm_add_ps(val, _mm_movehl_ps(val, val));
val = _mm_add_ss(val, _mm_movehdup_ps(val));
@@ -1616,7 +1616,7 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float
for (; i + 3 < n; i += 4) {
float32x4_t val = ggml_v_expf(vsubq_f32(vld1q_f32(x + i),
vdupq_n_f32(max)));
- vst1q_f32(y + i, val);
+ if (y) vst1q_f32(y + i, val);
sum += (ggml_float)vaddvq_f32(val);
}
#endif
@@ -1624,7 +1624,7 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float
for (; i < n; ++i) {
float val = expf(x[i] - max);
sum += (ggml_float)val;
- y[i] = val;
+ if (y) y[i] = val;
}
return sum;
}
diff --git a/whisper.cpp/whisper.cpp b/whisper.cpp/whisper.cpp
index f1fb860a..bf749a0c 100644
--- a/whisper.cpp/whisper.cpp
+++ b/whisper.cpp/whisper.cpp
@@ -1,5 +1,6 @@
// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-
// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#include "llama.cpp/ggml-vector.h"
#include "whisper.h"
#define GGML_USE_CUDA
@@ -5119,12 +5120,7 @@ static void whisper_process_logits(
// populate the logprobs array (log_softmax)
{
const float logit_max = *std::max_element(logits.begin(), logits.end());
- float logsumexp = 0.0f;
- for (int i = 0; i < n_logits; ++i) {
- if (logits[i] > -INFINITY) {
- logsumexp += expf(logits[i] - logit_max);
- }
- }
+ float logsumexp = ggml_vec_soft_max_f32(n_logits, 0, logits.data(), logit_max); // [jart]
logsumexp = logf(logsumexp) + logit_max;
for (int i = 0; i < n_logits; ++i) {
@@ -5142,13 +5138,10 @@ static void whisper_process_logits(
// logsumexp over timestamps
float timestamp_logprob = -INFINITY;
{
- float logsumexp = 0.0f;
const float logprob_max = *std::max_element(logprobs.begin() + vocab.token_beg, logprobs.end());
- for (int i = vocab.token_beg; i < n_logits; ++i) {
- if (logprobs[i] > -INFINITY) {
- logsumexp += expf(logprobs[i] - logprob_max);
- }
- }
+ float logsumexp = ggml_vec_soft_max_f32(n_logits - vocab.token_beg, 0,
+ &logprobs[vocab.token_beg],
+ logprob_max); // [jart]
if (logsumexp > 0.0f) {
timestamp_logprob = logf(logsumexp) + logprob_max;
}
@@ -5170,12 +5163,7 @@ static void whisper_process_logits(
// populate the logprobs array (log_softmax)
{
const float logit_max = *std::max_element(logits.begin(), logits.end());
- float logsumexp = 0.0f;
- for (int i = 0; i < n_logits; ++i) {
- if (logits[i] > -INFINITY) {
- logsumexp += expf(logits[i] - logit_max);
- }
- }
+ float logsumexp = ggml_vec_soft_max_f32(n_logits, 0, logits.data(), logit_max); // [jart]
logsumexp = logf(logsumexp) + logit_max;
for (int i = 0; i < n_logits; ++i) {
@@ -5192,15 +5180,7 @@ static void whisper_process_logits(
}
// compute probs
- {
- for (int i = 0; i < n_logits; ++i) {
- if (logits[i] == -INFINITY) {
- probs[i] = 0.0f;
- } else {
- probs[i] = expf(logprobs[i]);
- }
- }
- }
+ ggml_vec_soft_max_f32(n_logits, &probs[0], logprobs.data(), 0); // [jart]
#if 0
// print first 100 logits - token string : logit