Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Make whisperfile 12% faster in GPU mode
Justine Tunneyb3bdc62cccbd
diff --git a/llama.cpp/ggml-vector.cpp b/llama.cpp/ggml-vector.cppindex 60d9d034..03145b0f 100644--- a/llama.cpp/ggml-vector.cpp+++ b/llama.cpp/ggml-vector.cpp@@ -1,7 +1,6 @@#include <cosmo.h>#include <sys/auxv.h>#include <libc/sysv/consts/hwcap.h>-#define restrict#include "ggml-vector.h"extern "C" void ggml_fp16_to_fp32_row_amd_avx512bf16(const ggml_fp16_t * x, float * y, int64_t n);@@ -48,60 +47,60 @@ extern "C" void ggml_fp32_to_bf16_row_amd_k8(const float * x, ggml_bf16_t * y, iextern "C" void ggml_fp32_to_bf16_row_arm82(const float * x, ggml_bf16_t * y, int64_t n);extern "C" void ggml_fp32_to_bf16_row_arm80(const float * x, ggml_bf16_t * y, int64_t n);-extern "C" void ggml_vec_dot_f32_amd_avx512bf16(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f32_amd_avx512(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f32_amd_avx2(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f32_amd_f16c(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f32_amd_fma(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f32_amd_avx(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f32_amd_ssse3(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f32_amd_k8(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f32_arm82(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f32_arm80(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);--extern "C" void ggml_vec_dot_f16_amd_avx512bf16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f16_amd_avx512(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f16_amd_avx2(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f16_amd_f16c(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f16_amd_fma(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f16_amd_avx(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f16_amd_ssse3(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f16_amd_k8(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f16_arm82(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_f16_arm80(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);--extern "C" void ggml_vec_dot_bf16_amd_avx512bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_bf16_amd_avx512(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_bf16_amd_avx2(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_bf16_amd_f16c(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_bf16_amd_fma(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_bf16_amd_avx(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_bf16_amd_ssse3(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_bf16_amd_k8(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_bf16_arm82(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);-extern "C" void ggml_vec_dot_bf16_arm80(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);--extern "C" void ggml_vec_dot_f16_unroll_amd_avx512bf16(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);-extern "C" void ggml_vec_dot_f16_unroll_amd_avx512(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);-extern "C" void ggml_vec_dot_f16_unroll_amd_avx2(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);-extern "C" void ggml_vec_dot_f16_unroll_amd_f16c(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);-extern "C" void ggml_vec_dot_f16_unroll_amd_fma(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);-extern "C" void ggml_vec_dot_f16_unroll_amd_avx(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);-extern "C" void ggml_vec_dot_f16_unroll_amd_ssse3(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);-extern "C" void ggml_vec_dot_f16_unroll_amd_k8(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);-extern "C" void ggml_vec_dot_f16_unroll_arm82(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);-extern "C" void ggml_vec_dot_f16_unroll_arm80(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);--extern "C" void ggml_vec_mad_f32_unroll_amd_avx512bf16(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);-extern "C" void ggml_vec_mad_f32_unroll_amd_avx512(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);-extern "C" void ggml_vec_mad_f32_unroll_amd_avx2(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);-extern "C" void ggml_vec_mad_f32_unroll_amd_f16c(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);-extern "C" void ggml_vec_mad_f32_unroll_amd_fma(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);-extern "C" void ggml_vec_mad_f32_unroll_amd_avx(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);-extern "C" void ggml_vec_mad_f32_unroll_amd_ssse3(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);-extern "C" void ggml_vec_mad_f32_unroll_amd_k8(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);-extern "C" void ggml_vec_mad_f32_unroll_arm82(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);-extern "C" void ggml_vec_mad_f32_unroll_arm80(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);+extern "C" void ggml_vec_dot_f32_amd_avx512bf16(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_amd_avx512(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_amd_avx2(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_amd_f16c(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_amd_fma(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_amd_avx(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_amd_ssse3(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_amd_k8(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_arm82(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_arm80(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);++extern "C" void ggml_vec_dot_f16_amd_avx512bf16(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_amd_avx512(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_amd_avx2(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_amd_f16c(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_amd_fma(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_amd_avx(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_amd_ssse3(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_amd_k8(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_arm82(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_arm80(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);++extern "C" void ggml_vec_dot_bf16_amd_avx512bf16(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_amd_avx512(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_amd_avx2(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_amd_f16c(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_amd_fma(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_amd_avx(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_amd_ssse3(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_amd_k8(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_arm82(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_arm80(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);++extern "C" void ggml_vec_dot_f16_unroll_amd_avx512bf16(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);+extern "C" void ggml_vec_dot_f16_unroll_amd_avx512(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);+extern "C" void ggml_vec_dot_f16_unroll_amd_avx2(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);+extern "C" void ggml_vec_dot_f16_unroll_amd_f16c(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);+extern "C" void ggml_vec_dot_f16_unroll_amd_fma(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);+extern "C" void ggml_vec_dot_f16_unroll_amd_avx(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);+extern "C" void ggml_vec_dot_f16_unroll_amd_ssse3(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);+extern "C" void ggml_vec_dot_f16_unroll_amd_k8(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);+extern "C" void ggml_vec_dot_f16_unroll_arm82(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);+extern "C" void ggml_vec_dot_f16_unroll_arm80(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);++extern "C" void ggml_vec_mad_f32_unroll_amd_avx512bf16(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);+extern "C" void ggml_vec_mad_f32_unroll_amd_avx512(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);+extern "C" void ggml_vec_mad_f32_unroll_amd_avx2(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);+extern "C" void ggml_vec_mad_f32_unroll_amd_f16c(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);+extern "C" void ggml_vec_mad_f32_unroll_amd_fma(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);+extern "C" void ggml_vec_mad_f32_unroll_amd_avx(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);+extern "C" void ggml_vec_mad_f32_unroll_amd_ssse3(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);+extern "C" void ggml_vec_mad_f32_unroll_amd_k8(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);+extern "C" void ggml_vec_mad_f32_unroll_arm82(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);+extern "C" void ggml_vec_mad_f32_unroll_arm80(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);extern "C" void ggml_vec_set_i8_amd_avx512bf16(const int n, int8_t * x, const int8_t v);extern "C" void ggml_vec_set_i8_amd_avx512(const int n, int8_t * x, const int8_t v);@@ -290,27 +289,27 @@ extern "C" void ggml_vec_scale_f16_amd_k8(const int n, ggml_fp16_t * y, const flextern "C" void ggml_vec_scale_f16_arm82(const int n, ggml_fp16_t * y, const float v);extern "C" void ggml_vec_scale_f16_arm80(const int n, ggml_fp16_t * y, const float v);-extern "C" void ggml_vec_mad_f32_amd_avx512bf16(const int n, float * restrict y, const float * restrict x, const float v);-extern "C" void ggml_vec_mad_f32_amd_avx512(const int n, float * restrict y, const float * restrict x, const float v);-extern "C" void ggml_vec_mad_f32_amd_avx2(const int n, float * restrict y, const float * restrict x, const float v);-extern "C" void ggml_vec_mad_f32_amd_f16c(const int n, float * restrict y, const float * restrict x, const float v);-extern "C" void ggml_vec_mad_f32_amd_fma(const int n, float * restrict y, const float * restrict x, const float v);-extern "C" void ggml_vec_mad_f32_amd_avx(const int n, float * restrict y, const float * restrict x, const float v);-extern "C" void ggml_vec_mad_f32_amd_ssse3(const int n, float * restrict y, const float * restrict x, const float v);-extern "C" void ggml_vec_mad_f32_amd_k8(const int n, float * restrict y, const float * restrict x, const float v);-extern "C" void ggml_vec_mad_f32_arm82(const int n, float * restrict y, const float * restrict x, const float v);-extern "C" void ggml_vec_mad_f32_arm80(const int n, float * restrict y, const float * restrict x, const float v);--extern "C" void ggml_vec_mad_f16_amd_avx512bf16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);-extern "C" void ggml_vec_mad_f16_amd_avx512(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);-extern "C" void ggml_vec_mad_f16_amd_avx2(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);-extern "C" void ggml_vec_mad_f16_amd_f16c(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);-extern "C" void ggml_vec_mad_f16_amd_fma(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);-extern "C" void ggml_vec_mad_f16_amd_avx(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);-extern "C" void ggml_vec_mad_f16_amd_ssse3(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);-extern "C" void ggml_vec_mad_f16_amd_k8(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);-extern "C" void ggml_vec_mad_f16_arm82(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);-extern "C" void ggml_vec_mad_f16_arm80(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);+extern "C" void ggml_vec_mad_f32_amd_avx512bf16(const int n, float * y, const float * x, const float v);+extern "C" void ggml_vec_mad_f32_amd_avx512(const int n, float * y, const float * x, const float v);+extern "C" void ggml_vec_mad_f32_amd_avx2(const int n, float * y, const float * x, const float v);+extern "C" void ggml_vec_mad_f32_amd_f16c(const int n, float * y, const float * x, const float v);+extern "C" void ggml_vec_mad_f32_amd_fma(const int n, float * y, const float * x, const float v);+extern "C" void ggml_vec_mad_f32_amd_avx(const int n, float * y, const float * x, const float v);+extern "C" void ggml_vec_mad_f32_amd_ssse3(const int n, float * y, const float * x, const float v);+extern "C" void ggml_vec_mad_f32_amd_k8(const int n, float * y, const float * x, const float v);+extern "C" void ggml_vec_mad_f32_arm82(const int n, float * y, const float * x, const float v);+extern "C" void ggml_vec_mad_f32_arm80(const int n, float * y, const float * x, const float v);++extern "C" void ggml_vec_mad_f16_amd_avx512bf16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);+extern "C" void ggml_vec_mad_f16_amd_avx512(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);+extern "C" void ggml_vec_mad_f16_amd_avx2(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);+extern "C" void ggml_vec_mad_f16_amd_f16c(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);+extern "C" void ggml_vec_mad_f16_amd_fma(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);+extern "C" void ggml_vec_mad_f16_amd_avx(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);+extern "C" void ggml_vec_mad_f16_amd_ssse3(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);+extern "C" void ggml_vec_mad_f16_amd_k8(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);+extern "C" void ggml_vec_mad_f16_arm82(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);+extern "C" void ggml_vec_mad_f16_arm80(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);extern "C" void ggml_vec_norm_f32_amd_avx512bf16 (const int n, float * s, const float * x);extern "C" void ggml_vec_norm_f32_amd_avx512 (const int n, float * s, const float * x);@@ -1286,23 +1285,23 @@ void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {return funcs.ptr_ggml_fp32_to_bf16_row(x, y, n);}-void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc) {+void ggml_vec_dot_f32(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc) {return funcs.ptr_ggml_vec_dot_f32(n, s, bs, x, bx, y, by, nrc);}-void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc) {+void ggml_vec_dot_f16(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc) {return funcs.ptr_ggml_vec_dot_f16(n, s, bs, x, bx, y, by, nrc);}-void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc) {+void ggml_vec_dot_bf16(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc) {return funcs.ptr_ggml_vec_dot_bf16(n, s, bs, x, bx, y, by, nrc);}-void ggml_vec_dot_f16_unroll(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y) {+void ggml_vec_dot_f16_unroll(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y) {return funcs.ptr_ggml_vec_dot_f16_unroll(n, xs, s, xv, y);}-void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv) {+void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv) {return funcs.ptr_ggml_vec_mad_f32_unroll(n, xs, vs, y, xv, vv);}@@ -1374,11 +1373,11 @@ void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {return funcs.ptr_ggml_vec_scale_f16(n, y, v);}-void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v) {+void ggml_vec_mad_f32(const int n, float * y, const float * x, const float v) {return funcs.ptr_ggml_vec_mad_f32(n, y, x, v);}-void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v) {+void ggml_vec_mad_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v) {return funcs.ptr_ggml_vec_mad_f16(n, y, x, v);}diff --git a/llama.cpp/ggml-vector.h b/llama.cpp/ggml-vector.hindex 11215976..938d5c37 100644--- a/llama.cpp/ggml-vector.h+++ b/llama.cpp/ggml-vector.h@@ -10,11 +10,11 @@ void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n);void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n);void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n);void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n);-void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);-void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);-void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);-void ggml_vec_dot_f16_unroll(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);-void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);+void ggml_vec_dot_f32(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);+void ggml_vec_dot_f16(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);+void ggml_vec_dot_bf16(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);+void ggml_vec_dot_f16_unroll(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);+void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v);void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v);void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v);@@ -32,8 +32,8 @@ void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y)void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y);void ggml_vec_scale_f32(const int n, float * y, const float v);void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v);-void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v);-void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);+void ggml_vec_mad_f32(const int n, float * y, const float * x, const float v);+void ggml_vec_mad_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);void ggml_vec_norm_f32 (const int n, float * s, const float * x);void ggml_vec_sqr_f32 (const int n, float * y, const float * x);void ggml_vec_sqrt_f32 (const int n, float * y, const float * x);diff --git a/llama.cpp/ggml-vector.inc b/llama.cpp/ggml-vector.incindex ef87bd4e..74c64861 100644--- a/llama.cpp/ggml-vector.inc+++ b/llama.cpp/ggml-vector.inc@@ -1582,14 +1582,14 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, floatfor (; i + 15 < n; i += 16) {__m512 val = ggml_v_expf(_mm512_sub_ps(_mm512_loadu_ps(x + i),_mm512_set1_ps(max)));- _mm512_storeu_ps(y + i, val);+ if (y) _mm512_storeu_ps(y + i, val);sum += (ggml_float)_mm512_reduce_add_ps(val);}#elif defined(__AVX2__) && defined(__FMA__)for (; i + 7 < n; i += 8) {__m256 val = ggml_v_expf(_mm256_sub_ps(_mm256_loadu_ps(x + i),_mm256_set1_ps(max)));- _mm256_storeu_ps(y + i, val);+ if (y) _mm256_storeu_ps(y + i, val);__m128 val2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),_mm256_castps256_ps128(val));val2 = _mm_add_ps(val2, _mm_movehl_ps(val2, val2));@@ -1600,7 +1600,7 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, floatfor (; i + 3 < n; i += 4) {__m128 val = ggml_v_expf(_mm_sub_ps(_mm_loadu_ps(x + i),_mm_set1_ps(max)));- _mm_storeu_ps(y + i, val);+ if (y) _mm_storeu_ps(y + i, val);#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)val = _mm_add_ps(val, _mm_movehl_ps(val, val));val = _mm_add_ss(val, _mm_movehdup_ps(val));@@ -1616,7 +1616,7 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, floatfor (; i + 3 < n; i += 4) {float32x4_t val = ggml_v_expf(vsubq_f32(vld1q_f32(x + i),vdupq_n_f32(max)));- vst1q_f32(y + i, val);+ if (y) vst1q_f32(y + i, val);sum += (ggml_float)vaddvq_f32(val);}#endif@@ -1624,7 +1624,7 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, floatfor (; i < n; ++i) {float val = expf(x[i] - max);sum += (ggml_float)val;- y[i] = val;+ if (y) y[i] = val;}return sum;}diff --git a/whisper.cpp/whisper.cpp b/whisper.cpp/whisper.cppindex f1fb860a..bf749a0c 100644--- a/whisper.cpp/whisper.cpp+++ b/whisper.cpp/whisper.cpp@@ -1,5 +1,6 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+#include "llama.cpp/ggml-vector.h"#include "whisper.h"#define GGML_USE_CUDA@@ -5119,12 +5120,7 @@ static void whisper_process_logits(// populate the logprobs array (log_softmax){const float logit_max = *std::max_element(logits.begin(), logits.end());- float logsumexp = 0.0f;- for (int i = 0; i < n_logits; ++i) {- if (logits[i] > -INFINITY) {- logsumexp += expf(logits[i] - logit_max);- }- }+ float logsumexp = ggml_vec_soft_max_f32(n_logits, 0, logits.data(), logit_max); // [jart]logsumexp = logf(logsumexp) + logit_max;for (int i = 0; i < n_logits; ++i) {@@ -5142,13 +5138,10 @@ static void whisper_process_logits(// logsumexp over timestampsfloat timestamp_logprob = -INFINITY;{- float logsumexp = 0.0f;const float logprob_max = *std::max_element(logprobs.begin() + vocab.token_beg, logprobs.end());- for (int i = vocab.token_beg; i < n_logits; ++i) {- if (logprobs[i] > -INFINITY) {- logsumexp += expf(logprobs[i] - logprob_max);- }- }+ float logsumexp = ggml_vec_soft_max_f32(n_logits - vocab.token_beg, 0,+ &logprobs[vocab.token_beg],+ logprob_max); // [jart]if (logsumexp > 0.0f) {timestamp_logprob = logf(logsumexp) + logprob_max;}@@ -5170,12 +5163,7 @@ static void whisper_process_logits(// populate the logprobs array (log_softmax){const float logit_max = *std::max_element(logits.begin(), logits.end());- float logsumexp = 0.0f;- for (int i = 0; i < n_logits; ++i) {- if (logits[i] > -INFINITY) {- logsumexp += expf(logits[i] - logit_max);- }- }+ float logsumexp = ggml_vec_soft_max_f32(n_logits, 0, logits.data(), logit_max); // [jart]logsumexp = logf(logsumexp) + logit_max;for (int i = 0; i < n_logits; ++i) {@@ -5192,15 +5180,7 @@ static void whisper_process_logits(}// compute probs- {- for (int i = 0; i < n_logits; ++i) {- if (logits[i] == -INFINITY) {- probs[i] = 0.0f;- } else {- probs[i] = expf(logprobs[i]);- }- }- }+ ggml_vec_soft_max_f32(n_logits, &probs[0], logprobs.data(), 0); // [jart]#if 0// print first 100 logits - token string : logit