Skip to content

Distribute and run LLMs with a single file.

Add better support for AVX512 CPUs

Justine Tunney45130eb7a3bf
diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk
index a6dce126..a3d7072a 100644
--- a/llama.cpp/BUILD.mk
+++ b/llama.cpp/BUILD.mk
@@ -60,6 +60,7 @@ o/$(MODE)/llama.cpp/ggml-quants-amd-ssse3.o: private TARGET_ARCH += -Xx86_64-mtu
o/$(MODE)/llama.cpp/ggml-quants-amd-avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge -Xx86_64-mavx
o/$(MODE)/llama.cpp/ggml-quants-amd-avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mavx -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2
o/$(MODE)/llama.cpp/ggml-quants-amd-avx512.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mavx -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f
+o/$(MODE)/llama.cpp/ggml-quants-amd-avx512vl.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mavx -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512bw -Xx86_64-mavx512dq -Xx86_64-mavx512vl
o/$(MODE)/llama.cpp/ggml-vector.o: private CXXFLAGS += -Os
o/$(MODE)/llama.cpp/ggml-vector-amd-k8.o: private TARGET_ARCH += -Xx86_64-mtune=k8
@@ -69,7 +70,8 @@ o/$(MODE)/llama.cpp/ggml-vector-amd-fma.o: private TARGET_ARCH += -Xx86_64-mtune
o/$(MODE)/llama.cpp/ggml-vector-amd-f16c.o: private TARGET_ARCH += -Xx86_64-mtune=ivybridge -Xx86_64-mavx -Xx86_64-mf16c
o/$(MODE)/llama.cpp/ggml-vector-amd-avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mavx -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2
o/$(MODE)/llama.cpp/ggml-vector-amd-avx512.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mavx -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f
-o/$(MODE)/llama.cpp/ggml-vector-amd-avx512bf16.o: private TARGET_ARCH += -Xx86_64-mtune=znver4 -Xx86_64-mavx -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512vl -Xx86_64-mavx512bf16
+o/$(MODE)/llama.cpp/ggml-vector-amd-avx512vl.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mavx -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512bw -Xx86_64-mavx512dq -Xx86_64-mavx512vl
+o/$(MODE)/llama.cpp/ggml-vector-amd-avx512bf16.o: private TARGET_ARCH += -Xx86_64-mtune=znver4 -Xx86_64-mavx -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512bw -Xx86_64-mavx512dq -Xx86_64-mavx512vl -Xx86_64-mavx512bf16
o/$(MODE)/llama.cpp/ggml-vector-arm82.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+fp16
$(LLAMA_CPP_OBJS): llama.cpp/BUILD.mk
diff --git a/llama.cpp/ggml-quants-amd-avx512vl.c b/llama.cpp/ggml-quants-amd-avx512vl.c
new file mode 100644
index 00000000..4c2d559b
--- /dev/null
+++ b/llama.cpp/ggml-quants-amd-avx512vl.c
@@ -0,0 +1,100 @@
+#ifdef __x86_64__
+#define quantize_row_q4_0_ref quantize_row_q4_0_ref_amd_avx512vl
+#define quantize_row_q4_1_ref quantize_row_q4_1_ref_amd_avx512vl
+#define quantize_row_q5_0_ref quantize_row_q5_0_ref_amd_avx512vl
+#define quantize_row_q5_1_ref quantize_row_q5_1_ref_amd_avx512vl
+#define quantize_row_q8_0_ref quantize_row_q8_0_ref_amd_avx512vl
+#define quantize_row_q8_1_ref quantize_row_q8_1_ref_amd_avx512vl
+#define quantize_row_q2_K_ref quantize_row_q2_K_ref_amd_avx512vl
+#define quantize_row_q3_K_ref quantize_row_q3_K_ref_amd_avx512vl
+#define quantize_row_q4_K_ref quantize_row_q4_K_ref_amd_avx512vl
+#define quantize_row_q5_K_ref quantize_row_q5_K_ref_amd_avx512vl
+#define quantize_row_q6_K_ref quantize_row_q6_K_ref_amd_avx512vl
+#define quantize_row_q8_K_ref quantize_row_q8_K_ref_amd_avx512vl
+#define quantize_row_iq3_xxs_ref quantize_row_iq3_xxs_ref_amd_avx512vl
+#define quantize_row_iq4_nl_ref quantize_row_iq4_nl_ref_amd_avx512vl
+#define quantize_row_iq4_xs_ref quantize_row_iq4_xs_ref_amd_avx512vl
+#define quantize_row_iq3_s_ref quantize_row_iq3_s_ref_amd_avx512vl
+#define quantize_row_iq2_s_ref quantize_row_iq2_s_ref_amd_avx512vl
+#define quantize_row_q4_0 quantize_row_q4_0_amd_avx512vl
+#define quantize_row_q4_1 quantize_row_q4_1_amd_avx512vl
+#define quantize_row_q5_0 quantize_row_q5_0_amd_avx512vl
+#define quantize_row_q5_1 quantize_row_q5_1_amd_avx512vl
+#define quantize_row_q8_0 quantize_row_q8_0_amd_avx512vl
+#define quantize_row_q8_1 quantize_row_q8_1_amd_avx512vl
+#define quantize_row_q2_K quantize_row_q2_K_amd_avx512vl
+#define quantize_row_q3_K quantize_row_q3_K_amd_avx512vl
+#define quantize_row_q4_K quantize_row_q4_K_amd_avx512vl
+#define quantize_row_q5_K quantize_row_q5_K_amd_avx512vl
+#define quantize_row_q6_K quantize_row_q6_K_amd_avx512vl
+#define quantize_row_q8_K quantize_row_q8_K_amd_avx512vl
+#define quantize_row_iq3_xxs quantize_row_iq3_xxs_amd_avx512vl
+#define quantize_row_iq4_nl quantize_row_iq4_nl_amd_avx512vl
+#define quantize_row_iq4_xs quantize_row_iq4_xs_amd_avx512vl
+#define quantize_row_iq3_s quantize_row_iq3_s_amd_avx512vl
+#define quantize_row_iq2_s quantize_row_iq2_s_amd_avx512vl
+#define dequantize_row_q4_0 dequantize_row_q4_0_amd_avx512vl
+#define dequantize_row_q4_1 dequantize_row_q4_1_amd_avx512vl
+#define dequantize_row_q5_0 dequantize_row_q5_0_amd_avx512vl
+#define dequantize_row_q5_1 dequantize_row_q5_1_amd_avx512vl
+#define dequantize_row_q8_0 dequantize_row_q8_0_amd_avx512vl
+#define dequantize_row_q2_K dequantize_row_q2_K_amd_avx512vl
+#define dequantize_row_q3_K dequantize_row_q3_K_amd_avx512vl
+#define dequantize_row_q4_K dequantize_row_q4_K_amd_avx512vl
+#define dequantize_row_q5_K dequantize_row_q5_K_amd_avx512vl
+#define dequantize_row_q6_K dequantize_row_q6_K_amd_avx512vl
+#define dequantize_row_q8_K dequantize_row_q8_K_amd_avx512vl
+#define dequantize_row_iq2_xxs dequantize_row_iq2_xxs_amd_avx512vl
+#define dequantize_row_iq2_xs dequantize_row_iq2_xs_amd_avx512vl
+#define dequantize_row_iq2_s dequantize_row_iq2_s_amd_avx512vl
+#define dequantize_row_iq3_xxs dequantize_row_iq3_xxs_amd_avx512vl
+#define dequantize_row_iq1_s dequantize_row_iq1_s_amd_avx512vl
+#define dequantize_row_iq1_m dequantize_row_iq1_m_amd_avx512vl
+#define dequantize_row_iq4_nl dequantize_row_iq4_nl_amd_avx512vl
+#define dequantize_row_iq4_xs dequantize_row_iq4_xs_amd_avx512vl
+#define dequantize_row_iq3_s dequantize_row_iq3_s_amd_avx512vl
+#define ggml_vec_dot_q4_0_q8_0 ggml_vec_dot_q4_0_q8_0_amd_avx512vl
+#define ggml_vec_dot_q4_1_q8_1 ggml_vec_dot_q4_1_q8_1_amd_avx512vl
+#define ggml_vec_dot_q5_0_q8_0 ggml_vec_dot_q5_0_q8_0_amd_avx512vl
+#define ggml_vec_dot_q5_1_q8_1 ggml_vec_dot_q5_1_q8_1_amd_avx512vl
+#define ggml_vec_dot_q8_0_q8_0 ggml_vec_dot_q8_0_q8_0_amd_avx512vl
+#define ggml_vec_dot_q2_K_q8_K ggml_vec_dot_q2_K_q8_K_amd_avx512vl
+#define ggml_vec_dot_q3_K_q8_K ggml_vec_dot_q3_K_q8_K_amd_avx512vl
+#define ggml_vec_dot_q4_K_q8_K ggml_vec_dot_q4_K_q8_K_amd_avx512vl
+#define ggml_vec_dot_q5_K_q8_K ggml_vec_dot_q5_K_q8_K_amd_avx512vl
+#define ggml_vec_dot_q6_K_q8_K ggml_vec_dot_q6_K_q8_K_amd_avx512vl
+#define ggml_vec_dot_iq2_xxs_q8_K ggml_vec_dot_iq2_xxs_q8_K_amd_avx512vl
+#define ggml_vec_dot_iq2_xs_q8_K ggml_vec_dot_iq2_xs_q8_K_amd_avx512vl
+#define ggml_vec_dot_iq2_s_q8_K ggml_vec_dot_iq2_s_q8_K_amd_avx512vl
+#define ggml_vec_dot_iq3_xxs_q8_K ggml_vec_dot_iq3_xxs_q8_K_amd_avx512vl
+#define ggml_vec_dot_iq1_s_q8_K ggml_vec_dot_iq1_s_q8_K_amd_avx512vl
+#define ggml_vec_dot_iq1_m_q8_K ggml_vec_dot_iq1_m_q8_K_amd_avx512vl
+#define ggml_vec_dot_iq4_nl_q8_0 ggml_vec_dot_iq4_nl_q8_0_amd_avx512vl
+#define ggml_vec_dot_iq4_xs_q8_K ggml_vec_dot_iq4_xs_q8_K_amd_avx512vl
+#define ggml_vec_dot_iq3_s_q8_K ggml_vec_dot_iq3_s_q8_K_amd_avx512vl
+#define quantize_iq2_xxs quantize_iq2_xxs_amd_avx512vl
+#define quantize_iq2_xs quantize_iq2_xs_amd_avx512vl
+#define quantize_iq2_s quantize_iq2_s_amd_avx512vl
+#define quantize_iq3_xxs quantize_iq3_xxs_amd_avx512vl
+#define quantize_iq1_s quantize_iq1_s_amd_avx512vl
+#define quantize_iq1_m quantize_iq1_m_amd_avx512vl
+#define quantize_iq4_nl quantize_iq4_nl_amd_avx512vl
+#define quantize_iq4_xs quantize_iq4_xs_amd_avx512vl
+#define quantize_iq3_s quantize_iq3_s_amd_avx512vl
+#define quantize_q2_K quantize_q2_K_amd_avx512vl
+#define quantize_q3_K quantize_q3_K_amd_avx512vl
+#define quantize_q4_K quantize_q4_K_amd_avx512vl
+#define quantize_q5_K quantize_q5_K_amd_avx512vl
+#define quantize_q6_K quantize_q6_K_amd_avx512vl
+#define quantize_q4_0 quantize_q4_0_amd_avx512vl
+#define quantize_q4_1 quantize_q4_1_amd_avx512vl
+#define quantize_q5_0 quantize_q5_0_amd_avx512vl
+#define quantize_q5_1 quantize_q5_1_amd_avx512vl
+#define quantize_q8_0 quantize_q8_0_amd_avx512vl
+#define iq2xs_init_impl iq2xs_init_impl_amd_avx512vl
+#define iq2xs_free_impl iq2xs_free_impl_amd_avx512vl
+#define iq3xs_init_impl iq3xs_init_impl_amd_avx512vl
+#define iq3xs_free_impl iq3xs_free_impl_amd_avx512vl
+#define ggml_validate_row_data ggml_validate_row_data_amd_avx512vl
+#include "ggml-quants.inc"
+#endif // __x86_64__
diff --git a/llama.cpp/ggml-quants.cpp b/llama.cpp/ggml-quants.cpp
index a06fbceb..ac7fcf48 100644
--- a/llama.cpp/ggml-quants.cpp
+++ b/llama.cpp/ggml-quants.cpp
@@ -3,6 +3,7 @@
#include <libc/sysv/consts/hwcap.h>
#include "ggml-quants.h"
+extern "C" void quantize_row_q4_0_ref_amd_avx512vl(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_0_ref_amd_avx512(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_0_ref_amd_avx2(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_0_ref_amd_avx(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
@@ -10,6 +11,7 @@ extern "C" void quantize_row_q4_0_ref_amd_ssse3(const float * GGML_RESTRICT x, b
extern "C" void quantize_row_q4_0_ref_amd_k8(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_0_ref_arm80(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_1_ref_amd_avx512vl(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_1_ref_amd_avx512(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_1_ref_amd_avx2(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_1_ref_amd_avx(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
@@ -17,6 +19,7 @@ extern "C" void quantize_row_q4_1_ref_amd_ssse3(const float * GGML_RESTRICT x, b
extern "C" void quantize_row_q4_1_ref_amd_k8(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_1_ref_arm80(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_0_ref_amd_avx512vl(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_0_ref_amd_avx512(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_0_ref_amd_avx2(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_0_ref_amd_avx(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
@@ -24,6 +27,7 @@ extern "C" void quantize_row_q5_0_ref_amd_ssse3(const float * GGML_RESTRICT x, b
extern "C" void quantize_row_q5_0_ref_amd_k8(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_0_ref_arm80(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_1_ref_amd_avx512vl(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_1_ref_amd_avx512(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_1_ref_amd_avx2(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_1_ref_amd_avx(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);
@@ -31,6 +35,7 @@ extern "C" void quantize_row_q5_1_ref_amd_ssse3(const float * GGML_RESTRICT x, b
extern "C" void quantize_row_q5_1_ref_amd_k8(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_1_ref_arm80(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_0_ref_amd_avx512vl(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_0_ref_amd_avx512(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_0_ref_amd_avx2(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_0_ref_amd_avx(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);
@@ -38,6 +43,7 @@ extern "C" void quantize_row_q8_0_ref_amd_ssse3(const float * GGML_RESTRICT x, b
extern "C" void quantize_row_q8_0_ref_amd_k8(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_0_ref_arm80(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_1_ref_amd_avx512vl(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_1_ref_amd_avx512(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_1_ref_amd_avx2(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_1_ref_amd_avx(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);
@@ -45,6 +51,7 @@ extern "C" void quantize_row_q8_1_ref_amd_ssse3(const float * GGML_RESTRICT x, b
extern "C" void quantize_row_q8_1_ref_amd_k8(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_1_ref_arm80(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q2_K_ref_amd_avx512vl(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q2_K_ref_amd_avx512(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q2_K_ref_amd_avx2(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q2_K_ref_amd_avx(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);
@@ -52,6 +59,7 @@ extern "C" void quantize_row_q2_K_ref_amd_ssse3(const float * GGML_RESTRICT x, b
extern "C" void quantize_row_q2_K_ref_amd_k8(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q2_K_ref_arm80(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q3_K_ref_amd_avx512vl(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q3_K_ref_amd_avx512(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q3_K_ref_amd_avx2(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q3_K_ref_amd_avx(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);
@@ -59,6 +67,7 @@ extern "C" void quantize_row_q3_K_ref_amd_ssse3(const float * GGML_RESTRICT x, b
extern "C" void quantize_row_q3_K_ref_amd_k8(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q3_K_ref_arm80(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_K_ref_amd_avx512vl(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_K_ref_amd_avx512(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_K_ref_amd_avx2(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_K_ref_amd_avx(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);
@@ -66,6 +75,7 @@ extern "C" void quantize_row_q4_K_ref_amd_ssse3(const float * GGML_RESTRICT x, b
extern "C" void quantize_row_q4_K_ref_amd_k8(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_K_ref_arm80(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_K_ref_amd_avx512vl(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_K_ref_amd_avx512(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_K_ref_amd_avx2(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_K_ref_amd_avx(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);
@@ -73,6 +83,7 @@ extern "C" void quantize_row_q5_K_ref_amd_ssse3(const float * GGML_RESTRICT x, b
extern "C" void quantize_row_q5_K_ref_amd_k8(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_K_ref_arm80(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q6_K_ref_amd_avx512vl(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q6_K_ref_amd_avx512(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q6_K_ref_amd_avx2(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q6_K_ref_amd_avx(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);
@@ -80,6 +91,7 @@ extern "C" void quantize_row_q6_K_ref_amd_ssse3(const float * GGML_RESTRICT x, b
extern "C" void quantize_row_q6_K_ref_amd_k8(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q6_K_ref_arm80(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_K_ref_amd_avx512vl(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_K_ref_amd_avx512(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_K_ref_amd_avx2(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_K_ref_amd_avx(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);
@@ -87,6 +99,7 @@ extern "C" void quantize_row_q8_K_ref_amd_ssse3(const float * GGML_RESTRICT x, b
extern "C" void quantize_row_q8_K_ref_amd_k8(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_K_ref_arm80(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_xxs_ref_amd_avx512vl(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_xxs_ref_amd_avx512(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_xxs_ref_amd_avx2(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_xxs_ref_amd_avx(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
@@ -94,6 +107,7 @@ extern "C" void quantize_row_iq3_xxs_ref_amd_ssse3(const float * GGML_RESTRICT x
extern "C" void quantize_row_iq3_xxs_ref_amd_k8(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_xxs_ref_arm80(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_nl_ref_amd_avx512vl (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_nl_ref_amd_avx512 (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_nl_ref_amd_avx2 (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_nl_ref_amd_avx (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);
@@ -101,6 +115,7 @@ extern "C" void quantize_row_iq4_nl_ref_amd_ssse3 (const float * GGML_RESTRICT x
extern "C" void quantize_row_iq4_nl_ref_amd_k8 (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_nl_ref_arm80 (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_xs_ref_amd_avx512vl (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_xs_ref_amd_avx512 (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_xs_ref_amd_avx2 (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_xs_ref_amd_avx (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);
@@ -108,6 +123,7 @@ extern "C" void quantize_row_iq4_xs_ref_amd_ssse3 (const float * GGML_RESTRICT x
extern "C" void quantize_row_iq4_xs_ref_amd_k8 (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_xs_ref_arm80 (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_s_ref_amd_avx512vl (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_s_ref_amd_avx512 (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_s_ref_amd_avx2 (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_s_ref_amd_avx (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);
@@ -115,6 +131,7 @@ extern "C" void quantize_row_iq3_s_ref_amd_ssse3 (const float * GGML_RESTRICT x
extern "C" void quantize_row_iq3_s_ref_amd_k8 (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_s_ref_arm80 (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq2_s_ref_amd_avx512vl (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq2_s_ref_amd_avx512 (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq2_s_ref_amd_avx2 (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq2_s_ref_amd_avx (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);
@@ -122,6 +139,7 @@ extern "C" void quantize_row_iq2_s_ref_amd_ssse3 (const float * GGML_RESTRICT x
extern "C" void quantize_row_iq2_s_ref_amd_k8 (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq2_s_ref_arm80 (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_0_amd_avx512vl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_0_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_0_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_0_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -129,6 +147,7 @@ extern "C" void quantize_row_q4_0_amd_ssse3(const float * GGML_RESTRICT x, void
extern "C" void quantize_row_q4_0_amd_k8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_0_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_1_amd_avx512vl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_1_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_1_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_1_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -136,6 +155,7 @@ extern "C" void quantize_row_q4_1_amd_ssse3(const float * GGML_RESTRICT x, void
extern "C" void quantize_row_q4_1_amd_k8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_1_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_0_amd_avx512vl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_0_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_0_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_0_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -143,6 +163,7 @@ extern "C" void quantize_row_q5_0_amd_ssse3(const float * GGML_RESTRICT x, void
extern "C" void quantize_row_q5_0_amd_k8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_0_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_1_amd_avx512vl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_1_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_1_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_1_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -150,6 +171,7 @@ extern "C" void quantize_row_q5_1_amd_ssse3(const float * GGML_RESTRICT x, void
extern "C" void quantize_row_q5_1_amd_k8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_1_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_0_amd_avx512vl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_0_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_0_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_0_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -157,6 +179,7 @@ extern "C" void quantize_row_q8_0_amd_ssse3(const float * GGML_RESTRICT x, void
extern "C" void quantize_row_q8_0_amd_k8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_0_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_1_amd_avx512vl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_1_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_1_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_1_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -164,6 +187,7 @@ extern "C" void quantize_row_q8_1_amd_ssse3(const float * GGML_RESTRICT x, void
extern "C" void quantize_row_q8_1_amd_k8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_1_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q2_K_amd_avx512vl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q2_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q2_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q2_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -171,6 +195,7 @@ extern "C" void quantize_row_q2_K_amd_ssse3(const float * GGML_RESTRICT x, void
extern "C" void quantize_row_q2_K_amd_k8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q2_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q3_K_amd_avx512vl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q3_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q3_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q3_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -178,6 +203,7 @@ extern "C" void quantize_row_q3_K_amd_ssse3(const float * GGML_RESTRICT x, void
extern "C" void quantize_row_q3_K_amd_k8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q3_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q4_K_amd_avx512vl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -185,6 +211,7 @@ extern "C" void quantize_row_q4_K_amd_ssse3(const float * GGML_RESTRICT x, void
extern "C" void quantize_row_q4_K_amd_k8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q4_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q5_K_amd_avx512vl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -192,6 +219,7 @@ extern "C" void quantize_row_q5_K_amd_ssse3(const float * GGML_RESTRICT x, void
extern "C" void quantize_row_q5_K_amd_k8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q5_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q6_K_amd_avx512vl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q6_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q6_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q6_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -199,6 +227,7 @@ extern "C" void quantize_row_q6_K_amd_ssse3(const float * GGML_RESTRICT x, void
extern "C" void quantize_row_q6_K_amd_k8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q6_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_q8_K_amd_avx512vl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_K_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_K_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_K_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -206,6 +235,7 @@ extern "C" void quantize_row_q8_K_amd_ssse3(const float * GGML_RESTRICT x, void
extern "C" void quantize_row_q8_K_amd_k8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_q8_K_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_xxs_amd_avx512vl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_xxs_amd_avx512(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_xxs_amd_avx2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_xxs_amd_avx(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -213,6 +243,7 @@ extern "C" void quantize_row_iq3_xxs_amd_ssse3(const float * GGML_RESTRICT x, vo
extern "C" void quantize_row_iq3_xxs_amd_k8(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_xxs_arm80(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_nl_amd_avx512vl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_nl_amd_avx512 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_nl_amd_avx2 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_nl_amd_avx (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -220,6 +251,7 @@ extern "C" void quantize_row_iq4_nl_amd_ssse3 (const float * GGML_RESTRICT x, vo
extern "C" void quantize_row_iq4_nl_amd_k8 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_nl_arm80 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq4_xs_amd_avx512vl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_xs_amd_avx512 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_xs_amd_avx2 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_xs_amd_avx (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -227,6 +259,7 @@ extern "C" void quantize_row_iq4_xs_amd_ssse3 (const float * GGML_RESTRICT x, vo
extern "C" void quantize_row_iq4_xs_amd_k8 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq4_xs_arm80 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq3_s_amd_avx512vl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_s_amd_avx512 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_s_amd_avx2 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_s_amd_avx (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -234,6 +267,7 @@ extern "C" void quantize_row_iq3_s_amd_ssse3 (const float * GGML_RESTRICT x, vo
extern "C" void quantize_row_iq3_s_amd_k8 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq3_s_arm80 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void quantize_row_iq2_s_amd_avx512vl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq2_s_amd_avx512 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq2_s_amd_avx2 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq2_s_amd_avx (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -241,6 +275,7 @@ extern "C" void quantize_row_iq2_s_amd_ssse3 (const float * GGML_RESTRICT x, vo
extern "C" void quantize_row_iq2_s_amd_k8 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
extern "C" void quantize_row_iq2_s_arm80 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q4_0_amd_avx512vl(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q4_0_amd_avx512(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q4_0_amd_avx2(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q4_0_amd_avx(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -248,6 +283,7 @@ extern "C" void dequantize_row_q4_0_amd_ssse3(const block_q4_0 * GGML_RESTRICT x
extern "C" void dequantize_row_q4_0_amd_k8(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q4_0_arm80(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q4_1_amd_avx512vl(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q4_1_amd_avx512(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q4_1_amd_avx2(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q4_1_amd_avx(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -255,6 +291,7 @@ extern "C" void dequantize_row_q4_1_amd_ssse3(const block_q4_1 * GGML_RESTRICT x
extern "C" void dequantize_row_q4_1_amd_k8(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q4_1_arm80(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q5_0_amd_avx512vl(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q5_0_amd_avx512(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q5_0_amd_avx2(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q5_0_amd_avx(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -262,6 +299,7 @@ extern "C" void dequantize_row_q5_0_amd_ssse3(const block_q5_0 * GGML_RESTRICT x
extern "C" void dequantize_row_q5_0_amd_k8(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q5_0_arm80(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q5_1_amd_avx512vl(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q5_1_amd_avx512(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q5_1_amd_avx2(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q5_1_amd_avx(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -269,6 +307,7 @@ extern "C" void dequantize_row_q5_1_amd_ssse3(const block_q5_1 * GGML_RESTRICT x
extern "C" void dequantize_row_q5_1_amd_k8(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q5_1_arm80(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q8_0_amd_avx512vl(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q8_0_amd_avx512(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q8_0_amd_avx2(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q8_0_amd_avx(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -276,6 +315,7 @@ extern "C" void dequantize_row_q8_0_amd_ssse3(const block_q8_0 * GGML_RESTRICT x
extern "C" void dequantize_row_q8_0_amd_k8(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q8_0_arm80(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q2_K_amd_avx512vl(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q2_K_amd_avx512(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q2_K_amd_avx2(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q2_K_amd_avx(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -283,6 +323,7 @@ extern "C" void dequantize_row_q2_K_amd_ssse3(const block_q2_K * GGML_RESTRICT x
extern "C" void dequantize_row_q2_K_amd_k8(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q2_K_arm80(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q3_K_amd_avx512vl(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q3_K_amd_avx512(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q3_K_amd_avx2(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q3_K_amd_avx(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -290,6 +331,7 @@ extern "C" void dequantize_row_q3_K_amd_ssse3(const block_q3_K * GGML_RESTRICT x
extern "C" void dequantize_row_q3_K_amd_k8(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q3_K_arm80(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q4_K_amd_avx512vl(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q4_K_amd_avx512(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q4_K_amd_avx2(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q4_K_amd_avx(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -297,6 +339,7 @@ extern "C" void dequantize_row_q4_K_amd_ssse3(const block_q4_K * GGML_RESTRICT x
extern "C" void dequantize_row_q4_K_amd_k8(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q4_K_arm80(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q5_K_amd_avx512vl(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q5_K_amd_avx512(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q5_K_amd_avx2(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q5_K_amd_avx(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -304,6 +347,7 @@ extern "C" void dequantize_row_q5_K_amd_ssse3(const block_q5_K * GGML_RESTRICT x
extern "C" void dequantize_row_q5_K_amd_k8(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q5_K_arm80(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q6_K_amd_avx512vl(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q6_K_amd_avx512(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q6_K_amd_avx2(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q6_K_amd_avx(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -311,6 +355,7 @@ extern "C" void dequantize_row_q6_K_amd_ssse3(const block_q6_K * GGML_RESTRICT x
extern "C" void dequantize_row_q6_K_amd_k8(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q6_K_arm80(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_q8_K_amd_avx512vl(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q8_K_amd_avx512(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q8_K_amd_avx2(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q8_K_amd_avx(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -318,6 +363,7 @@ extern "C" void dequantize_row_q8_K_amd_ssse3(const block_q8_K * GGML_RESTRICT x
extern "C" void dequantize_row_q8_K_amd_k8(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_q8_K_arm80(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq2_xxs_amd_avx512vl(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq2_xxs_amd_avx512(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq2_xxs_amd_avx2(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq2_xxs_amd_avx(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -325,6 +371,7 @@ extern "C" void dequantize_row_iq2_xxs_amd_ssse3(const block_iq2_xxs * GGML_REST
extern "C" void dequantize_row_iq2_xxs_amd_k8(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq2_xxs_arm80(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq2_xs_amd_avx512vl (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq2_xs_amd_avx512 (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq2_xs_amd_avx2 (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq2_xs_amd_avx (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -332,6 +379,7 @@ extern "C" void dequantize_row_iq2_xs_amd_ssse3 (const block_iq2_xs * GGML_REST
extern "C" void dequantize_row_iq2_xs_amd_k8 (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq2_xs_arm80 (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq2_s_amd_avx512vl (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq2_s_amd_avx512 (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq2_s_amd_avx2 (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq2_s_amd_avx (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -339,6 +387,7 @@ extern "C" void dequantize_row_iq2_s_amd_ssse3 (const block_iq2_s * GGML_REST
extern "C" void dequantize_row_iq2_s_amd_k8 (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq2_s_arm80 (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq3_xxs_amd_avx512vl(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq3_xxs_amd_avx512(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq3_xxs_amd_avx2(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq3_xxs_amd_avx(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -346,6 +395,7 @@ extern "C" void dequantize_row_iq3_xxs_amd_ssse3(const block_iq3_xxs * GGML_REST
extern "C" void dequantize_row_iq3_xxs_amd_k8(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq3_xxs_arm80(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq1_s_amd_avx512vl (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq1_s_amd_avx512 (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq1_s_amd_avx2 (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq1_s_amd_avx (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -353,6 +403,7 @@ extern "C" void dequantize_row_iq1_s_amd_ssse3 (const block_iq1_s * GGML_REST
extern "C" void dequantize_row_iq1_s_amd_k8 (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq1_s_arm80 (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq1_m_amd_avx512vl (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq1_m_amd_avx512 (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq1_m_amd_avx2 (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq1_m_amd_avx (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -360,6 +411,7 @@ extern "C" void dequantize_row_iq1_m_amd_ssse3 (const block_iq1_m * GGML_REST
extern "C" void dequantize_row_iq1_m_amd_k8 (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq1_m_arm80 (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq4_nl_amd_avx512vl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq4_nl_amd_avx512 (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq4_nl_amd_avx2 (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq4_nl_amd_avx (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -367,6 +419,7 @@ extern "C" void dequantize_row_iq4_nl_amd_ssse3 (const block_iq4_nl * GGML_REST
extern "C" void dequantize_row_iq4_nl_amd_k8 (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq4_nl_arm80 (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq4_xs_amd_avx512vl (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq4_xs_amd_avx512 (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq4_xs_amd_avx2 (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq4_xs_amd_avx (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -374,6 +427,7 @@ extern "C" void dequantize_row_iq4_xs_amd_ssse3 (const block_iq4_xs * GGML_REST
extern "C" void dequantize_row_iq4_xs_amd_k8 (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq4_xs_arm80 (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void dequantize_row_iq3_s_amd_avx512vl (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq3_s_amd_avx512 (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq3_s_amd_avx2 (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq3_s_amd_avx (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
@@ -381,6 +435,7 @@ extern "C" void dequantize_row_iq3_s_amd_ssse3 (const block_iq3_s * GGML_REST
extern "C" void dequantize_row_iq3_s_amd_k8 (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
extern "C" void dequantize_row_iq3_s_arm80 (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+extern "C" void ggml_vec_dot_q4_0_q8_0_amd_avx512vl(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q4_0_q8_0_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q4_0_q8_0_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q4_0_q8_0_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -388,6 +443,7 @@ extern "C" void ggml_vec_dot_q4_0_q8_0_amd_ssse3(int n, float * GGML_RESTRICT s,
extern "C" void ggml_vec_dot_q4_0_q8_0_amd_k8(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q4_0_q8_0_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q4_1_q8_1_amd_avx512vl(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q4_1_q8_1_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q4_1_q8_1_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q4_1_q8_1_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -395,6 +451,7 @@ extern "C" void ggml_vec_dot_q4_1_q8_1_amd_ssse3(int n, float * GGML_RESTRICT s,
extern "C" void ggml_vec_dot_q4_1_q8_1_amd_k8(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q4_1_q8_1_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q5_0_q8_0_amd_avx512vl(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q5_0_q8_0_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q5_0_q8_0_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q5_0_q8_0_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -402,6 +459,7 @@ extern "C" void ggml_vec_dot_q5_0_q8_0_amd_ssse3(int n, float * GGML_RESTRICT s,
extern "C" void ggml_vec_dot_q5_0_q8_0_amd_k8(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q5_0_q8_0_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q5_1_q8_1_amd_avx512vl(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q5_1_q8_1_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q5_1_q8_1_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q5_1_q8_1_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -409,6 +467,7 @@ extern "C" void ggml_vec_dot_q5_1_q8_1_amd_ssse3(int n, float * GGML_RESTRICT s,
extern "C" void ggml_vec_dot_q5_1_q8_1_amd_k8(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q5_1_q8_1_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q8_0_q8_0_amd_avx512vl(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q8_0_q8_0_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q8_0_q8_0_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q8_0_q8_0_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -416,6 +475,7 @@ extern "C" void ggml_vec_dot_q8_0_q8_0_amd_ssse3(int n, float * GGML_RESTRICT s,
extern "C" void ggml_vec_dot_q8_0_q8_0_amd_k8(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q8_0_q8_0_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q2_K_q8_K_amd_avx512vl(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q2_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q2_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q2_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -423,6 +483,7 @@ extern "C" void ggml_vec_dot_q2_K_q8_K_amd_ssse3(int n, float * GGML_RESTRICT s,
extern "C" void ggml_vec_dot_q2_K_q8_K_amd_k8(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q2_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q3_K_q8_K_amd_avx512vl(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q3_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q3_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q3_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -430,6 +491,7 @@ extern "C" void ggml_vec_dot_q3_K_q8_K_amd_ssse3(int n, float * GGML_RESTRICT s,
extern "C" void ggml_vec_dot_q3_K_q8_K_amd_k8(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q3_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q4_K_q8_K_amd_avx512vl(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q4_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q4_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q4_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -437,6 +499,7 @@ extern "C" void ggml_vec_dot_q4_K_q8_K_amd_ssse3(int n, float * GGML_RESTRICT s,
extern "C" void ggml_vec_dot_q4_K_q8_K_amd_k8(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q4_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q5_K_q8_K_amd_avx512vl(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q5_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q5_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q5_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -444,6 +507,7 @@ extern "C" void ggml_vec_dot_q5_K_q8_K_amd_ssse3(int n, float * GGML_RESTRICT s,
extern "C" void ggml_vec_dot_q5_K_q8_K_amd_k8(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q5_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_q6_K_q8_K_amd_avx512vl(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q6_K_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q6_K_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q6_K_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -451,6 +515,7 @@ extern "C" void ggml_vec_dot_q6_K_q8_K_amd_ssse3(int n, float * GGML_RESTRICT s,
extern "C" void ggml_vec_dot_q6_K_q8_K_amd_k8(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_q6_K_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq2_xxs_q8_K_amd_avx512vl(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq2_xxs_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq2_xxs_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq2_xxs_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -458,6 +523,7 @@ extern "C" void ggml_vec_dot_iq2_xxs_q8_K_amd_ssse3(int n, float * GGML_RESTRICT
extern "C" void ggml_vec_dot_iq2_xxs_q8_K_amd_k8(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq2_xxs_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq2_xs_q8_K_amd_avx512vl (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq2_xs_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq2_xs_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq2_xs_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -465,6 +531,7 @@ extern "C" void ggml_vec_dot_iq2_xs_q8_K_amd_ssse3 (int n, float * GGML_RESTRICT
extern "C" void ggml_vec_dot_iq2_xs_q8_K_amd_k8 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq2_xs_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq2_s_q8_K_amd_avx512vl (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq2_s_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq2_s_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq2_s_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -472,6 +539,7 @@ extern "C" void ggml_vec_dot_iq2_s_q8_K_amd_ssse3 (int n, float * GGML_RESTRICT
extern "C" void ggml_vec_dot_iq2_s_q8_K_amd_k8 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq2_s_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq3_xxs_q8_K_amd_avx512vl(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq3_xxs_q8_K_amd_avx512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq3_xxs_q8_K_amd_avx2(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq3_xxs_q8_K_amd_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -479,6 +547,7 @@ extern "C" void ggml_vec_dot_iq3_xxs_q8_K_amd_ssse3(int n, float * GGML_RESTRICT
extern "C" void ggml_vec_dot_iq3_xxs_q8_K_amd_k8(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq3_xxs_q8_K_arm80(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq1_s_q8_K_amd_avx512vl (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq1_s_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq1_s_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq1_s_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -486,6 +555,7 @@ extern "C" void ggml_vec_dot_iq1_s_q8_K_amd_ssse3 (int n, float * GGML_RESTRICT
extern "C" void ggml_vec_dot_iq1_s_q8_K_amd_k8 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq1_s_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq1_m_q8_K_amd_avx512vl (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq1_m_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq1_m_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq1_m_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -493,6 +563,7 @@ extern "C" void ggml_vec_dot_iq1_m_q8_K_amd_ssse3 (int n, float * GGML_RESTRICT
extern "C" void ggml_vec_dot_iq1_m_q8_K_amd_k8 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq1_m_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq4_nl_q8_0_amd_avx512vl (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq4_nl_q8_0_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq4_nl_q8_0_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq4_nl_q8_0_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -500,6 +571,7 @@ extern "C" void ggml_vec_dot_iq4_nl_q8_0_amd_ssse3 (int n, float * GGML_RESTRICT
extern "C" void ggml_vec_dot_iq4_nl_q8_0_amd_k8 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq4_nl_q8_0_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq4_xs_q8_K_amd_avx512vl (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq4_xs_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq4_xs_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq4_xs_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -507,6 +579,7 @@ extern "C" void ggml_vec_dot_iq4_xs_q8_K_amd_ssse3 (int n, float * GGML_RESTRICT
extern "C" void ggml_vec_dot_iq4_xs_q8_K_amd_k8 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq4_xs_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" void ggml_vec_dot_iq3_s_q8_K_amd_avx512vl (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq3_s_q8_K_amd_avx512 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq3_s_q8_K_amd_avx2 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq3_s_q8_K_amd_avx (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -514,6 +587,7 @@ extern "C" void ggml_vec_dot_iq3_s_q8_K_amd_ssse3 (int n, float * GGML_RESTRICT
extern "C" void ggml_vec_dot_iq3_s_q8_K_amd_k8 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
extern "C" void ggml_vec_dot_iq3_s_q8_K_arm80 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+extern "C" size_t quantize_iq2_xxs_amd_avx512vl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq2_xxs_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq2_xxs_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq2_xxs_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -521,6 +595,7 @@ extern "C" size_t quantize_iq2_xxs_amd_ssse3(const float * GGML_RESTRICT src, vo
extern "C" size_t quantize_iq2_xxs_amd_k8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq2_xxs_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq2_xs_amd_avx512vl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq2_xs_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq2_xs_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq2_xs_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -528,6 +603,7 @@ extern "C" size_t quantize_iq2_xs_amd_ssse3 (const float * GGML_RESTRICT src, vo
extern "C" size_t quantize_iq2_xs_amd_k8 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq2_xs_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq2_s_amd_avx512vl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq2_s_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq2_s_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq2_s_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -535,6 +611,7 @@ extern "C" size_t quantize_iq2_s_amd_ssse3 (const float * GGML_RESTRICT src, vo
extern "C" size_t quantize_iq2_s_amd_k8 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq2_s_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq3_xxs_amd_avx512vl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq3_xxs_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq3_xxs_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq3_xxs_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -542,6 +619,7 @@ extern "C" size_t quantize_iq3_xxs_amd_ssse3(const float * GGML_RESTRICT src, vo
extern "C" size_t quantize_iq3_xxs_amd_k8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq3_xxs_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq1_s_amd_avx512vl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq1_s_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq1_s_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq1_s_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -549,6 +627,7 @@ extern "C" size_t quantize_iq1_s_amd_ssse3 (const float * GGML_RESTRICT src, vo
extern "C" size_t quantize_iq1_s_amd_k8 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq1_s_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq1_m_amd_avx512vl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq1_m_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq1_m_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq1_m_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -556,6 +635,7 @@ extern "C" size_t quantize_iq1_m_amd_ssse3 (const float * GGML_RESTRICT src, vo
extern "C" size_t quantize_iq1_m_amd_k8 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq1_m_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq4_nl_amd_avx512vl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq4_nl_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq4_nl_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq4_nl_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -563,6 +643,7 @@ extern "C" size_t quantize_iq4_nl_amd_ssse3 (const float * GGML_RESTRICT src, vo
extern "C" size_t quantize_iq4_nl_amd_k8 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq4_nl_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq4_xs_amd_avx512vl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq4_xs_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq4_xs_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq4_xs_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -570,6 +651,7 @@ extern "C" size_t quantize_iq4_xs_amd_ssse3 (const float * GGML_RESTRICT src, vo
extern "C" size_t quantize_iq4_xs_amd_k8 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq4_xs_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_iq3_s_amd_avx512vl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq3_s_amd_avx512 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq3_s_amd_avx2 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq3_s_amd_avx (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -577,6 +659,7 @@ extern "C" size_t quantize_iq3_s_amd_ssse3 (const float * GGML_RESTRICT src, vo
extern "C" size_t quantize_iq3_s_amd_k8 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_iq3_s_arm80 (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q2_K_amd_avx512vl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q2_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q2_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q2_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -584,6 +667,7 @@ extern "C" size_t quantize_q2_K_amd_ssse3(const float * GGML_RESTRICT src, void
extern "C" size_t quantize_q2_K_amd_k8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q2_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q3_K_amd_avx512vl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q3_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q3_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q3_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -591,6 +675,7 @@ extern "C" size_t quantize_q3_K_amd_ssse3(const float * GGML_RESTRICT src, void
extern "C" size_t quantize_q3_K_amd_k8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q3_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q4_K_amd_avx512vl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q4_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q4_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q4_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -598,6 +683,7 @@ extern "C" size_t quantize_q4_K_amd_ssse3(const float * GGML_RESTRICT src, void
extern "C" size_t quantize_q4_K_amd_k8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q4_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q5_K_amd_avx512vl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q5_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q5_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q5_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -605,6 +691,7 @@ extern "C" size_t quantize_q5_K_amd_ssse3(const float * GGML_RESTRICT src, void
extern "C" size_t quantize_q5_K_amd_k8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q5_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q6_K_amd_avx512vl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q6_K_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q6_K_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q6_K_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -612,6 +699,7 @@ extern "C" size_t quantize_q6_K_amd_ssse3(const float * GGML_RESTRICT src, void
extern "C" size_t quantize_q6_K_amd_k8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q6_K_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q4_0_amd_avx512vl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q4_0_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q4_0_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q4_0_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -619,6 +707,7 @@ extern "C" size_t quantize_q4_0_amd_ssse3(const float * GGML_RESTRICT src, void
extern "C" size_t quantize_q4_0_amd_k8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q4_0_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q4_1_amd_avx512vl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q4_1_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q4_1_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q4_1_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -626,6 +715,7 @@ extern "C" size_t quantize_q4_1_amd_ssse3(const float * GGML_RESTRICT src, void
extern "C" size_t quantize_q4_1_amd_k8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q4_1_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q5_0_amd_avx512vl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q5_0_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q5_0_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q5_0_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -633,6 +723,7 @@ extern "C" size_t quantize_q5_0_amd_ssse3(const float * GGML_RESTRICT src, void
extern "C" size_t quantize_q5_0_amd_k8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q5_0_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q5_1_amd_avx512vl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q5_1_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q5_1_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q5_1_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -640,6 +731,7 @@ extern "C" size_t quantize_q5_1_amd_ssse3(const float * GGML_RESTRICT src, void
extern "C" size_t quantize_q5_1_amd_k8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q5_1_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" size_t quantize_q8_0_amd_avx512vl(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q8_0_amd_avx512(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q8_0_amd_avx2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q8_0_amd_avx(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
@@ -647,6 +739,7 @@ extern "C" size_t quantize_q8_0_amd_ssse3(const float * GGML_RESTRICT src, void
extern "C" size_t quantize_q8_0_amd_k8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
extern "C" size_t quantize_q8_0_arm80(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+extern "C" void iq2xs_init_impl_amd_avx512vl(enum ggml_type type);
extern "C" void iq2xs_init_impl_amd_avx512(enum ggml_type type);
extern "C" void iq2xs_init_impl_amd_avx2(enum ggml_type type);
extern "C" void iq2xs_init_impl_amd_avx(enum ggml_type type);
@@ -654,6 +747,7 @@ extern "C" void iq2xs_init_impl_amd_ssse3(enum ggml_type type);
extern "C" void iq2xs_init_impl_amd_k8(enum ggml_type type);
extern "C" void iq2xs_init_impl_arm80(enum ggml_type type);
+extern "C" void iq2xs_free_impl_amd_avx512vl(enum ggml_type type);
extern "C" void iq2xs_free_impl_amd_avx512(enum ggml_type type);
extern "C" void iq2xs_free_impl_amd_avx2(enum ggml_type type);
extern "C" void iq2xs_free_impl_amd_avx(enum ggml_type type);
@@ -661,6 +755,7 @@ extern "C" void iq2xs_free_impl_amd_ssse3(enum ggml_type type);
extern "C" void iq2xs_free_impl_amd_k8(enum ggml_type type);
extern "C" void iq2xs_free_impl_arm80(enum ggml_type type);
+extern "C" void iq3xs_init_impl_amd_avx512vl(int grid_size);
extern "C" void iq3xs_init_impl_amd_avx512(int grid_size);
extern "C" void iq3xs_init_impl_amd_avx2(int grid_size);
extern "C" void iq3xs_init_impl_amd_avx(int grid_size);
@@ -668,6 +763,7 @@ extern "C" void iq3xs_init_impl_amd_ssse3(int grid_size);
extern "C" void iq3xs_init_impl_amd_k8(int grid_size);
extern "C" void iq3xs_init_impl_arm80(int grid_size);
+extern "C" void iq3xs_free_impl_amd_avx512vl(int grid_size);
extern "C" void iq3xs_free_impl_amd_avx512(int grid_size);
extern "C" void iq3xs_free_impl_amd_avx2(int grid_size);
extern "C" void iq3xs_free_impl_amd_avx(int grid_size);
@@ -675,6 +771,7 @@ extern "C" void iq3xs_free_impl_amd_ssse3(int grid_size);
extern "C" void iq3xs_free_impl_amd_k8(int grid_size);
extern "C" void iq3xs_free_impl_arm80(int grid_size);
+extern "C" bool ggml_validate_row_data_amd_avx512vl(enum ggml_type type, const void * data, size_t nbytes);
extern "C" bool ggml_validate_row_data_amd_avx512(enum ggml_type type, const void * data, size_t nbytes);
extern "C" bool ggml_validate_row_data_amd_avx2(enum ggml_type type, const void * data, size_t nbytes);
extern "C" bool ggml_validate_row_data_amd_avx(enum ggml_type type, const void * data, size_t nbytes);
@@ -782,6 +879,108 @@ static const struct QuantFuncs {
typeof(ggml_validate_row_data) *ptr_ggml_validate_row_data;
QuantFuncs() {
+#ifdef __x86_64__
+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX) && X86_HAVE(AVX2) && X86_HAVE(AVX512F) && X86_HAVE(AVX512BW) && X86_HAVE(AVX512DQ) && X86_HAVE(AVX512VL)) {
+ ptr_quantize_row_q4_0_ref = quantize_row_q4_0_ref_amd_avx512vl;
+ ptr_quantize_row_q4_1_ref = quantize_row_q4_1_ref_amd_avx512vl;
+ ptr_quantize_row_q5_0_ref = quantize_row_q5_0_ref_amd_avx512vl;
+ ptr_quantize_row_q5_1_ref = quantize_row_q5_1_ref_amd_avx512vl;
+ ptr_quantize_row_q8_0_ref = quantize_row_q8_0_ref_amd_avx512vl;
+ ptr_quantize_row_q8_1_ref = quantize_row_q8_1_ref_amd_avx512vl;
+ ptr_quantize_row_q2_K_ref = quantize_row_q2_K_ref_amd_avx512vl;
+ ptr_quantize_row_q3_K_ref = quantize_row_q3_K_ref_amd_avx512vl;
+ ptr_quantize_row_q4_K_ref = quantize_row_q4_K_ref_amd_avx512vl;
+ ptr_quantize_row_q5_K_ref = quantize_row_q5_K_ref_amd_avx512vl;
+ ptr_quantize_row_q6_K_ref = quantize_row_q6_K_ref_amd_avx512vl;
+ ptr_quantize_row_q8_K_ref = quantize_row_q8_K_ref_amd_avx512vl;
+ ptr_quantize_row_iq3_xxs_ref = quantize_row_iq3_xxs_ref_amd_avx512vl;
+ ptr_quantize_row_iq4_nl_ref = quantize_row_iq4_nl_ref_amd_avx512vl;
+ ptr_quantize_row_iq4_xs_ref = quantize_row_iq4_xs_ref_amd_avx512vl;
+ ptr_quantize_row_iq3_s_ref = quantize_row_iq3_s_ref_amd_avx512vl;
+ ptr_quantize_row_iq2_s_ref = quantize_row_iq2_s_ref_amd_avx512vl;
+ ptr_quantize_row_q4_0 = quantize_row_q4_0_amd_avx512vl;
+ ptr_quantize_row_q4_1 = quantize_row_q4_1_amd_avx512vl;
+ ptr_quantize_row_q5_0 = quantize_row_q5_0_amd_avx512vl;
+ ptr_quantize_row_q5_1 = quantize_row_q5_1_amd_avx512vl;
+ ptr_quantize_row_q8_0 = quantize_row_q8_0_amd_avx512vl;
+ ptr_quantize_row_q8_1 = quantize_row_q8_1_amd_avx512vl;
+ ptr_quantize_row_q2_K = quantize_row_q2_K_amd_avx512vl;
+ ptr_quantize_row_q3_K = quantize_row_q3_K_amd_avx512vl;
+ ptr_quantize_row_q4_K = quantize_row_q4_K_amd_avx512vl;
+ ptr_quantize_row_q5_K = quantize_row_q5_K_amd_avx512vl;
+ ptr_quantize_row_q6_K = quantize_row_q6_K_amd_avx512vl;
+ ptr_quantize_row_q8_K = quantize_row_q8_K_amd_avx512vl;
+ ptr_quantize_row_iq3_xxs = quantize_row_iq3_xxs_amd_avx512vl;
+ ptr_quantize_row_iq4_nl = quantize_row_iq4_nl_amd_avx512vl;
+ ptr_quantize_row_iq4_xs = quantize_row_iq4_xs_amd_avx512vl;
+ ptr_quantize_row_iq3_s = quantize_row_iq3_s_amd_avx512vl;
+ ptr_quantize_row_iq2_s = quantize_row_iq2_s_amd_avx512vl;
+ ptr_dequantize_row_q4_0 = dequantize_row_q4_0_amd_avx512vl;
+ ptr_dequantize_row_q4_1 = dequantize_row_q4_1_amd_avx512vl;
+ ptr_dequantize_row_q5_0 = dequantize_row_q5_0_amd_avx512vl;
+ ptr_dequantize_row_q5_1 = dequantize_row_q5_1_amd_avx512vl;
+ ptr_dequantize_row_q8_0 = dequantize_row_q8_0_amd_avx512vl;
+ ptr_dequantize_row_q2_K = dequantize_row_q2_K_amd_avx512vl;
+ ptr_dequantize_row_q3_K = dequantize_row_q3_K_amd_avx512vl;
+ ptr_dequantize_row_q4_K = dequantize_row_q4_K_amd_avx512vl;
+ ptr_dequantize_row_q5_K = dequantize_row_q5_K_amd_avx512vl;
+ ptr_dequantize_row_q6_K = dequantize_row_q6_K_amd_avx512vl;
+ ptr_dequantize_row_q8_K = dequantize_row_q8_K_amd_avx512vl;
+ ptr_dequantize_row_iq2_xxs = dequantize_row_iq2_xxs_amd_avx512vl;
+ ptr_dequantize_row_iq2_xs = dequantize_row_iq2_xs_amd_avx512vl;
+ ptr_dequantize_row_iq2_s = dequantize_row_iq2_s_amd_avx512vl;
+ ptr_dequantize_row_iq3_xxs = dequantize_row_iq3_xxs_amd_avx512vl;
+ ptr_dequantize_row_iq1_s = dequantize_row_iq1_s_amd_avx512vl;
+ ptr_dequantize_row_iq1_m = dequantize_row_iq1_m_amd_avx512vl;
+ ptr_dequantize_row_iq4_nl = dequantize_row_iq4_nl_amd_avx512vl;
+ ptr_dequantize_row_iq4_xs = dequantize_row_iq4_xs_amd_avx512vl;
+ ptr_dequantize_row_iq3_s = dequantize_row_iq3_s_amd_avx512vl;
+ ptr_ggml_vec_dot_q4_0_q8_0 = ggml_vec_dot_q4_0_q8_0_amd_avx512vl;
+ ptr_ggml_vec_dot_q4_1_q8_1 = ggml_vec_dot_q4_1_q8_1_amd_avx512vl;
+ ptr_ggml_vec_dot_q5_0_q8_0 = ggml_vec_dot_q5_0_q8_0_amd_avx512vl;
+ ptr_ggml_vec_dot_q5_1_q8_1 = ggml_vec_dot_q5_1_q8_1_amd_avx512vl;
+ ptr_ggml_vec_dot_q8_0_q8_0 = ggml_vec_dot_q8_0_q8_0_amd_avx512vl;
+ ptr_ggml_vec_dot_q2_K_q8_K = ggml_vec_dot_q2_K_q8_K_amd_avx512vl;
+ ptr_ggml_vec_dot_q3_K_q8_K = ggml_vec_dot_q3_K_q8_K_amd_avx512vl;
+ ptr_ggml_vec_dot_q4_K_q8_K = ggml_vec_dot_q4_K_q8_K_amd_avx512vl;
+ ptr_ggml_vec_dot_q5_K_q8_K = ggml_vec_dot_q5_K_q8_K_amd_avx512vl;
+ ptr_ggml_vec_dot_q6_K_q8_K = ggml_vec_dot_q6_K_q8_K_amd_avx512vl;
+ ptr_ggml_vec_dot_iq2_xxs_q8_K = ggml_vec_dot_iq2_xxs_q8_K_amd_avx512vl;
+ ptr_ggml_vec_dot_iq2_xs_q8_K = ggml_vec_dot_iq2_xs_q8_K_amd_avx512vl;
+ ptr_ggml_vec_dot_iq2_s_q8_K = ggml_vec_dot_iq2_s_q8_K_amd_avx512vl;
+ ptr_ggml_vec_dot_iq3_xxs_q8_K = ggml_vec_dot_iq3_xxs_q8_K_amd_avx512vl;
+ ptr_ggml_vec_dot_iq1_s_q8_K = ggml_vec_dot_iq1_s_q8_K_amd_avx512vl;
+ ptr_ggml_vec_dot_iq1_m_q8_K = ggml_vec_dot_iq1_m_q8_K_amd_avx512vl;
+ ptr_ggml_vec_dot_iq4_nl_q8_0 = ggml_vec_dot_iq4_nl_q8_0_amd_avx512vl;
+ ptr_ggml_vec_dot_iq4_xs_q8_K = ggml_vec_dot_iq4_xs_q8_K_amd_avx512vl;
+ ptr_ggml_vec_dot_iq3_s_q8_K = ggml_vec_dot_iq3_s_q8_K_amd_avx512vl;
+ ptr_quantize_iq2_xxs = quantize_iq2_xxs_amd_avx512vl;
+ ptr_quantize_iq2_xs = quantize_iq2_xs_amd_avx512vl;
+ ptr_quantize_iq2_s = quantize_iq2_s_amd_avx512vl;
+ ptr_quantize_iq3_xxs = quantize_iq3_xxs_amd_avx512vl;
+ ptr_quantize_iq1_s = quantize_iq1_s_amd_avx512vl;
+ ptr_quantize_iq1_m = quantize_iq1_m_amd_avx512vl;
+ ptr_quantize_iq4_nl = quantize_iq4_nl_amd_avx512vl;
+ ptr_quantize_iq4_xs = quantize_iq4_xs_amd_avx512vl;
+ ptr_quantize_iq3_s = quantize_iq3_s_amd_avx512vl;
+ ptr_quantize_q2_K = quantize_q2_K_amd_avx512vl;
+ ptr_quantize_q3_K = quantize_q3_K_amd_avx512vl;
+ ptr_quantize_q4_K = quantize_q4_K_amd_avx512vl;
+ ptr_quantize_q5_K = quantize_q5_K_amd_avx512vl;
+ ptr_quantize_q6_K = quantize_q6_K_amd_avx512vl;
+ ptr_quantize_q4_0 = quantize_q4_0_amd_avx512vl;
+ ptr_quantize_q4_1 = quantize_q4_1_amd_avx512vl;
+ ptr_quantize_q5_0 = quantize_q5_0_amd_avx512vl;
+ ptr_quantize_q5_1 = quantize_q5_1_amd_avx512vl;
+ ptr_quantize_q8_0 = quantize_q8_0_amd_avx512vl;
+ ptr_iq2xs_init_impl = iq2xs_init_impl_amd_avx512vl;
+ ptr_iq2xs_free_impl = iq2xs_free_impl_amd_avx512vl;
+ ptr_iq3xs_init_impl = iq3xs_init_impl_amd_avx512vl;
+ ptr_iq3xs_free_impl = iq3xs_free_impl_amd_avx512vl;
+ ptr_ggml_validate_row_data = ggml_validate_row_data_amd_avx512vl;
+ return;
+ }
+#endif
#ifdef __x86_64__
if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX) && X86_HAVE(AVX2) && X86_HAVE(AVX512F)) {
ptr_quantize_row_q4_0_ref = quantize_row_q4_0_ref_amd_avx512;
diff --git a/llama.cpp/ggml-quants.py b/llama.cpp/ggml-quants.py
index 48b4e972..320a98e0 100644
--- a/llama.cpp/ggml-quants.py
+++ b/llama.cpp/ggml-quants.py
@@ -12,6 +12,7 @@ FUNCS.append(('ggml_validate_row_data', 'bool ggml_validate_row_data(enum ggml_t
# END SPECIAL FUNCTIONS
ARCHS = (
+ ('amd_avx512vl', '__x86_64__', ('FMA', 'F16C', 'AVX', 'AVX2', 'AVX512F', 'AVX512BW', 'AVX512DQ', 'AVX512VL')),
('amd_avx512', '__x86_64__', ('FMA', 'F16C', 'AVX', 'AVX2', 'AVX512F')),
('amd_avx2', '__x86_64__', ('FMA', 'F16C', 'AVX', 'AVX2')),
('amd_avx', '__x86_64__', ('AVX',)),
diff --git a/llama.cpp/ggml-vector-amd-avx512vl.c b/llama.cpp/ggml-vector-amd-avx512vl.c
new file mode 100644
index 00000000..4e13c22d
--- /dev/null
+++ b/llama.cpp/ggml-vector-amd-avx512vl.c
@@ -0,0 +1,59 @@
+#ifdef __x86_64__
+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_amd_avx512vl
+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx512vl
+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx512vl
+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx512vl
+#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx512vl
+#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx512vl
+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx512vl
+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_amd_avx512vl
+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_amd_avx512vl
+#define ggml_vec_set_i8 ggml_vec_set_i8_amd_avx512vl
+#define ggml_vec_set_i16 ggml_vec_set_i16_amd_avx512vl
+#define ggml_vec_set_i32 ggml_vec_set_i32_amd_avx512vl
+#define ggml_vec_set_f16 ggml_vec_set_f16_amd_avx512vl
+#define ggml_vec_set_bf16 ggml_vec_set_bf16_amd_avx512vl
+#define ggml_vec_add_f32 ggml_vec_add_f32_amd_avx512vl
+#define ggml_vec_add1_f32 ggml_vec_add1_f32_amd_avx512vl
+#define ggml_vec_acc_f32 ggml_vec_acc_f32_amd_avx512vl
+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_amd_avx512vl
+#define ggml_vec_sub_f32 ggml_vec_sub_f32_amd_avx512vl
+#define ggml_vec_set_f32 ggml_vec_set_f32_amd_avx512vl
+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_amd_avx512vl
+#define ggml_vec_neg_f32 ggml_vec_neg_f32_amd_avx512vl
+#define ggml_vec_mul_f32 ggml_vec_mul_f32_amd_avx512vl
+#define ggml_vec_div_f32 ggml_vec_div_f32_amd_avx512vl
+#define ggml_vec_scale_f32 ggml_vec_scale_f32_amd_avx512vl
+#define ggml_vec_scale_f16 ggml_vec_scale_f16_amd_avx512vl
+#define ggml_vec_mad_f32 ggml_vec_mad_f32_amd_avx512vl
+#define ggml_vec_mad_f16 ggml_vec_mad_f16_amd_avx512vl
+#define ggml_vec_norm_f32 ggml_vec_norm_f32_amd_avx512vl
+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_amd_avx512vl
+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_amd_avx512vl
+#define ggml_vec_log_f32 ggml_vec_log_f32_amd_avx512vl
+#define ggml_vec_abs_f32 ggml_vec_abs_f32_amd_avx512vl
+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_amd_avx512vl
+#define ggml_vec_step_f32 ggml_vec_step_f32_amd_avx512vl
+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_amd_avx512vl
+#define ggml_vec_elu_f32 ggml_vec_elu_f32_amd_avx512vl
+#define ggml_vec_relu_f32 ggml_vec_relu_f32_amd_avx512vl
+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_amd_avx512vl
+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_amd_avx512vl
+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_amd_avx512vl
+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_amd_avx512vl
+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_amd_avx512vl
+#define ggml_vec_silu_f32 ggml_vec_silu_f32_amd_avx512vl
+#define ggml_silu_backward_f32 ggml_silu_backward_f32_amd_avx512vl
+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_amd_avx512vl
+#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_avx512vl
+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_avx512vl
+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_avx512vl
+#define ggml_vec_sum_bf16_ggf ggml_vec_sum_bf16_ggf_amd_avx512vl
+#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx512vl
+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx512vl
+#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_avx512vl
+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx512vl
+#define ggml_vec_sigmoid_f32 ggml_vec_sigmoid_f32_amd_avx512vl
+#define GGML_VECTOR
+#include "ggml-vector.inc"
+#endif // __x86_64__
diff --git a/llama.cpp/ggml-vector.cpp b/llama.cpp/ggml-vector.cpp
index 03145b0f..f370ee6d 100644
--- a/llama.cpp/ggml-vector.cpp
+++ b/llama.cpp/ggml-vector.cpp
@@ -4,6 +4,7 @@
#include "ggml-vector.h"
extern "C" void ggml_fp16_to_fp32_row_amd_avx512bf16(const ggml_fp16_t * x, float * y, int64_t n);
+extern "C" void ggml_fp16_to_fp32_row_amd_avx512vl(const ggml_fp16_t * x, float * y, int64_t n);
extern "C" void ggml_fp16_to_fp32_row_amd_avx512(const ggml_fp16_t * x, float * y, int64_t n);
extern "C" void ggml_fp16_to_fp32_row_amd_avx2(const ggml_fp16_t * x, float * y, int64_t n);
extern "C" void ggml_fp16_to_fp32_row_amd_f16c(const ggml_fp16_t * x, float * y, int64_t n);
@@ -15,6 +16,7 @@ extern "C" void ggml_fp16_to_fp32_row_arm82(const ggml_fp16_t * x, float * y, in
extern "C" void ggml_fp16_to_fp32_row_arm80(const ggml_fp16_t * x, float * y, int64_t n);
extern "C" void ggml_fp32_to_fp16_row_amd_avx512bf16(const float * x, ggml_fp16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_fp16_row_amd_avx512vl(const float * x, ggml_fp16_t * y, int64_t n);
extern "C" void ggml_fp32_to_fp16_row_amd_avx512(const float * x, ggml_fp16_t * y, int64_t n);
extern "C" void ggml_fp32_to_fp16_row_amd_avx2(const float * x, ggml_fp16_t * y, int64_t n);
extern "C" void ggml_fp32_to_fp16_row_amd_f16c(const float * x, ggml_fp16_t * y, int64_t n);
@@ -26,6 +28,7 @@ extern "C" void ggml_fp32_to_fp16_row_arm82(const float * x, ggml_fp16_t * y, in
extern "C" void ggml_fp32_to_fp16_row_arm80(const float * x, ggml_fp16_t * y, int64_t n);
extern "C" void ggml_bf16_to_fp32_row_amd_avx512bf16(const ggml_bf16_t * x, float * y, int64_t n);
+extern "C" void ggml_bf16_to_fp32_row_amd_avx512vl(const ggml_bf16_t * x, float * y, int64_t n);
extern "C" void ggml_bf16_to_fp32_row_amd_avx512(const ggml_bf16_t * x, float * y, int64_t n);
extern "C" void ggml_bf16_to_fp32_row_amd_avx2(const ggml_bf16_t * x, float * y, int64_t n);
extern "C" void ggml_bf16_to_fp32_row_amd_f16c(const ggml_bf16_t * x, float * y, int64_t n);
@@ -37,6 +40,7 @@ extern "C" void ggml_bf16_to_fp32_row_arm82(const ggml_bf16_t * x, float * y, in
extern "C" void ggml_bf16_to_fp32_row_arm80(const ggml_bf16_t * x, float * y, int64_t n);
extern "C" void ggml_fp32_to_bf16_row_amd_avx512bf16(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_amd_avx512vl(const float * x, ggml_bf16_t * y, int64_t n);
extern "C" void ggml_fp32_to_bf16_row_amd_avx512(const float * x, ggml_bf16_t * y, int64_t n);
extern "C" void ggml_fp32_to_bf16_row_amd_avx2(const float * x, ggml_bf16_t * y, int64_t n);
extern "C" void ggml_fp32_to_bf16_row_amd_f16c(const float * x, ggml_bf16_t * y, int64_t n);
@@ -48,6 +52,7 @@ extern "C" void ggml_fp32_to_bf16_row_arm82(const float * x, ggml_bf16_t * y, in
extern "C" void ggml_fp32_to_bf16_row_arm80(const float * x, ggml_bf16_t * y, int64_t n);
extern "C" void ggml_vec_dot_f32_amd_avx512bf16(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_amd_avx512vl(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
extern "C" void ggml_vec_dot_f32_amd_avx512(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
extern "C" void ggml_vec_dot_f32_amd_avx2(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
extern "C" void ggml_vec_dot_f32_amd_f16c(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
@@ -59,6 +64,7 @@ extern "C" void ggml_vec_dot_f32_arm82(int n, float * s, size_t bs, const float
extern "C" void ggml_vec_dot_f32_arm80(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
extern "C" void ggml_vec_dot_f16_amd_avx512bf16(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_amd_avx512vl(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
extern "C" void ggml_vec_dot_f16_amd_avx512(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
extern "C" void ggml_vec_dot_f16_amd_avx2(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
extern "C" void ggml_vec_dot_f16_amd_f16c(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
@@ -70,6 +76,7 @@ extern "C" void ggml_vec_dot_f16_arm82(int n, float * s, size_t bs, ggml_fp16_t
extern "C" void ggml_vec_dot_f16_arm80(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
extern "C" void ggml_vec_dot_bf16_amd_avx512bf16(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_amd_avx512vl(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
extern "C" void ggml_vec_dot_bf16_amd_avx512(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
extern "C" void ggml_vec_dot_bf16_amd_avx2(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
extern "C" void ggml_vec_dot_bf16_amd_f16c(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
@@ -81,6 +88,7 @@ extern "C" void ggml_vec_dot_bf16_arm82(int n, float * s, size_t bs, ggml_bf16_t
extern "C" void ggml_vec_dot_bf16_arm80(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
extern "C" void ggml_vec_dot_f16_unroll_amd_avx512bf16(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
+extern "C" void ggml_vec_dot_f16_unroll_amd_avx512vl(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
extern "C" void ggml_vec_dot_f16_unroll_amd_avx512(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
extern "C" void ggml_vec_dot_f16_unroll_amd_avx2(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
extern "C" void ggml_vec_dot_f16_unroll_amd_f16c(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
@@ -92,6 +100,7 @@ extern "C" void ggml_vec_dot_f16_unroll_arm82(const int n, const int xs, float *
extern "C" void ggml_vec_dot_f16_unroll_arm80(const int n, const int xs, float * s, void * xv, ggml_fp16_t * y);
extern "C" void ggml_vec_mad_f32_unroll_amd_avx512bf16(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
+extern "C" void ggml_vec_mad_f32_unroll_amd_avx512vl(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
extern "C" void ggml_vec_mad_f32_unroll_amd_avx512(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
extern "C" void ggml_vec_mad_f32_unroll_amd_avx2(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
extern "C" void ggml_vec_mad_f32_unroll_amd_f16c(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
@@ -103,6 +112,7 @@ extern "C" void ggml_vec_mad_f32_unroll_arm82(const int n, const int xs, const i
extern "C" void ggml_vec_mad_f32_unroll_arm80(const int n, const int xs, const int vs, float * y, const float * xv, const float * vv);
extern "C" void ggml_vec_set_i8_amd_avx512bf16(const int n, int8_t * x, const int8_t v);
+extern "C" void ggml_vec_set_i8_amd_avx512vl(const int n, int8_t * x, const int8_t v);
extern "C" void ggml_vec_set_i8_amd_avx512(const int n, int8_t * x, const int8_t v);
extern "C" void ggml_vec_set_i8_amd_avx2(const int n, int8_t * x, const int8_t v);
extern "C" void ggml_vec_set_i8_amd_f16c(const int n, int8_t * x, const int8_t v);
@@ -114,6 +124,7 @@ extern "C" void ggml_vec_set_i8_arm82(const int n, int8_t * x, const int8_t v);
extern "C" void ggml_vec_set_i8_arm80(const int n, int8_t * x, const int8_t v);
extern "C" void ggml_vec_set_i16_amd_avx512bf16(const int n, int16_t * x, const int16_t v);
+extern "C" void ggml_vec_set_i16_amd_avx512vl(const int n, int16_t * x, const int16_t v);
extern "C" void ggml_vec_set_i16_amd_avx512(const int n, int16_t * x, const int16_t v);
extern "C" void ggml_vec_set_i16_amd_avx2(const int n, int16_t * x, const int16_t v);
extern "C" void ggml_vec_set_i16_amd_f16c(const int n, int16_t * x, const int16_t v);
@@ -125,6 +136,7 @@ extern "C" void ggml_vec_set_i16_arm82(const int n, int16_t * x, const int16_t v
extern "C" void ggml_vec_set_i16_arm80(const int n, int16_t * x, const int16_t v);
extern "C" void ggml_vec_set_i32_amd_avx512bf16(const int n, int32_t * x, const int32_t v);
+extern "C" void ggml_vec_set_i32_amd_avx512vl(const int n, int32_t * x, const int32_t v);
extern "C" void ggml_vec_set_i32_amd_avx512(const int n, int32_t * x, const int32_t v);
extern "C" void ggml_vec_set_i32_amd_avx2(const int n, int32_t * x, const int32_t v);
extern "C" void ggml_vec_set_i32_amd_f16c(const int n, int32_t * x, const int32_t v);
@@ -136,6 +148,7 @@ extern "C" void ggml_vec_set_i32_arm82(const int n, int32_t * x, const int32_t v
extern "C" void ggml_vec_set_i32_arm80(const int n, int32_t * x, const int32_t v);
extern "C" void ggml_vec_set_f16_amd_avx512bf16(const int n, ggml_fp16_t * x, const int32_t v);
+extern "C" void ggml_vec_set_f16_amd_avx512vl(const int n, ggml_fp16_t * x, const int32_t v);
extern "C" void ggml_vec_set_f16_amd_avx512(const int n, ggml_fp16_t * x, const int32_t v);
extern "C" void ggml_vec_set_f16_amd_avx2(const int n, ggml_fp16_t * x, const int32_t v);
extern "C" void ggml_vec_set_f16_amd_f16c(const int n, ggml_fp16_t * x, const int32_t v);
@@ -147,6 +160,7 @@ extern "C" void ggml_vec_set_f16_arm82(const int n, ggml_fp16_t * x, const int32
extern "C" void ggml_vec_set_f16_arm80(const int n, ggml_fp16_t * x, const int32_t v);
extern "C" void ggml_vec_set_bf16_amd_avx512bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
+extern "C" void ggml_vec_set_bf16_amd_avx512vl(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
extern "C" void ggml_vec_set_bf16_amd_avx512(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
extern "C" void ggml_vec_set_bf16_amd_avx2(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
extern "C" void ggml_vec_set_bf16_amd_f16c(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
@@ -158,6 +172,7 @@ extern "C" void ggml_vec_set_bf16_arm82(const int n, ggml_bf16_t * x, const ggml
extern "C" void ggml_vec_set_bf16_arm80(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
extern "C" void ggml_vec_add_f32_amd_avx512bf16 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_add_f32_amd_avx512vl (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_add_f32_amd_avx512 (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_add_f32_amd_avx2 (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_add_f32_amd_f16c (const int n, float * z, const float * x, const float * y);
@@ -169,6 +184,7 @@ extern "C" void ggml_vec_add_f32_arm82 (const int n, float * z, const float * x,
extern "C" void ggml_vec_add_f32_arm80 (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_add1_f32_amd_avx512bf16(const int n, float * z, const float * x, const float v);
+extern "C" void ggml_vec_add1_f32_amd_avx512vl(const int n, float * z, const float * x, const float v);
extern "C" void ggml_vec_add1_f32_amd_avx512(const int n, float * z, const float * x, const float v);
extern "C" void ggml_vec_add1_f32_amd_avx2(const int n, float * z, const float * x, const float v);
extern "C" void ggml_vec_add1_f32_amd_f16c(const int n, float * z, const float * x, const float v);
@@ -180,6 +196,7 @@ extern "C" void ggml_vec_add1_f32_arm82(const int n, float * z, const float * x,
extern "C" void ggml_vec_add1_f32_arm80(const int n, float * z, const float * x, const float v);
extern "C" void ggml_vec_acc_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_acc_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_acc_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_acc_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_acc_f32_amd_f16c (const int n, float * y, const float * x);
@@ -191,6 +208,7 @@ extern "C" void ggml_vec_acc_f32_arm82 (const int n, float * y, const float * x)
extern "C" void ggml_vec_acc_f32_arm80 (const int n, float * y, const float * x);
extern "C" void ggml_vec_acc1_f32_amd_avx512bf16(const int n, float * y, const float v);
+extern "C" void ggml_vec_acc1_f32_amd_avx512vl(const int n, float * y, const float v);
extern "C" void ggml_vec_acc1_f32_amd_avx512(const int n, float * y, const float v);
extern "C" void ggml_vec_acc1_f32_amd_avx2(const int n, float * y, const float v);
extern "C" void ggml_vec_acc1_f32_amd_f16c(const int n, float * y, const float v);
@@ -202,6 +220,7 @@ extern "C" void ggml_vec_acc1_f32_arm82(const int n, float * y, const float v)
extern "C" void ggml_vec_acc1_f32_arm80(const int n, float * y, const float v);
extern "C" void ggml_vec_sub_f32_amd_avx512bf16 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_sub_f32_amd_avx512vl (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_sub_f32_amd_avx512 (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_sub_f32_amd_avx2 (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_sub_f32_amd_f16c (const int n, float * z, const float * x, const float * y);
@@ -213,6 +232,7 @@ extern "C" void ggml_vec_sub_f32_arm82 (const int n, float * z, const float * x,
extern "C" void ggml_vec_sub_f32_arm80 (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_set_f32_amd_avx512bf16 (const int n, float * x, const float v);
+extern "C" void ggml_vec_set_f32_amd_avx512vl (const int n, float * x, const float v);
extern "C" void ggml_vec_set_f32_amd_avx512 (const int n, float * x, const float v);
extern "C" void ggml_vec_set_f32_amd_avx2 (const int n, float * x, const float v);
extern "C" void ggml_vec_set_f32_amd_f16c (const int n, float * x, const float v);
@@ -224,6 +244,7 @@ extern "C" void ggml_vec_set_f32_arm82 (const int n, float * x, const float v)
extern "C" void ggml_vec_set_f32_arm80 (const int n, float * x, const float v);
extern "C" void ggml_vec_cpy_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_cpy_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_cpy_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_cpy_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_cpy_f32_amd_f16c (const int n, float * y, const float * x);
@@ -235,6 +256,7 @@ extern "C" void ggml_vec_cpy_f32_arm82 (const int n, float * y, const float * x)
extern "C" void ggml_vec_cpy_f32_arm80 (const int n, float * y, const float * x);
extern "C" void ggml_vec_neg_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_neg_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_neg_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_neg_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_neg_f32_amd_f16c (const int n, float * y, const float * x);
@@ -246,6 +268,7 @@ extern "C" void ggml_vec_neg_f32_arm82 (const int n, float * y, const float * x)
extern "C" void ggml_vec_neg_f32_arm80 (const int n, float * y, const float * x);
extern "C" void ggml_vec_mul_f32_amd_avx512bf16 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_mul_f32_amd_avx512vl (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_mul_f32_amd_avx512 (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_mul_f32_amd_avx2 (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_mul_f32_amd_f16c (const int n, float * z, const float * x, const float * y);
@@ -257,6 +280,7 @@ extern "C" void ggml_vec_mul_f32_arm82 (const int n, float * z, const float * x,
extern "C" void ggml_vec_mul_f32_arm80 (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_div_f32_amd_avx512bf16 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_div_f32_amd_avx512vl (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_div_f32_amd_avx512 (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_div_f32_amd_avx2 (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_div_f32_amd_f16c (const int n, float * z, const float * x, const float * y);
@@ -268,6 +292,7 @@ extern "C" void ggml_vec_div_f32_arm82 (const int n, float * z, const float * x,
extern "C" void ggml_vec_div_f32_arm80 (const int n, float * z, const float * x, const float * y);
extern "C" void ggml_vec_scale_f32_amd_avx512bf16(const int n, float * y, const float v);
+extern "C" void ggml_vec_scale_f32_amd_avx512vl(const int n, float * y, const float v);
extern "C" void ggml_vec_scale_f32_amd_avx512(const int n, float * y, const float v);
extern "C" void ggml_vec_scale_f32_amd_avx2(const int n, float * y, const float v);
extern "C" void ggml_vec_scale_f32_amd_f16c(const int n, float * y, const float v);
@@ -279,6 +304,7 @@ extern "C" void ggml_vec_scale_f32_arm82(const int n, float * y, const float v
extern "C" void ggml_vec_scale_f32_arm80(const int n, float * y, const float v);
extern "C" void ggml_vec_scale_f16_amd_avx512bf16(const int n, ggml_fp16_t * y, const float v);
+extern "C" void ggml_vec_scale_f16_amd_avx512vl(const int n, ggml_fp16_t * y, const float v);
extern "C" void ggml_vec_scale_f16_amd_avx512(const int n, ggml_fp16_t * y, const float v);
extern "C" void ggml_vec_scale_f16_amd_avx2(const int n, ggml_fp16_t * y, const float v);
extern "C" void ggml_vec_scale_f16_amd_f16c(const int n, ggml_fp16_t * y, const float v);
@@ -290,6 +316,7 @@ extern "C" void ggml_vec_scale_f16_arm82(const int n, ggml_fp16_t * y, const flo
extern "C" void ggml_vec_scale_f16_arm80(const int n, ggml_fp16_t * y, const float v);
extern "C" void ggml_vec_mad_f32_amd_avx512bf16(const int n, float * y, const float * x, const float v);
+extern "C" void ggml_vec_mad_f32_amd_avx512vl(const int n, float * y, const float * x, const float v);
extern "C" void ggml_vec_mad_f32_amd_avx512(const int n, float * y, const float * x, const float v);
extern "C" void ggml_vec_mad_f32_amd_avx2(const int n, float * y, const float * x, const float v);
extern "C" void ggml_vec_mad_f32_amd_f16c(const int n, float * y, const float * x, const float v);
@@ -301,6 +328,7 @@ extern "C" void ggml_vec_mad_f32_arm82(const int n, float * y, const float * x,
extern "C" void ggml_vec_mad_f32_arm80(const int n, float * y, const float * x, const float v);
extern "C" void ggml_vec_mad_f16_amd_avx512bf16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
+extern "C" void ggml_vec_mad_f16_amd_avx512vl(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
extern "C" void ggml_vec_mad_f16_amd_avx512(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
extern "C" void ggml_vec_mad_f16_amd_avx2(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
extern "C" void ggml_vec_mad_f16_amd_f16c(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
@@ -312,6 +340,7 @@ extern "C" void ggml_vec_mad_f16_arm82(const int n, ggml_fp16_t * y, const ggml_
extern "C" void ggml_vec_mad_f16_arm80(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float v);
extern "C" void ggml_vec_norm_f32_amd_avx512bf16 (const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_f32_amd_avx512vl (const int n, float * s, const float * x);
extern "C" void ggml_vec_norm_f32_amd_avx512 (const int n, float * s, const float * x);
extern "C" void ggml_vec_norm_f32_amd_avx2 (const int n, float * s, const float * x);
extern "C" void ggml_vec_norm_f32_amd_f16c (const int n, float * s, const float * x);
@@ -323,6 +352,7 @@ extern "C" void ggml_vec_norm_f32_arm82 (const int n, float * s, const float * x
extern "C" void ggml_vec_norm_f32_arm80 (const int n, float * s, const float * x);
extern "C" void ggml_vec_sqr_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqr_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_sqr_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_sqr_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_sqr_f32_amd_f16c (const int n, float * y, const float * x);
@@ -334,6 +364,7 @@ extern "C" void ggml_vec_sqr_f32_arm82 (const int n, float * y, const float * x
extern "C" void ggml_vec_sqr_f32_arm80 (const int n, float * y, const float * x);
extern "C" void ggml_vec_sqrt_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqrt_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_sqrt_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_sqrt_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_sqrt_f32_amd_f16c (const int n, float * y, const float * x);
@@ -345,6 +376,7 @@ extern "C" void ggml_vec_sqrt_f32_arm82 (const int n, float * y, const float * x
extern "C" void ggml_vec_sqrt_f32_arm80 (const int n, float * y, const float * x);
extern "C" void ggml_vec_log_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_log_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_log_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_log_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_log_f32_amd_f16c (const int n, float * y, const float * x);
@@ -356,6 +388,7 @@ extern "C" void ggml_vec_log_f32_arm82 (const int n, float * y, const float * x
extern "C" void ggml_vec_log_f32_arm80 (const int n, float * y, const float * x);
extern "C" void ggml_vec_abs_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_abs_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_abs_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_abs_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_abs_f32_amd_f16c (const int n, float * y, const float * x);
@@ -367,6 +400,7 @@ extern "C" void ggml_vec_abs_f32_arm82 (const int n, float * y, const float * x
extern "C" void ggml_vec_abs_f32_arm80 (const int n, float * y, const float * x);
extern "C" void ggml_vec_sgn_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sgn_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_sgn_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_sgn_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_sgn_f32_amd_f16c (const int n, float * y, const float * x);
@@ -378,6 +412,7 @@ extern "C" void ggml_vec_sgn_f32_arm82 (const int n, float * y, const float * x
extern "C" void ggml_vec_sgn_f32_arm80 (const int n, float * y, const float * x);
extern "C" void ggml_vec_step_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_step_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_step_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_step_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_step_f32_amd_f16c (const int n, float * y, const float * x);
@@ -389,6 +424,7 @@ extern "C" void ggml_vec_step_f32_arm82 (const int n, float * y, const float * x
extern "C" void ggml_vec_step_f32_arm80 (const int n, float * y, const float * x);
extern "C" void ggml_vec_tanh_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_tanh_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_tanh_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_tanh_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_tanh_f32_amd_f16c (const int n, float * y, const float * x);
@@ -400,6 +436,7 @@ extern "C" void ggml_vec_tanh_f32_arm82 (const int n, float * y, const float * x
extern "C" void ggml_vec_tanh_f32_arm80 (const int n, float * y, const float * x);
extern "C" void ggml_vec_elu_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_elu_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_elu_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_elu_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_elu_f32_amd_f16c (const int n, float * y, const float * x);
@@ -411,6 +448,7 @@ extern "C" void ggml_vec_elu_f32_arm82 (const int n, float * y, const float * x
extern "C" void ggml_vec_elu_f32_arm80 (const int n, float * y, const float * x);
extern "C" void ggml_vec_relu_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_relu_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_relu_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_relu_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_relu_f32_amd_f16c (const int n, float * y, const float * x);
@@ -422,6 +460,7 @@ extern "C" void ggml_vec_relu_f32_arm82 (const int n, float * y, const float * x
extern "C" void ggml_vec_relu_f32_arm80 (const int n, float * y, const float * x);
extern "C" void ggml_vec_leaky_relu_f32_amd_avx512bf16 (const int n, float * y, const float * x, const float ns);
+extern "C" void ggml_vec_leaky_relu_f32_amd_avx512vl (const int n, float * y, const float * x, const float ns);
extern "C" void ggml_vec_leaky_relu_f32_amd_avx512 (const int n, float * y, const float * x, const float ns);
extern "C" void ggml_vec_leaky_relu_f32_amd_avx2 (const int n, float * y, const float * x, const float ns);
extern "C" void ggml_vec_leaky_relu_f32_amd_f16c (const int n, float * y, const float * x, const float ns);
@@ -433,6 +472,7 @@ extern "C" void ggml_vec_leaky_relu_f32_arm82 (const int n, float * y, const flo
extern "C" void ggml_vec_leaky_relu_f32_arm80 (const int n, float * y, const float * x, const float ns);
extern "C" void ggml_vec_hardswish_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardswish_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_hardswish_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_hardswish_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_hardswish_f32_amd_f16c (const int n, float * y, const float * x);
@@ -444,6 +484,7 @@ extern "C" void ggml_vec_hardswish_f32_arm82 (const int n, float * y, const floa
extern "C" void ggml_vec_hardswish_f32_arm80 (const int n, float * y, const float * x);
extern "C" void ggml_vec_hardsigmoid_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardsigmoid_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_hardsigmoid_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_hardsigmoid_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_hardsigmoid_f32_amd_f16c (const int n, float * y, const float * x);
@@ -455,6 +496,7 @@ extern "C" void ggml_vec_hardsigmoid_f32_arm82 (const int n, float * y, const fl
extern "C" void ggml_vec_hardsigmoid_f32_arm80 (const int n, float * y, const float * x);
extern "C" void ggml_vec_gelu_f32_amd_avx512bf16(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_f32_amd_avx512vl(const int n, float * y, const float * x);
extern "C" void ggml_vec_gelu_f32_amd_avx512(const int n, float * y, const float * x);
extern "C" void ggml_vec_gelu_f32_amd_avx2(const int n, float * y, const float * x);
extern "C" void ggml_vec_gelu_f32_amd_f16c(const int n, float * y, const float * x);
@@ -466,6 +508,7 @@ extern "C" void ggml_vec_gelu_f32_arm82(const int n, float * y, const float * x)
extern "C" void ggml_vec_gelu_f32_arm80(const int n, float * y, const float * x);
extern "C" void ggml_vec_gelu_quick_f32_amd_avx512bf16(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_quick_f32_amd_avx512vl(const int n, float * y, const float * x);
extern "C" void ggml_vec_gelu_quick_f32_amd_avx512(const int n, float * y, const float * x);
extern "C" void ggml_vec_gelu_quick_f32_amd_avx2(const int n, float * y, const float * x);
extern "C" void ggml_vec_gelu_quick_f32_amd_f16c(const int n, float * y, const float * x);
@@ -477,6 +520,7 @@ extern "C" void ggml_vec_gelu_quick_f32_arm82(const int n, float * y, const floa
extern "C" void ggml_vec_gelu_quick_f32_arm80(const int n, float * y, const float * x);
extern "C" void ggml_vec_silu_f32_amd_avx512bf16(const int n, float * y, const float * x);
+extern "C" void ggml_vec_silu_f32_amd_avx512vl(const int n, float * y, const float * x);
extern "C" void ggml_vec_silu_f32_amd_avx512(const int n, float * y, const float * x);
extern "C" void ggml_vec_silu_f32_amd_avx2(const int n, float * y, const float * x);
extern "C" void ggml_vec_silu_f32_amd_f16c(const int n, float * y, const float * x);
@@ -488,6 +532,7 @@ extern "C" void ggml_vec_silu_f32_arm82(const int n, float * y, const float * x)
extern "C" void ggml_vec_silu_f32_arm80(const int n, float * y, const float * x);
extern "C" float ggml_silu_backward_f32_amd_avx512bf16(float x, float dy);
+extern "C" float ggml_silu_backward_f32_amd_avx512vl(float x, float dy);
extern "C" float ggml_silu_backward_f32_amd_avx512(float x, float dy);
extern "C" float ggml_silu_backward_f32_amd_avx2(float x, float dy);
extern "C" float ggml_silu_backward_f32_amd_f16c(float x, float dy);
@@ -499,6 +544,7 @@ extern "C" float ggml_silu_backward_f32_arm82(float x, float dy);
extern "C" float ggml_silu_backward_f32_arm80(float x, float dy);
extern "C" void ggml_vec_silu_backward_f32_amd_avx512bf16(const int n, float * dx, const float * x, const float * dy);
+extern "C" void ggml_vec_silu_backward_f32_amd_avx512vl(const int n, float * dx, const float * x, const float * dy);
extern "C" void ggml_vec_silu_backward_f32_amd_avx512(const int n, float * dx, const float * x, const float * dy);
extern "C" void ggml_vec_silu_backward_f32_amd_avx2(const int n, float * dx, const float * x, const float * dy);
extern "C" void ggml_vec_silu_backward_f32_amd_f16c(const int n, float * dx, const float * x, const float * dy);
@@ -510,6 +556,7 @@ extern "C" void ggml_vec_silu_backward_f32_arm82(const int n, float * dx, const
extern "C" void ggml_vec_silu_backward_f32_arm80(const int n, float * dx, const float * x, const float * dy);
extern "C" void ggml_vec_sum_f32_amd_avx512bf16(const int n, float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_amd_avx512vl(const int n, float * s, const float * x);
extern "C" void ggml_vec_sum_f32_amd_avx512(const int n, float * s, const float * x);
extern "C" void ggml_vec_sum_f32_amd_avx2(const int n, float * s, const float * x);
extern "C" void ggml_vec_sum_f32_amd_f16c(const int n, float * s, const float * x);
@@ -521,6 +568,7 @@ extern "C" void ggml_vec_sum_f32_arm82(const int n, float * s, const float * x);
extern "C" void ggml_vec_sum_f32_arm80(const int n, float * s, const float * x);
extern "C" void ggml_vec_sum_f32_ggf_amd_avx512bf16(const int n, ggml_float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_ggf_amd_avx512vl(const int n, ggml_float * s, const float * x);
extern "C" void ggml_vec_sum_f32_ggf_amd_avx512(const int n, ggml_float * s, const float * x);
extern "C" void ggml_vec_sum_f32_ggf_amd_avx2(const int n, ggml_float * s, const float * x);
extern "C" void ggml_vec_sum_f32_ggf_amd_f16c(const int n, ggml_float * s, const float * x);
@@ -532,6 +580,7 @@ extern "C" void ggml_vec_sum_f32_ggf_arm82(const int n, ggml_float * s, const fl
extern "C" void ggml_vec_sum_f32_ggf_arm80(const int n, ggml_float * s, const float * x);
extern "C" void ggml_vec_sum_f16_ggf_amd_avx512bf16(const int n, float * s, const ggml_fp16_t * x);
+extern "C" void ggml_vec_sum_f16_ggf_amd_avx512vl(const int n, float * s, const ggml_fp16_t * x);
extern "C" void ggml_vec_sum_f16_ggf_amd_avx512(const int n, float * s, const ggml_fp16_t * x);
extern "C" void ggml_vec_sum_f16_ggf_amd_avx2(const int n, float * s, const ggml_fp16_t * x);
extern "C" void ggml_vec_sum_f16_ggf_amd_f16c(const int n, float * s, const ggml_fp16_t * x);
@@ -543,6 +592,7 @@ extern "C" void ggml_vec_sum_f16_ggf_arm82(const int n, float * s, const ggml_fp
extern "C" void ggml_vec_sum_f16_ggf_arm80(const int n, float * s, const ggml_fp16_t * x);
extern "C" void ggml_vec_sum_bf16_ggf_amd_avx512bf16(const int n, float * s, const ggml_bf16_t * x);
+extern "C" void ggml_vec_sum_bf16_ggf_amd_avx512vl(const int n, float * s, const ggml_bf16_t * x);
extern "C" void ggml_vec_sum_bf16_ggf_amd_avx512(const int n, float * s, const ggml_bf16_t * x);
extern "C" void ggml_vec_sum_bf16_ggf_amd_avx2(const int n, float * s, const ggml_bf16_t * x);
extern "C" void ggml_vec_sum_bf16_ggf_amd_f16c(const int n, float * s, const ggml_bf16_t * x);
@@ -554,6 +604,7 @@ extern "C" void ggml_vec_sum_bf16_ggf_arm82(const int n, float * s, const ggml_b
extern "C" void ggml_vec_sum_bf16_ggf_arm80(const int n, float * s, const ggml_bf16_t * x);
extern "C" void ggml_vec_max_f32_amd_avx512bf16(const int n, float * s, const float * x);
+extern "C" void ggml_vec_max_f32_amd_avx512vl(const int n, float * s, const float * x);
extern "C" void ggml_vec_max_f32_amd_avx512(const int n, float * s, const float * x);
extern "C" void ggml_vec_max_f32_amd_avx2(const int n, float * s, const float * x);
extern "C" void ggml_vec_max_f32_amd_f16c(const int n, float * s, const float * x);
@@ -565,6 +616,7 @@ extern "C" void ggml_vec_max_f32_arm82(const int n, float * s, const float * x);
extern "C" void ggml_vec_max_f32_arm80(const int n, float * s, const float * x);
extern "C" void ggml_vec_argmax_f32_amd_avx512bf16(const int n, int * s, const float * x);
+extern "C" void ggml_vec_argmax_f32_amd_avx512vl(const int n, int * s, const float * x);
extern "C" void ggml_vec_argmax_f32_amd_avx512(const int n, int * s, const float * x);
extern "C" void ggml_vec_argmax_f32_amd_avx2(const int n, int * s, const float * x);
extern "C" void ggml_vec_argmax_f32_amd_f16c(const int n, int * s, const float * x);
@@ -576,6 +628,7 @@ extern "C" void ggml_vec_argmax_f32_arm82(const int n, int * s, const float * x)
extern "C" void ggml_vec_argmax_f32_arm80(const int n, int * s, const float * x);
extern "C" ggml_float ggml_vec_soft_max_f32_amd_avx512bf16(const int n, float * y, const float * x, float max);
+extern "C" ggml_float ggml_vec_soft_max_f32_amd_avx512vl(const int n, float * y, const float * x, float max);
extern "C" ggml_float ggml_vec_soft_max_f32_amd_avx512(const int n, float * y, const float * x, float max);
extern "C" ggml_float ggml_vec_soft_max_f32_amd_avx2(const int n, float * y, const float * x, float max);
extern "C" ggml_float ggml_vec_soft_max_f32_amd_f16c(const int n, float * y, const float * x, float max);
@@ -587,6 +640,7 @@ extern "C" ggml_float ggml_vec_soft_max_f32_arm82(const int n, float * y, const
extern "C" ggml_float ggml_vec_soft_max_f32_arm80(const int n, float * y, const float * x, float max);
extern "C" void ggml_vec_norm_inv_f32_amd_avx512bf16(const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_inv_f32_amd_avx512vl(const int n, float * s, const float * x);
extern "C" void ggml_vec_norm_inv_f32_amd_avx512(const int n, float * s, const float * x);
extern "C" void ggml_vec_norm_inv_f32_amd_avx2(const int n, float * s, const float * x);
extern "C" void ggml_vec_norm_inv_f32_amd_f16c(const int n, float * s, const float * x);
@@ -598,6 +652,7 @@ extern "C" void ggml_vec_norm_inv_f32_arm82(const int n, float * s, const float
extern "C" void ggml_vec_norm_inv_f32_arm80(const int n, float * s, const float * x);
extern "C" void ggml_vec_sigmoid_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sigmoid_f32_amd_avx512vl (const int n, float * y, const float * x);
extern "C" void ggml_vec_sigmoid_f32_amd_avx512 (const int n, float * y, const float * x);
extern "C" void ggml_vec_sigmoid_f32_amd_avx2 (const int n, float * y, const float * x);
extern "C" void ggml_vec_sigmoid_f32_amd_f16c (const int n, float * y, const float * x);
@@ -667,7 +722,7 @@ static const struct VectorFuncs {
VectorFuncs() {
#ifdef __x86_64__
- if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2) && X86_HAVE(AVX512F) && X86_HAVE(AVX512VL) && X86_HAVE(AVX512_BF16)) {
+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2) && X86_HAVE(AVX512F) && X86_HAVE(AVX512BW) && X86_HAVE(AVX512DQ) && X86_HAVE(AVX512VL) && X86_HAVE(AVX512_BF16)) {
ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_avx512bf16;
ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx512bf16;
ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx512bf16;
@@ -726,6 +781,66 @@ static const struct VectorFuncs {
return;
}
#endif
+#ifdef __x86_64__
+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2) && X86_HAVE(AVX512F) && X86_HAVE(AVX512BW) && X86_HAVE(AVX512DQ) && X86_HAVE(AVX512VL)) {
+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_avx512vl;
+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx512vl;
+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx512vl;
+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx512vl;
+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx512vl;
+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx512vl;
+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx512vl;
+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_amd_avx512vl;
+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_amd_avx512vl;
+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_amd_avx512vl;
+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_amd_avx512vl;
+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_amd_avx512vl;
+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_amd_avx512vl;
+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_amd_avx512vl;
+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_amd_avx512vl;
+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_amd_avx512vl;
+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_amd_avx512vl;
+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_amd_avx512vl;
+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_amd_avx512vl;
+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_amd_avx512vl;
+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_amd_avx512vl;
+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_amd_avx512vl;
+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_amd_avx512vl;
+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_amd_avx512vl;
+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_amd_avx512vl;
+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_amd_avx512vl;
+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_amd_avx512vl;
+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_amd_avx512vl;
+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_amd_avx512vl;
+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_amd_avx512vl;
+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_amd_avx512vl;
+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_amd_avx512vl;
+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_amd_avx512vl;
+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_amd_avx512vl;
+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_amd_avx512vl;
+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_amd_avx512vl;
+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_amd_avx512vl;
+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_amd_avx512vl;
+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_amd_avx512vl;
+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_amd_avx512vl;
+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_amd_avx512vl;
+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_amd_avx512vl;
+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_amd_avx512vl;
+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_amd_avx512vl;
+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_amd_avx512vl;
+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_amd_avx512vl;
+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_avx512vl;
+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_avx512vl;
+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_avx512vl;
+ ptr_ggml_vec_sum_bf16_ggf = ggml_vec_sum_bf16_ggf_amd_avx512vl;
+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx512vl;
+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx512vl;
+ ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_avx512vl;
+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx512vl;
+ ptr_ggml_vec_sigmoid_f32 = ggml_vec_sigmoid_f32_amd_avx512vl;
+ return;
+ }
+#endif
#ifdef __x86_64__
if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2) && X86_HAVE(AVX512F)) {
ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_avx512;
diff --git a/llama.cpp/ggml-vector.py b/llama.cpp/ggml-vector.py
index dc77e815..fd334efd 100644
--- a/llama.cpp/ggml-vector.py
+++ b/llama.cpp/ggml-vector.py
@@ -12,7 +12,8 @@ FUNCS = [(re.search(r'(?<= )\w+', proto).group(0), proto)
# END SPECIAL FUNCTIONS
ARCHS = (
- ('amd_avx512bf16', '__x86_64__', ('X86_HAVE(FMA)', 'X86_HAVE(F16C)', 'X86_HAVE(AVX2)', 'X86_HAVE(AVX512F)', 'X86_HAVE(AVX512VL)', 'X86_HAVE(AVX512_BF16)')),
+ ('amd_avx512bf16', '__x86_64__', ('X86_HAVE(FMA)', 'X86_HAVE(F16C)', 'X86_HAVE(AVX2)', 'X86_HAVE(AVX512F)', 'X86_HAVE(AVX512BW)', 'X86_HAVE(AVX512DQ)', 'X86_HAVE(AVX512VL)', 'X86_HAVE(AVX512_BF16)')),
+ ('amd_avx512vl', '__x86_64__', ('X86_HAVE(FMA)', 'X86_HAVE(F16C)', 'X86_HAVE(AVX2)', 'X86_HAVE(AVX512F)', 'X86_HAVE(AVX512BW)', 'X86_HAVE(AVX512DQ)', 'X86_HAVE(AVX512VL)')),
('amd_avx512', '__x86_64__', ('X86_HAVE(FMA)', 'X86_HAVE(F16C)', 'X86_HAVE(AVX2)', 'X86_HAVE(AVX512F)')),
('amd_avx2', '__x86_64__', ('X86_HAVE(FMA)', 'X86_HAVE(F16C)', 'X86_HAVE(AVX2)')),
('amd_f16c', '__x86_64__', ('X86_HAVE(AVX)', 'X86_HAVE(F16C)')),