Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Make GGML vector ops go faster across hardware
Justine Tunneyc9d7393f82b9
diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mkindex f09c18f5..22443b02 100644--- a/llama.cpp/BUILD.mk+++ b/llama.cpp/BUILD.mk@@ -48,6 +48,15 @@ o/$(MODE)/llama.cpp/ggml-quants-amd-avx.o: private TARGET_ARCH += -Xx86_64-mtuneo/$(MODE)/llama.cpp/ggml-quants-amd-avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2o/$(MODE)/llama.cpp/ggml-quants-amd-avx512.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f+o/$(MODE)/llama.cpp/ggml-vector.o: private CXXFLAGS += -Os+o/$(MODE)/llama.cpp/ggml-vector-amd-avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge+o/$(MODE)/llama.cpp/ggml-vector-amd-fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mfma+o/$(MODE)/llama.cpp/ggml-vector-amd-f16c.o: private TARGET_ARCH += -Xx86_64-mtune=ivybridge -Xx86_64-mf16c+o/$(MODE)/llama.cpp/ggml-vector-amd-avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2+o/$(MODE)/llama.cpp/ggml-vector-amd-avx512.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f+o/$(MODE)/llama.cpp/ggml-vector-amd-avx512bf16.o: private TARGET_ARCH += -Xx86_64-mtune=znver4 -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512vl -Xx86_64-mavx512bf16+o/$(MODE)/llama.cpp/ggml-vector-arm82.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+fp16+$(LLAMA_CPP_OBJS): llama.cpp/BUILD.mk.PHONY: o/$(MODE)/llama.cppdiff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.hindex 90870cb4..72df530a 100644--- a/llama.cpp/ggml-impl.h+++ b/llama.cpp/ggml-impl.h@@ -19,6 +19,83 @@#define MIN(a, b) ((a) < (b) ? (a) : (b))#define MAX(a, b) ((a) > (b) ? (a) : (b))+/**+ * Converts brain16 to float32.+ *+ * The bfloat16 floating point format has the following structure:+ *+ * ┌sign+ * │+ * │ ┌exponent+ * │ │+ * │ │ ┌mantissa+ * │ │ │+ * │┌──┴───┐┌─┴───┐+ * 0b0000000000000000 brain16+ *+ * Since bf16 has the same number of exponent bits as a 32bit float,+ * encoding and decoding numbers becomes relatively straightforward.+ *+ * ┌sign+ * │+ * │ ┌exponent+ * │ │+ * │ │ ┌mantissa+ * │ │ │+ * │┌──┴───┐┌─┴───────────────────┐+ * 0b00000000000000000000000000000000 IEEE binary32+ *+ * For comparison, the standard fp16 format has fewer exponent bits.+ *+ * ┌sign+ * │+ * │ ┌exponent+ * │ │+ * │ │ ┌mantissa+ * │ │ │+ * │┌─┴─┐┌─┴──────┐+ * 0b0000000000000000 IEEE binary16+ *+ * @see IEEE 754-2008+ */+static inline float ggml_compute_bf16_to_fp32(ggml_bf16_t h) {+ union {+ float f;+ uint32_t i;+ } u;+ u.i = (uint32_t)h.bits << 16;+ return u.f;+}++/**+ * Converts float32 to brain16.+ *+ * This function is binary identical to AMD Zen4 VCVTNEPS2BF16.+ * Subnormals shall be flushed to zero, and NANs will be quiet.+ * This code should vectorize nicely if using modern compilers.+ */+static inline ggml_bf16_t ggml_compute_fp32_to_bf16(float s) {+ ggml_bf16_t h;+ union {+ float f;+ uint32_t i;+ } u;+ u.f = s;+ if ((u.i & 0x7fffffff) > 0x7f800000) { /* nan */+ h.bits = (u.i >> 16) | 64; /* force to quiet */+ return h;+ }+ if (!(u.i & 0x7f800000)) { /* subnormal */+ h.bits = (u.i & 0x80000000) >> 16; /* flush to zero */+ return h;+ }+ h.bits = (u.i + (0x7fff + ((u.i >> 16) & 1))) >> 16;+ return h;+}++#define GGML_FP32_TO_BF16(x) ggml_compute_fp32_to_bf16(x)+#define GGML_BF16_TO_FP32(x) ggml_compute_bf16_to_fp32(x)+#ifdef __cplusplusextern "C" {#endif@@ -520,9 +597,6 @@ size_t ggml_hash_insert ( struct ggml_hash_set hash_set, struct ggml// return index, asserts if table is fullsize_t ggml_hash_find_or_insert( struct ggml_hash_set hash_set, struct ggml_tensor * key);-#define GGML_FP32_TO_BF16(x) ggml_fp32_to_bf16(x)-#define GGML_BF16_TO_FP32(x) ggml_bf16_to_fp32(x)-#ifdef __cplusplus}#endifdiff --git a/llama.cpp/ggml-quants.py b/llama.cpp/ggml-quants.pyindex c3d0a469..9a9d0fd6 100644--- a/llama.cpp/ggml-quants.py+++ b/llama.cpp/ggml-quants.py@@ -42,36 +42,21 @@ with open('llama.cpp/ggml-quants.cpp', 'w') as f:f.write(' typeof(%s) *ptr_%s;\n' % (func, func))f.write('\n')f.write(' QuantFuncs() {\n')- f.write('#ifdef __x86_64__\n')for arch, mac, needs in ARCHS:- if mac == '__x86_64__':- f.write(' if (%s) {\n' % (' && '.join('X86_HAVE(%s)' % (need) for need in needs) or '1'))- for func, proto in FUNCS:- f.write(' ptr_%s = %s_%s;\n' % (func, func, arch))- f.write(' return;\n')- f.write(' }\n')- f.write('#else\n')- for func, proto in FUNCS:- f.write(' ptr_%s = %s_arm80;\n' % (func, func))- f.write('#endif\n')+ f.write('#ifdef %s\n' % (mac))+ f.write(' if (%s) {\n' % (' && '.join('X86_HAVE(%s)' % (need) for need in needs) or '1'))+ for func, proto in FUNCS:+ f.write(' ptr_%s = %s_%s;\n' % (func, func, arch))+ f.write(' return;\n')+ f.write(' }\n')+ f.write('#endif\n')f.write(' }\n')f.write('} funcs;\n')f.write('\n')for func, proto in FUNCS:proto = proto.replace(';', '')+ args = [s.split(' ')[-1] for s in re.search(r'(?<=\().*(?=\))', proto).group(0).split(',')]f.write(proto + ' {\n')- if 'imatrix' in proto:- args = 'src, dst, nrows, n_per_row, imatrix'- elif 'quantize' in proto:- args = 'x, y, k'- elif 'vec_dot' in proto:- args = 'n, s, bs, vx, bx, vy, by, nrc'- elif 'grid_size' in proto:- args = 'grid_size'- elif 'validate' in proto:- args = 'type, data, nbytes'- else:- args = 'type'f.write(' return funcs.ptr_%s(%s);\n' % (func, args))f.write('}\n')f.write('\n')diff --git a/llama.cpp/ggml-vector-amd-avx.c b/llama.cpp/ggml-vector-amd-avx.cnew file mode 100644index 00000000..a47d29b5--- /dev/null+++ b/llama.cpp/ggml-vector-amd-avx.c@@ -0,0 +1,57 @@+#ifdef __x86_64__+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_amd_avx+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx+#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx+#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_amd_avx+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_amd_avx+#define ggml_vec_set_i8 ggml_vec_set_i8_amd_avx+#define ggml_vec_set_i16 ggml_vec_set_i16_amd_avx+#define ggml_vec_set_i32 ggml_vec_set_i32_amd_avx+#define ggml_vec_set_f16 ggml_vec_set_f16_amd_avx+#define ggml_vec_set_bf16 ggml_vec_set_bf16_amd_avx+#define ggml_vec_add_f32 ggml_vec_add_f32_amd_avx+#define ggml_vec_add1_f32 ggml_vec_add1_f32_amd_avx+#define ggml_vec_acc_f32 ggml_vec_acc_f32_amd_avx+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_amd_avx+#define ggml_vec_sub_f32 ggml_vec_sub_f32_amd_avx+#define ggml_vec_set_f32 ggml_vec_set_f32_amd_avx+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_amd_avx+#define ggml_vec_neg_f32 ggml_vec_neg_f32_amd_avx+#define ggml_vec_mul_f32 ggml_vec_mul_f32_amd_avx+#define ggml_vec_div_f32 ggml_vec_div_f32_amd_avx+#define ggml_vec_scale_f32 ggml_vec_scale_f32_amd_avx+#define ggml_vec_scale_f16 ggml_vec_scale_f16_amd_avx+#define ggml_vec_mad_f32 ggml_vec_mad_f32_amd_avx+#define ggml_vec_mad_f16 ggml_vec_mad_f16_amd_avx+#define ggml_vec_norm_f32 ggml_vec_norm_f32_amd_avx+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_amd_avx+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_amd_avx+#define ggml_vec_log_f32 ggml_vec_log_f32_amd_avx+#define ggml_vec_abs_f32 ggml_vec_abs_f32_amd_avx+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_amd_avx+#define ggml_vec_step_f32 ggml_vec_step_f32_amd_avx+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_amd_avx+#define ggml_vec_elu_f32 ggml_vec_elu_f32_amd_avx+#define ggml_vec_relu_f32 ggml_vec_relu_f32_amd_avx+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_amd_avx+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_amd_avx+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_amd_avx+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_amd_avx+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_amd_avx+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_amd_avx+#define ggml_vec_silu_f32 ggml_vec_silu_f32_amd_avx+#define ggml_silu_backward_f32 ggml_silu_backward_f32_amd_avx+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_amd_avx+#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_avx+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_avx+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_avx+#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx+#define GGML_VECTOR+#include "ggml-vector.inc"+#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-avx2.c b/llama.cpp/ggml-vector-amd-avx2.cnew file mode 100644index 00000000..2daeb599--- /dev/null+++ b/llama.cpp/ggml-vector-amd-avx2.c@@ -0,0 +1,57 @@+#ifdef __x86_64__+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_amd_avx2+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx2+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx2+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx2+#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx2+#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx2+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx2+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_amd_avx2+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_amd_avx2+#define ggml_vec_set_i8 ggml_vec_set_i8_amd_avx2+#define ggml_vec_set_i16 ggml_vec_set_i16_amd_avx2+#define ggml_vec_set_i32 ggml_vec_set_i32_amd_avx2+#define ggml_vec_set_f16 ggml_vec_set_f16_amd_avx2+#define ggml_vec_set_bf16 ggml_vec_set_bf16_amd_avx2+#define ggml_vec_add_f32 ggml_vec_add_f32_amd_avx2+#define ggml_vec_add1_f32 ggml_vec_add1_f32_amd_avx2+#define ggml_vec_acc_f32 ggml_vec_acc_f32_amd_avx2+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_amd_avx2+#define ggml_vec_sub_f32 ggml_vec_sub_f32_amd_avx2+#define ggml_vec_set_f32 ggml_vec_set_f32_amd_avx2+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_amd_avx2+#define ggml_vec_neg_f32 ggml_vec_neg_f32_amd_avx2+#define ggml_vec_mul_f32 ggml_vec_mul_f32_amd_avx2+#define ggml_vec_div_f32 ggml_vec_div_f32_amd_avx2+#define ggml_vec_scale_f32 ggml_vec_scale_f32_amd_avx2+#define ggml_vec_scale_f16 ggml_vec_scale_f16_amd_avx2+#define ggml_vec_mad_f32 ggml_vec_mad_f32_amd_avx2+#define ggml_vec_mad_f16 ggml_vec_mad_f16_amd_avx2+#define ggml_vec_norm_f32 ggml_vec_norm_f32_amd_avx2+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_amd_avx2+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_amd_avx2+#define ggml_vec_log_f32 ggml_vec_log_f32_amd_avx2+#define ggml_vec_abs_f32 ggml_vec_abs_f32_amd_avx2+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_amd_avx2+#define ggml_vec_step_f32 ggml_vec_step_f32_amd_avx2+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_amd_avx2+#define ggml_vec_elu_f32 ggml_vec_elu_f32_amd_avx2+#define ggml_vec_relu_f32 ggml_vec_relu_f32_amd_avx2+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_amd_avx2+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_amd_avx2+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_amd_avx2+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_amd_avx2+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_amd_avx2+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_amd_avx2+#define ggml_vec_silu_f32 ggml_vec_silu_f32_amd_avx2+#define ggml_silu_backward_f32 ggml_silu_backward_f32_amd_avx2+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_amd_avx2+#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_avx2+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_avx2+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_avx2+#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx2+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx2+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx2+#define GGML_VECTOR+#include "ggml-vector.inc"+#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-avx512.c b/llama.cpp/ggml-vector-amd-avx512.cnew file mode 100644index 00000000..325e6bb3--- /dev/null+++ b/llama.cpp/ggml-vector-amd-avx512.c@@ -0,0 +1,57 @@+#ifdef __x86_64__+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_amd_avx512+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx512+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx512+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx512+#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx512+#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx512+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx512+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_amd_avx512+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_amd_avx512+#define ggml_vec_set_i8 ggml_vec_set_i8_amd_avx512+#define ggml_vec_set_i16 ggml_vec_set_i16_amd_avx512+#define ggml_vec_set_i32 ggml_vec_set_i32_amd_avx512+#define ggml_vec_set_f16 ggml_vec_set_f16_amd_avx512+#define ggml_vec_set_bf16 ggml_vec_set_bf16_amd_avx512+#define ggml_vec_add_f32 ggml_vec_add_f32_amd_avx512+#define ggml_vec_add1_f32 ggml_vec_add1_f32_amd_avx512+#define ggml_vec_acc_f32 ggml_vec_acc_f32_amd_avx512+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_amd_avx512+#define ggml_vec_sub_f32 ggml_vec_sub_f32_amd_avx512+#define ggml_vec_set_f32 ggml_vec_set_f32_amd_avx512+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_amd_avx512+#define ggml_vec_neg_f32 ggml_vec_neg_f32_amd_avx512+#define ggml_vec_mul_f32 ggml_vec_mul_f32_amd_avx512+#define ggml_vec_div_f32 ggml_vec_div_f32_amd_avx512+#define ggml_vec_scale_f32 ggml_vec_scale_f32_amd_avx512+#define ggml_vec_scale_f16 ggml_vec_scale_f16_amd_avx512+#define ggml_vec_mad_f32 ggml_vec_mad_f32_amd_avx512+#define ggml_vec_mad_f16 ggml_vec_mad_f16_amd_avx512+#define ggml_vec_norm_f32 ggml_vec_norm_f32_amd_avx512+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_amd_avx512+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_amd_avx512+#define ggml_vec_log_f32 ggml_vec_log_f32_amd_avx512+#define ggml_vec_abs_f32 ggml_vec_abs_f32_amd_avx512+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_amd_avx512+#define ggml_vec_step_f32 ggml_vec_step_f32_amd_avx512+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_amd_avx512+#define ggml_vec_elu_f32 ggml_vec_elu_f32_amd_avx512+#define ggml_vec_relu_f32 ggml_vec_relu_f32_amd_avx512+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_amd_avx512+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_amd_avx512+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_amd_avx512+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_amd_avx512+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_amd_avx512+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_amd_avx512+#define ggml_vec_silu_f32 ggml_vec_silu_f32_amd_avx512+#define ggml_silu_backward_f32 ggml_silu_backward_f32_amd_avx512+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_amd_avx512+#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_avx512+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_avx512+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_avx512+#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx512+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx512+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx512+#define GGML_VECTOR+#include "ggml-vector.inc"+#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-avx512bf16.c b/llama.cpp/ggml-vector-amd-avx512bf16.cnew file mode 100644index 00000000..5f7a3a65--- /dev/null+++ b/llama.cpp/ggml-vector-amd-avx512bf16.c@@ -0,0 +1,57 @@+#ifdef __x86_64__+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_amd_avx512bf16+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx512bf16+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx512bf16+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx512bf16+#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx512bf16+#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx512bf16+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx512bf16+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_amd_avx512bf16+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_amd_avx512bf16+#define ggml_vec_set_i8 ggml_vec_set_i8_amd_avx512bf16+#define ggml_vec_set_i16 ggml_vec_set_i16_amd_avx512bf16+#define ggml_vec_set_i32 ggml_vec_set_i32_amd_avx512bf16+#define ggml_vec_set_f16 ggml_vec_set_f16_amd_avx512bf16+#define ggml_vec_set_bf16 ggml_vec_set_bf16_amd_avx512bf16+#define ggml_vec_add_f32 ggml_vec_add_f32_amd_avx512bf16+#define ggml_vec_add1_f32 ggml_vec_add1_f32_amd_avx512bf16+#define ggml_vec_acc_f32 ggml_vec_acc_f32_amd_avx512bf16+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_amd_avx512bf16+#define ggml_vec_sub_f32 ggml_vec_sub_f32_amd_avx512bf16+#define ggml_vec_set_f32 ggml_vec_set_f32_amd_avx512bf16+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_amd_avx512bf16+#define ggml_vec_neg_f32 ggml_vec_neg_f32_amd_avx512bf16+#define ggml_vec_mul_f32 ggml_vec_mul_f32_amd_avx512bf16+#define ggml_vec_div_f32 ggml_vec_div_f32_amd_avx512bf16+#define ggml_vec_scale_f32 ggml_vec_scale_f32_amd_avx512bf16+#define ggml_vec_scale_f16 ggml_vec_scale_f16_amd_avx512bf16+#define ggml_vec_mad_f32 ggml_vec_mad_f32_amd_avx512bf16+#define ggml_vec_mad_f16 ggml_vec_mad_f16_amd_avx512bf16+#define ggml_vec_norm_f32 ggml_vec_norm_f32_amd_avx512bf16+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_amd_avx512bf16+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_amd_avx512bf16+#define ggml_vec_log_f32 ggml_vec_log_f32_amd_avx512bf16+#define ggml_vec_abs_f32 ggml_vec_abs_f32_amd_avx512bf16+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_amd_avx512bf16+#define ggml_vec_step_f32 ggml_vec_step_f32_amd_avx512bf16+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_amd_avx512bf16+#define ggml_vec_elu_f32 ggml_vec_elu_f32_amd_avx512bf16+#define ggml_vec_relu_f32 ggml_vec_relu_f32_amd_avx512bf16+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_amd_avx512bf16+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_amd_avx512bf16+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_amd_avx512bf16+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_amd_avx512bf16+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_amd_avx512bf16+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_amd_avx512bf16+#define ggml_vec_silu_f32 ggml_vec_silu_f32_amd_avx512bf16+#define ggml_silu_backward_f32 ggml_silu_backward_f32_amd_avx512bf16+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_amd_avx512bf16+#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_avx512bf16+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_avx512bf16+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_avx512bf16+#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx512bf16+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx512bf16+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx512bf16+#define GGML_VECTOR+#include "ggml-vector.inc"+#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-f16c.c b/llama.cpp/ggml-vector-amd-f16c.cnew file mode 100644index 00000000..5a0f2c9f--- /dev/null+++ b/llama.cpp/ggml-vector-amd-f16c.c@@ -0,0 +1,57 @@+#ifdef __x86_64__+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_amd_f16c+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_f16c+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_f16c+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_f16c+#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_f16c+#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_f16c+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_f16c+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_amd_f16c+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_amd_f16c+#define ggml_vec_set_i8 ggml_vec_set_i8_amd_f16c+#define ggml_vec_set_i16 ggml_vec_set_i16_amd_f16c+#define ggml_vec_set_i32 ggml_vec_set_i32_amd_f16c+#define ggml_vec_set_f16 ggml_vec_set_f16_amd_f16c+#define ggml_vec_set_bf16 ggml_vec_set_bf16_amd_f16c+#define ggml_vec_add_f32 ggml_vec_add_f32_amd_f16c+#define ggml_vec_add1_f32 ggml_vec_add1_f32_amd_f16c+#define ggml_vec_acc_f32 ggml_vec_acc_f32_amd_f16c+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_amd_f16c+#define ggml_vec_sub_f32 ggml_vec_sub_f32_amd_f16c+#define ggml_vec_set_f32 ggml_vec_set_f32_amd_f16c+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_amd_f16c+#define ggml_vec_neg_f32 ggml_vec_neg_f32_amd_f16c+#define ggml_vec_mul_f32 ggml_vec_mul_f32_amd_f16c+#define ggml_vec_div_f32 ggml_vec_div_f32_amd_f16c+#define ggml_vec_scale_f32 ggml_vec_scale_f32_amd_f16c+#define ggml_vec_scale_f16 ggml_vec_scale_f16_amd_f16c+#define ggml_vec_mad_f32 ggml_vec_mad_f32_amd_f16c+#define ggml_vec_mad_f16 ggml_vec_mad_f16_amd_f16c+#define ggml_vec_norm_f32 ggml_vec_norm_f32_amd_f16c+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_amd_f16c+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_amd_f16c+#define ggml_vec_log_f32 ggml_vec_log_f32_amd_f16c+#define ggml_vec_abs_f32 ggml_vec_abs_f32_amd_f16c+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_amd_f16c+#define ggml_vec_step_f32 ggml_vec_step_f32_amd_f16c+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_amd_f16c+#define ggml_vec_elu_f32 ggml_vec_elu_f32_amd_f16c+#define ggml_vec_relu_f32 ggml_vec_relu_f32_amd_f16c+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_amd_f16c+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_amd_f16c+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_amd_f16c+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_amd_f16c+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_amd_f16c+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_amd_f16c+#define ggml_vec_silu_f32 ggml_vec_silu_f32_amd_f16c+#define ggml_silu_backward_f32 ggml_silu_backward_f32_amd_f16c+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_amd_f16c+#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_f16c+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_f16c+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_f16c+#define ggml_vec_max_f32 ggml_vec_max_f32_amd_f16c+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_f16c+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_f16c+#define GGML_VECTOR+#include "ggml-vector.inc"+#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-fma.c b/llama.cpp/ggml-vector-amd-fma.cnew file mode 100644index 00000000..77e72ada--- /dev/null+++ b/llama.cpp/ggml-vector-amd-fma.c@@ -0,0 +1,57 @@+#ifdef __x86_64__+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_amd_fma+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_fma+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_fma+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_fma+#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_fma+#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_fma+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_fma+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_amd_fma+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_amd_fma+#define ggml_vec_set_i8 ggml_vec_set_i8_amd_fma+#define ggml_vec_set_i16 ggml_vec_set_i16_amd_fma+#define ggml_vec_set_i32 ggml_vec_set_i32_amd_fma+#define ggml_vec_set_f16 ggml_vec_set_f16_amd_fma+#define ggml_vec_set_bf16 ggml_vec_set_bf16_amd_fma+#define ggml_vec_add_f32 ggml_vec_add_f32_amd_fma+#define ggml_vec_add1_f32 ggml_vec_add1_f32_amd_fma+#define ggml_vec_acc_f32 ggml_vec_acc_f32_amd_fma+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_amd_fma+#define ggml_vec_sub_f32 ggml_vec_sub_f32_amd_fma+#define ggml_vec_set_f32 ggml_vec_set_f32_amd_fma+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_amd_fma+#define ggml_vec_neg_f32 ggml_vec_neg_f32_amd_fma+#define ggml_vec_mul_f32 ggml_vec_mul_f32_amd_fma+#define ggml_vec_div_f32 ggml_vec_div_f32_amd_fma+#define ggml_vec_scale_f32 ggml_vec_scale_f32_amd_fma+#define ggml_vec_scale_f16 ggml_vec_scale_f16_amd_fma+#define ggml_vec_mad_f32 ggml_vec_mad_f32_amd_fma+#define ggml_vec_mad_f16 ggml_vec_mad_f16_amd_fma+#define ggml_vec_norm_f32 ggml_vec_norm_f32_amd_fma+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_amd_fma+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_amd_fma+#define ggml_vec_log_f32 ggml_vec_log_f32_amd_fma+#define ggml_vec_abs_f32 ggml_vec_abs_f32_amd_fma+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_amd_fma+#define ggml_vec_step_f32 ggml_vec_step_f32_amd_fma+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_amd_fma+#define ggml_vec_elu_f32 ggml_vec_elu_f32_amd_fma+#define ggml_vec_relu_f32 ggml_vec_relu_f32_amd_fma+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_amd_fma+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_amd_fma+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_amd_fma+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_amd_fma+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_amd_fma+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_amd_fma+#define ggml_vec_silu_f32 ggml_vec_silu_f32_amd_fma+#define ggml_silu_backward_f32 ggml_silu_backward_f32_amd_fma+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_amd_fma+#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_fma+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_fma+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_fma+#define ggml_vec_max_f32 ggml_vec_max_f32_amd_fma+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_fma+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_fma+#define GGML_VECTOR+#include "ggml-vector.inc"+#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-arm80.c b/llama.cpp/ggml-vector-arm80.cnew file mode 100644index 00000000..64d51d24--- /dev/null+++ b/llama.cpp/ggml-vector-arm80.c@@ -0,0 +1,57 @@+#ifdef __aarch64__+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_arm80+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_arm80+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_arm80+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_arm80+#define ggml_vec_dot_f32 ggml_vec_dot_f32_arm80+#define ggml_vec_dot_f16 ggml_vec_dot_f16_arm80+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_arm80+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_arm80+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_arm80+#define ggml_vec_set_i8 ggml_vec_set_i8_arm80+#define ggml_vec_set_i16 ggml_vec_set_i16_arm80+#define ggml_vec_set_i32 ggml_vec_set_i32_arm80+#define ggml_vec_set_f16 ggml_vec_set_f16_arm80+#define ggml_vec_set_bf16 ggml_vec_set_bf16_arm80+#define ggml_vec_add_f32 ggml_vec_add_f32_arm80+#define ggml_vec_add1_f32 ggml_vec_add1_f32_arm80+#define ggml_vec_acc_f32 ggml_vec_acc_f32_arm80+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_arm80+#define ggml_vec_sub_f32 ggml_vec_sub_f32_arm80+#define ggml_vec_set_f32 ggml_vec_set_f32_arm80+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_arm80+#define ggml_vec_neg_f32 ggml_vec_neg_f32_arm80+#define ggml_vec_mul_f32 ggml_vec_mul_f32_arm80+#define ggml_vec_div_f32 ggml_vec_div_f32_arm80+#define ggml_vec_scale_f32 ggml_vec_scale_f32_arm80+#define ggml_vec_scale_f16 ggml_vec_scale_f16_arm80+#define ggml_vec_mad_f32 ggml_vec_mad_f32_arm80+#define ggml_vec_mad_f16 ggml_vec_mad_f16_arm80+#define ggml_vec_norm_f32 ggml_vec_norm_f32_arm80+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_arm80+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_arm80+#define ggml_vec_log_f32 ggml_vec_log_f32_arm80+#define ggml_vec_abs_f32 ggml_vec_abs_f32_arm80+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_arm80+#define ggml_vec_step_f32 ggml_vec_step_f32_arm80+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_arm80+#define ggml_vec_elu_f32 ggml_vec_elu_f32_arm80+#define ggml_vec_relu_f32 ggml_vec_relu_f32_arm80+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_arm80+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_arm80+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_arm80+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_arm80+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_arm80+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_arm80+#define ggml_vec_silu_f32 ggml_vec_silu_f32_arm80+#define ggml_silu_backward_f32 ggml_silu_backward_f32_arm80+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_arm80+#define ggml_vec_sum_f32 ggml_vec_sum_f32_arm80+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_arm80+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_arm80+#define ggml_vec_max_f32 ggml_vec_max_f32_arm80+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_arm80+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_arm80+#define GGML_VECTOR+#include "ggml-vector.inc"+#endif // __aarch64__diff --git a/llama.cpp/ggml-vector-arm82.c b/llama.cpp/ggml-vector-arm82.cnew file mode 100644index 00000000..fa07a23f--- /dev/null+++ b/llama.cpp/ggml-vector-arm82.c@@ -0,0 +1,57 @@+#ifdef __aarch64__+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_arm82+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_arm82+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_arm82+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_arm82+#define ggml_vec_dot_f32 ggml_vec_dot_f32_arm82+#define ggml_vec_dot_f16 ggml_vec_dot_f16_arm82+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_arm82+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_arm82+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_arm82+#define ggml_vec_set_i8 ggml_vec_set_i8_arm82+#define ggml_vec_set_i16 ggml_vec_set_i16_arm82+#define ggml_vec_set_i32 ggml_vec_set_i32_arm82+#define ggml_vec_set_f16 ggml_vec_set_f16_arm82+#define ggml_vec_set_bf16 ggml_vec_set_bf16_arm82+#define ggml_vec_add_f32 ggml_vec_add_f32_arm82+#define ggml_vec_add1_f32 ggml_vec_add1_f32_arm82+#define ggml_vec_acc_f32 ggml_vec_acc_f32_arm82+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_arm82+#define ggml_vec_sub_f32 ggml_vec_sub_f32_arm82+#define ggml_vec_set_f32 ggml_vec_set_f32_arm82+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_arm82+#define ggml_vec_neg_f32 ggml_vec_neg_f32_arm82+#define ggml_vec_mul_f32 ggml_vec_mul_f32_arm82+#define ggml_vec_div_f32 ggml_vec_div_f32_arm82+#define ggml_vec_scale_f32 ggml_vec_scale_f32_arm82+#define ggml_vec_scale_f16 ggml_vec_scale_f16_arm82+#define ggml_vec_mad_f32 ggml_vec_mad_f32_arm82+#define ggml_vec_mad_f16 ggml_vec_mad_f16_arm82+#define ggml_vec_norm_f32 ggml_vec_norm_f32_arm82+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_arm82+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_arm82+#define ggml_vec_log_f32 ggml_vec_log_f32_arm82+#define ggml_vec_abs_f32 ggml_vec_abs_f32_arm82+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_arm82+#define ggml_vec_step_f32 ggml_vec_step_f32_arm82+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_arm82+#define ggml_vec_elu_f32 ggml_vec_elu_f32_arm82+#define ggml_vec_relu_f32 ggml_vec_relu_f32_arm82+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_arm82+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_arm82+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_arm82+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_arm82+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_arm82+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_arm82+#define ggml_vec_silu_f32 ggml_vec_silu_f32_arm82+#define ggml_silu_backward_f32 ggml_silu_backward_f32_arm82+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_arm82+#define ggml_vec_sum_f32 ggml_vec_sum_f32_arm82+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_arm82+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_arm82+#define ggml_vec_max_f32 ggml_vec_max_f32_arm82+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_arm82+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_arm82+#define GGML_VECTOR+#include "ggml-vector.inc"+#endif // __aarch64__diff --git a/llama.cpp/ggml-vector.cpp b/llama.cpp/ggml-vector.cppnew file mode 100644index 00000000..a6aed0cb--- /dev/null+++ b/llama.cpp/ggml-vector.cpp@@ -0,0 +1,1217 @@+#include <cosmo.h>+#include <sys/auxv.h>+#include <libc/sysv/consts/hwcap.h>+#include "ggml-vector.h"++extern "C" void ggml_fp16_to_fp32_row_amd_avx512bf16(const ggml_fp16_t * x, float * y, int64_t n);+extern "C" void ggml_fp16_to_fp32_row_amd_avx512(const ggml_fp16_t * x, float * y, int64_t n);+extern "C" void ggml_fp16_to_fp32_row_amd_avx2(const ggml_fp16_t * x, float * y, int64_t n);+extern "C" void ggml_fp16_to_fp32_row_amd_f16c(const ggml_fp16_t * x, float * y, int64_t n);+extern "C" void ggml_fp16_to_fp32_row_amd_fma(const ggml_fp16_t * x, float * y, int64_t n);+extern "C" void ggml_fp16_to_fp32_row_amd_avx(const ggml_fp16_t * x, float * y, int64_t n);+extern "C" void ggml_fp16_to_fp32_row_arm82(const ggml_fp16_t * x, float * y, int64_t n);+extern "C" void ggml_fp16_to_fp32_row_arm80(const ggml_fp16_t * x, float * y, int64_t n);++extern "C" void ggml_fp32_to_fp16_row_amd_avx512bf16(const float * x, ggml_fp16_t * y, int64_t n);+extern "C" void ggml_fp32_to_fp16_row_amd_avx512(const float * x, ggml_fp16_t * y, int64_t n);+extern "C" void ggml_fp32_to_fp16_row_amd_avx2(const float * x, ggml_fp16_t * y, int64_t n);+extern "C" void ggml_fp32_to_fp16_row_amd_f16c(const float * x, ggml_fp16_t * y, int64_t n);+extern "C" void ggml_fp32_to_fp16_row_amd_fma(const float * x, ggml_fp16_t * y, int64_t n);+extern "C" void ggml_fp32_to_fp16_row_amd_avx(const float * x, ggml_fp16_t * y, int64_t n);+extern "C" void ggml_fp32_to_fp16_row_arm82(const float * x, ggml_fp16_t * y, int64_t n);+extern "C" void ggml_fp32_to_fp16_row_arm80(const float * x, ggml_fp16_t * y, int64_t n);++extern "C" void ggml_bf16_to_fp32_row_amd_avx512bf16(const ggml_bf16_t * x, float * y, int64_t n);+extern "C" void ggml_bf16_to_fp32_row_amd_avx512(const ggml_bf16_t * x, float * y, int64_t n);+extern "C" void ggml_bf16_to_fp32_row_amd_avx2(const ggml_bf16_t * x, float * y, int64_t n);+extern "C" void ggml_bf16_to_fp32_row_amd_f16c(const ggml_bf16_t * x, float * y, int64_t n);+extern "C" void ggml_bf16_to_fp32_row_amd_fma(const ggml_bf16_t * x, float * y, int64_t n);+extern "C" void ggml_bf16_to_fp32_row_amd_avx(const ggml_bf16_t * x, float * y, int64_t n);+extern "C" void ggml_bf16_to_fp32_row_arm82(const ggml_bf16_t * x, float * y, int64_t n);+extern "C" void ggml_bf16_to_fp32_row_arm80(const ggml_bf16_t * x, float * y, int64_t n);++extern "C" void ggml_fp32_to_bf16_row_amd_avx512bf16(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_amd_avx512(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_amd_avx2(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_amd_f16c(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_amd_fma(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_amd_avx(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_arm82(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_arm80(const float * x, ggml_bf16_t * y, int64_t n);++extern "C" void ggml_vec_dot_f32_amd_avx512bf16(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_amd_avx512(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_amd_avx2(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_amd_f16c(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_amd_fma(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_amd_avx(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_arm82(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f32_arm80(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);++extern "C" void ggml_vec_dot_f16_amd_avx512bf16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_amd_avx512(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_amd_avx2(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_amd_f16c(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_amd_fma(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_amd_avx(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_arm82(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_f16_arm80(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);++extern "C" void ggml_vec_dot_bf16_amd_avx512bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_amd_avx512(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_amd_avx2(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_amd_f16c(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_amd_fma(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_amd_avx(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_arm82(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);+extern "C" void ggml_vec_dot_bf16_arm80(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);++extern "C" void ggml_vec_dot_f16_unroll_amd_avx512bf16(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);+extern "C" void ggml_vec_dot_f16_unroll_amd_avx512(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);+extern "C" void ggml_vec_dot_f16_unroll_amd_avx2(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);+extern "C" void ggml_vec_dot_f16_unroll_amd_f16c(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);+extern "C" void ggml_vec_dot_f16_unroll_amd_fma(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);+extern "C" void ggml_vec_dot_f16_unroll_amd_avx(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);+extern "C" void ggml_vec_dot_f16_unroll_arm82(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);+extern "C" void ggml_vec_dot_f16_unroll_arm80(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);++extern "C" void ggml_vec_mad_f32_unroll_amd_avx512bf16(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);+extern "C" void ggml_vec_mad_f32_unroll_amd_avx512(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);+extern "C" void ggml_vec_mad_f32_unroll_amd_avx2(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);+extern "C" void ggml_vec_mad_f32_unroll_amd_f16c(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);+extern "C" void ggml_vec_mad_f32_unroll_amd_fma(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);+extern "C" void ggml_vec_mad_f32_unroll_amd_avx(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);+extern "C" void ggml_vec_mad_f32_unroll_arm82(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);+extern "C" void ggml_vec_mad_f32_unroll_arm80(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);++extern "C" void ggml_vec_set_i8_amd_avx512bf16(const int n, int8_t * x, const int8_t v);+extern "C" void ggml_vec_set_i8_amd_avx512(const int n, int8_t * x, const int8_t v);+extern "C" void ggml_vec_set_i8_amd_avx2(const int n, int8_t * x, const int8_t v);+extern "C" void ggml_vec_set_i8_amd_f16c(const int n, int8_t * x, const int8_t v);+extern "C" void ggml_vec_set_i8_amd_fma(const int n, int8_t * x, const int8_t v);+extern "C" void ggml_vec_set_i8_amd_avx(const int n, int8_t * x, const int8_t v);+extern "C" void ggml_vec_set_i8_arm82(const int n, int8_t * x, const int8_t v);+extern "C" void ggml_vec_set_i8_arm80(const int n, int8_t * x, const int8_t v);++extern "C" void ggml_vec_set_i16_amd_avx512bf16(const int n, int16_t * x, const int16_t v);+extern "C" void ggml_vec_set_i16_amd_avx512(const int n, int16_t * x, const int16_t v);+extern "C" void ggml_vec_set_i16_amd_avx2(const int n, int16_t * x, const int16_t v);+extern "C" void ggml_vec_set_i16_amd_f16c(const int n, int16_t * x, const int16_t v);+extern "C" void ggml_vec_set_i16_amd_fma(const int n, int16_t * x, const int16_t v);+extern "C" void ggml_vec_set_i16_amd_avx(const int n, int16_t * x, const int16_t v);+extern "C" void ggml_vec_set_i16_arm82(const int n, int16_t * x, const int16_t v);+extern "C" void ggml_vec_set_i16_arm80(const int n, int16_t * x, const int16_t v);++extern "C" void ggml_vec_set_i32_amd_avx512bf16(const int n, int32_t * x, const int32_t v);+extern "C" void ggml_vec_set_i32_amd_avx512(const int n, int32_t * x, const int32_t v);+extern "C" void ggml_vec_set_i32_amd_avx2(const int n, int32_t * x, const int32_t v);+extern "C" void ggml_vec_set_i32_amd_f16c(const int n, int32_t * x, const int32_t v);+extern "C" void ggml_vec_set_i32_amd_fma(const int n, int32_t * x, const int32_t v);+extern "C" void ggml_vec_set_i32_amd_avx(const int n, int32_t * x, const int32_t v);+extern "C" void ggml_vec_set_i32_arm82(const int n, int32_t * x, const int32_t v);+extern "C" void ggml_vec_set_i32_arm80(const int n, int32_t * x, const int32_t v);++extern "C" void ggml_vec_set_f16_amd_avx512bf16(const int n, ggml_fp16_t * x, const int32_t v);+extern "C" void ggml_vec_set_f16_amd_avx512(const int n, ggml_fp16_t * x, const int32_t v);+extern "C" void ggml_vec_set_f16_amd_avx2(const int n, ggml_fp16_t * x, const int32_t v);+extern "C" void ggml_vec_set_f16_amd_f16c(const int n, ggml_fp16_t * x, const int32_t v);+extern "C" void ggml_vec_set_f16_amd_fma(const int n, ggml_fp16_t * x, const int32_t v);+extern "C" void ggml_vec_set_f16_amd_avx(const int n, ggml_fp16_t * x, const int32_t v);+extern "C" void ggml_vec_set_f16_arm82(const int n, ggml_fp16_t * x, const int32_t v);+extern "C" void ggml_vec_set_f16_arm80(const int n, ggml_fp16_t * x, const int32_t v);++extern "C" void ggml_vec_set_bf16_amd_avx512bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v);+extern "C" void ggml_vec_set_bf16_amd_avx512(const int n, ggml_bf16_t * x, const ggml_bf16_t v);+extern "C" void ggml_vec_set_bf16_amd_avx2(const int n, ggml_bf16_t * x, const ggml_bf16_t v);+extern "C" void ggml_vec_set_bf16_amd_f16c(const int n, ggml_bf16_t * x, const ggml_bf16_t v);+extern "C" void ggml_vec_set_bf16_amd_fma(const int n, ggml_bf16_t * x, const ggml_bf16_t v);+extern "C" void ggml_vec_set_bf16_amd_avx(const int n, ggml_bf16_t * x, const ggml_bf16_t v);+extern "C" void ggml_vec_set_bf16_arm82(const int n, ggml_bf16_t * x, const ggml_bf16_t v);+extern "C" void ggml_vec_set_bf16_arm80(const int n, ggml_bf16_t * x, const ggml_bf16_t v);++extern "C" void ggml_vec_add_f32_amd_avx512bf16 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_add_f32_amd_avx512 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_add_f32_amd_avx2 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_add_f32_amd_f16c (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_add_f32_amd_fma (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_add_f32_amd_avx (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_add_f32_arm82 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_add_f32_arm80 (const int n, float * z, const float * x, const float * y);++extern "C" void ggml_vec_add1_f32_amd_avx512bf16(const int n, float * z, const float * x, const float v);+extern "C" void ggml_vec_add1_f32_amd_avx512(const int n, float * z, const float * x, const float v);+extern "C" void ggml_vec_add1_f32_amd_avx2(const int n, float * z, const float * x, const float v);+extern "C" void ggml_vec_add1_f32_amd_f16c(const int n, float * z, const float * x, const float v);+extern "C" void ggml_vec_add1_f32_amd_fma(const int n, float * z, const float * x, const float v);+extern "C" void ggml_vec_add1_f32_amd_avx(const int n, float * z, const float * x, const float v);+extern "C" void ggml_vec_add1_f32_arm82(const int n, float * z, const float * x, const float v);+extern "C" void ggml_vec_add1_f32_arm80(const int n, float * z, const float * x, const float v);++extern "C" void ggml_vec_acc_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_acc_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_acc_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_acc_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_acc_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_acc_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_acc_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_acc_f32_arm80 (const int n, float * y, const float * x);++extern "C" void ggml_vec_acc1_f32_amd_avx512bf16(const int n, float * y, const float v);+extern "C" void ggml_vec_acc1_f32_amd_avx512(const int n, float * y, const float v);+extern "C" void ggml_vec_acc1_f32_amd_avx2(const int n, float * y, const float v);+extern "C" void ggml_vec_acc1_f32_amd_f16c(const int n, float * y, const float v);+extern "C" void ggml_vec_acc1_f32_amd_fma(const int n, float * y, const float v);+extern "C" void ggml_vec_acc1_f32_amd_avx(const int n, float * y, const float v);+extern "C" void ggml_vec_acc1_f32_arm82(const int n, float * y, const float v);+extern "C" void ggml_vec_acc1_f32_arm80(const int n, float * y, const float v);++extern "C" void ggml_vec_sub_f32_amd_avx512bf16 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_sub_f32_amd_avx512 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_sub_f32_amd_avx2 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_sub_f32_amd_f16c (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_sub_f32_amd_fma (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_sub_f32_amd_avx (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_sub_f32_arm82 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_sub_f32_arm80 (const int n, float * z, const float * x, const float * y);++extern "C" void ggml_vec_set_f32_amd_avx512bf16 (const int n, float * x, const float v);+extern "C" void ggml_vec_set_f32_amd_avx512 (const int n, float * x, const float v);+extern "C" void ggml_vec_set_f32_amd_avx2 (const int n, float * x, const float v);+extern "C" void ggml_vec_set_f32_amd_f16c (const int n, float * x, const float v);+extern "C" void ggml_vec_set_f32_amd_fma (const int n, float * x, const float v);+extern "C" void ggml_vec_set_f32_amd_avx (const int n, float * x, const float v);+extern "C" void ggml_vec_set_f32_arm82 (const int n, float * x, const float v);+extern "C" void ggml_vec_set_f32_arm80 (const int n, float * x, const float v);++extern "C" void ggml_vec_cpy_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_cpy_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_cpy_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_cpy_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_cpy_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_cpy_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_cpy_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_cpy_f32_arm80 (const int n, float * y, const float * x);++extern "C" void ggml_vec_neg_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_neg_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_neg_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_neg_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_neg_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_neg_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_neg_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_neg_f32_arm80 (const int n, float * y, const float * x);++extern "C" void ggml_vec_mul_f32_amd_avx512bf16 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_mul_f32_amd_avx512 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_mul_f32_amd_avx2 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_mul_f32_amd_f16c (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_mul_f32_amd_fma (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_mul_f32_amd_avx (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_mul_f32_arm82 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_mul_f32_arm80 (const int n, float * z, const float * x, const float * y);++extern "C" void ggml_vec_div_f32_amd_avx512bf16 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_div_f32_amd_avx512 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_div_f32_amd_avx2 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_div_f32_amd_f16c (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_div_f32_amd_fma (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_div_f32_amd_avx (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_div_f32_arm82 (const int n, float * z, const float * x, const float * y);+extern "C" void ggml_vec_div_f32_arm80 (const int n, float * z, const float * x, const float * y);++extern "C" void ggml_vec_scale_f32_amd_avx512bf16(const int n, float * y, const float v);+extern "C" void ggml_vec_scale_f32_amd_avx512(const int n, float * y, const float v);+extern "C" void ggml_vec_scale_f32_amd_avx2(const int n, float * y, const float v);+extern "C" void ggml_vec_scale_f32_amd_f16c(const int n, float * y, const float v);+extern "C" void ggml_vec_scale_f32_amd_fma(const int n, float * y, const float v);+extern "C" void ggml_vec_scale_f32_amd_avx(const int n, float * y, const float v);+extern "C" void ggml_vec_scale_f32_arm82(const int n, float * y, const float v);+extern "C" void ggml_vec_scale_f32_arm80(const int n, float * y, const float v);++extern "C" void ggml_vec_scale_f16_amd_avx512bf16(const int n, ggml_fp16_t * y, const float v);+extern "C" void ggml_vec_scale_f16_amd_avx512(const int n, ggml_fp16_t * y, const float v);+extern "C" void ggml_vec_scale_f16_amd_avx2(const int n, ggml_fp16_t * y, const float v);+extern "C" void ggml_vec_scale_f16_amd_f16c(const int n, ggml_fp16_t * y, const float v);+extern "C" void ggml_vec_scale_f16_amd_fma(const int n, ggml_fp16_t * y, const float v);+extern "C" void ggml_vec_scale_f16_amd_avx(const int n, ggml_fp16_t * y, const float v);+extern "C" void ggml_vec_scale_f16_arm82(const int n, ggml_fp16_t * y, const float v);+extern "C" void ggml_vec_scale_f16_arm80(const int n, ggml_fp16_t * y, const float v);++extern "C" void ggml_vec_mad_f32_amd_avx512bf16(const int n, float * restrict y, const float * restrict x, const float v);+extern "C" void ggml_vec_mad_f32_amd_avx512(const int n, float * restrict y, const float * restrict x, const float v);+extern "C" void ggml_vec_mad_f32_amd_avx2(const int n, float * restrict y, const float * restrict x, const float v);+extern "C" void ggml_vec_mad_f32_amd_f16c(const int n, float * restrict y, const float * restrict x, const float v);+extern "C" void ggml_vec_mad_f32_amd_fma(const int n, float * restrict y, const float * restrict x, const float v);+extern "C" void ggml_vec_mad_f32_amd_avx(const int n, float * restrict y, const float * restrict x, const float v);+extern "C" void ggml_vec_mad_f32_arm82(const int n, float * restrict y, const float * restrict x, const float v);+extern "C" void ggml_vec_mad_f32_arm80(const int n, float * restrict y, const float * restrict x, const float v);++extern "C" void ggml_vec_mad_f16_amd_avx512bf16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);+extern "C" void ggml_vec_mad_f16_amd_avx512(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);+extern "C" void ggml_vec_mad_f16_amd_avx2(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);+extern "C" void ggml_vec_mad_f16_amd_f16c(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);+extern "C" void ggml_vec_mad_f16_amd_fma(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);+extern "C" void ggml_vec_mad_f16_amd_avx(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);+extern "C" void ggml_vec_mad_f16_arm82(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);+extern "C" void ggml_vec_mad_f16_arm80(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);++extern "C" void ggml_vec_norm_f32_amd_avx512bf16 (const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_f32_amd_avx512 (const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_f32_amd_avx2 (const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_f32_amd_f16c (const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_f32_amd_fma (const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_f32_amd_avx (const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_f32_arm82 (const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_f32_arm80 (const int n, float * s, const float * x);++extern "C" void ggml_vec_sqr_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sqr_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sqr_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sqr_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_sqr_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_sqr_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_sqr_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sqr_f32_arm80 (const int n, float * y, const float * x);++extern "C" void ggml_vec_sqrt_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sqrt_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sqrt_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sqrt_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_sqrt_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_sqrt_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_sqrt_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sqrt_f32_arm80 (const int n, float * y, const float * x);++extern "C" void ggml_vec_log_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_log_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_log_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_log_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_log_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_log_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_log_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_log_f32_arm80 (const int n, float * y, const float * x);++extern "C" void ggml_vec_abs_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_abs_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_abs_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_abs_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_abs_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_abs_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_abs_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_abs_f32_arm80 (const int n, float * y, const float * x);++extern "C" void ggml_vec_sgn_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sgn_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sgn_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sgn_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_sgn_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_sgn_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_sgn_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sgn_f32_arm80 (const int n, float * y, const float * x);++extern "C" void ggml_vec_step_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_step_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_step_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_step_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_step_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_step_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_step_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_step_f32_arm80 (const int n, float * y, const float * x);++extern "C" void ggml_vec_tanh_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_tanh_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_tanh_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_tanh_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_tanh_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_tanh_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_tanh_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_tanh_f32_arm80 (const int n, float * y, const float * x);++extern "C" void ggml_vec_elu_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_elu_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_elu_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_elu_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_elu_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_elu_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_elu_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_elu_f32_arm80 (const int n, float * y, const float * x);++extern "C" void ggml_vec_relu_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_relu_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_relu_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_relu_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_relu_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_relu_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_relu_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_relu_f32_arm80 (const int n, float * y, const float * x);++extern "C" void ggml_vec_leaky_relu_f32_amd_avx512bf16 (const int n, float * y, const float * x, const float ns);+extern "C" void ggml_vec_leaky_relu_f32_amd_avx512 (const int n, float * y, const float * x, const float ns);+extern "C" void ggml_vec_leaky_relu_f32_amd_avx2 (const int n, float * y, const float * x, const float ns);+extern "C" void ggml_vec_leaky_relu_f32_amd_f16c (const int n, float * y, const float * x, const float ns);+extern "C" void ggml_vec_leaky_relu_f32_amd_fma (const int n, float * y, const float * x, const float ns);+extern "C" void ggml_vec_leaky_relu_f32_amd_avx (const int n, float * y, const float * x, const float ns);+extern "C" void ggml_vec_leaky_relu_f32_arm82 (const int n, float * y, const float * x, const float ns);+extern "C" void ggml_vec_leaky_relu_f32_arm80 (const int n, float * y, const float * x, const float ns);++extern "C" void ggml_vec_hardswish_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_hardswish_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_hardswish_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_hardswish_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_hardswish_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_hardswish_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_hardswish_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_hardswish_f32_arm80 (const int n, float * y, const float * x);++extern "C" void ggml_vec_hardsigmoid_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_hardsigmoid_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_hardsigmoid_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_hardsigmoid_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_hardsigmoid_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_hardsigmoid_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_hardsigmoid_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_hardsigmoid_f32_arm80 (const int n, float * y, const float * x);++extern "C" void ggml_vec_gelu_f16_amd_avx512bf16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);+extern "C" void ggml_vec_gelu_f16_amd_avx512(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);+extern "C" void ggml_vec_gelu_f16_amd_avx2(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);+extern "C" void ggml_vec_gelu_f16_amd_f16c(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);+extern "C" void ggml_vec_gelu_f16_amd_fma(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);+extern "C" void ggml_vec_gelu_f16_amd_avx(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);+extern "C" void ggml_vec_gelu_f16_arm82(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);+extern "C" void ggml_vec_gelu_f16_arm80(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);++extern "C" void ggml_vec_gelu_f32_amd_avx512bf16(const int n, float * y, const float * x);+extern "C" void ggml_vec_gelu_f32_amd_avx512(const int n, float * y, const float * x);+extern "C" void ggml_vec_gelu_f32_amd_avx2(const int n, float * y, const float * x);+extern "C" void ggml_vec_gelu_f32_amd_f16c(const int n, float * y, const float * x);+extern "C" void ggml_vec_gelu_f32_amd_fma(const int n, float * y, const float * x);+extern "C" void ggml_vec_gelu_f32_amd_avx(const int n, float * y, const float * x);+extern "C" void ggml_vec_gelu_f32_arm82(const int n, float * y, const float * x);+extern "C" void ggml_vec_gelu_f32_arm80(const int n, float * y, const float * x);++extern "C" void ggml_vec_gelu_quick_f32_amd_avx512bf16(const int n, float * y, const float * x);+extern "C" void ggml_vec_gelu_quick_f32_amd_avx512(const int n, float * y, const float * x);+extern "C" void ggml_vec_gelu_quick_f32_amd_avx2(const int n, float * y, const float * x);+extern "C" void ggml_vec_gelu_quick_f32_amd_f16c(const int n, float * y, const float * x);+extern "C" void ggml_vec_gelu_quick_f32_amd_fma(const int n, float * y, const float * x);+extern "C" void ggml_vec_gelu_quick_f32_amd_avx(const int n, float * y, const float * x);+extern "C" void ggml_vec_gelu_quick_f32_arm82(const int n, float * y, const float * x);+extern "C" void ggml_vec_gelu_quick_f32_arm80(const int n, float * y, const float * x);++extern "C" void ggml_vec_silu_f32_amd_avx512bf16(const int n, float * y, const float * x);+extern "C" void ggml_vec_silu_f32_amd_avx512(const int n, float * y, const float * x);+extern "C" void ggml_vec_silu_f32_amd_avx2(const int n, float * y, const float * x);+extern "C" void ggml_vec_silu_f32_amd_f16c(const int n, float * y, const float * x);+extern "C" void ggml_vec_silu_f32_amd_fma(const int n, float * y, const float * x);+extern "C" void ggml_vec_silu_f32_amd_avx(const int n, float * y, const float * x);+extern "C" void ggml_vec_silu_f32_arm82(const int n, float * y, const float * x);+extern "C" void ggml_vec_silu_f32_arm80(const int n, float * y, const float * x);++extern "C" float ggml_silu_backward_f32_amd_avx512bf16(float x, float dy);+extern "C" float ggml_silu_backward_f32_amd_avx512(float x, float dy);+extern "C" float ggml_silu_backward_f32_amd_avx2(float x, float dy);+extern "C" float ggml_silu_backward_f32_amd_f16c(float x, float dy);+extern "C" float ggml_silu_backward_f32_amd_fma(float x, float dy);+extern "C" float ggml_silu_backward_f32_amd_avx(float x, float dy);+extern "C" float ggml_silu_backward_f32_arm82(float x, float dy);+extern "C" float ggml_silu_backward_f32_arm80(float x, float dy);++extern "C" void ggml_vec_silu_backward_f32_amd_avx512bf16(const int n, float * dx, const float * x, const float * dy);+extern "C" void ggml_vec_silu_backward_f32_amd_avx512(const int n, float * dx, const float * x, const float * dy);+extern "C" void ggml_vec_silu_backward_f32_amd_avx2(const int n, float * dx, const float * x, const float * dy);+extern "C" void ggml_vec_silu_backward_f32_amd_f16c(const int n, float * dx, const float * x, const float * dy);+extern "C" void ggml_vec_silu_backward_f32_amd_fma(const int n, float * dx, const float * x, const float * dy);+extern "C" void ggml_vec_silu_backward_f32_amd_avx(const int n, float * dx, const float * x, const float * dy);+extern "C" void ggml_vec_silu_backward_f32_arm82(const int n, float * dx, const float * x, const float * dy);+extern "C" void ggml_vec_silu_backward_f32_arm80(const int n, float * dx, const float * x, const float * dy);++extern "C" void ggml_vec_sum_f32_amd_avx512bf16(const int n, float * s, const float * x);+extern "C" void ggml_vec_sum_f32_amd_avx512(const int n, float * s, const float * x);+extern "C" void ggml_vec_sum_f32_amd_avx2(const int n, float * s, const float * x);+extern "C" void ggml_vec_sum_f32_amd_f16c(const int n, float * s, const float * x);+extern "C" void ggml_vec_sum_f32_amd_fma(const int n, float * s, const float * x);+extern "C" void ggml_vec_sum_f32_amd_avx(const int n, float * s, const float * x);+extern "C" void ggml_vec_sum_f32_arm82(const int n, float * s, const float * x);+extern "C" void ggml_vec_sum_f32_arm80(const int n, float * s, const float * x);++extern "C" void ggml_vec_sum_f32_ggf_amd_avx512bf16(const int n, ggml_float * s, const float * x);+extern "C" void ggml_vec_sum_f32_ggf_amd_avx512(const int n, ggml_float * s, const float * x);+extern "C" void ggml_vec_sum_f32_ggf_amd_avx2(const int n, ggml_float * s, const float * x);+extern "C" void ggml_vec_sum_f32_ggf_amd_f16c(const int n, ggml_float * s, const float * x);+extern "C" void ggml_vec_sum_f32_ggf_amd_fma(const int n, ggml_float * s, const float * x);+extern "C" void ggml_vec_sum_f32_ggf_amd_avx(const int n, ggml_float * s, const float * x);+extern "C" void ggml_vec_sum_f32_ggf_arm82(const int n, ggml_float * s, const float * x);+extern "C" void ggml_vec_sum_f32_ggf_arm80(const int n, ggml_float * s, const float * x);++extern "C" void ggml_vec_sum_f16_ggf_amd_avx512bf16(const int n, float * s, const ggml_fp16_t * x);+extern "C" void ggml_vec_sum_f16_ggf_amd_avx512(const int n, float * s, const ggml_fp16_t * x);+extern "C" void ggml_vec_sum_f16_ggf_amd_avx2(const int n, float * s, const ggml_fp16_t * x);+extern "C" void ggml_vec_sum_f16_ggf_amd_f16c(const int n, float * s, const ggml_fp16_t * x);+extern "C" void ggml_vec_sum_f16_ggf_amd_fma(const int n, float * s, const ggml_fp16_t * x);+extern "C" void ggml_vec_sum_f16_ggf_amd_avx(const int n, float * s, const ggml_fp16_t * x);+extern "C" void ggml_vec_sum_f16_ggf_arm82(const int n, float * s, const ggml_fp16_t * x);+extern "C" void ggml_vec_sum_f16_ggf_arm80(const int n, float * s, const ggml_fp16_t * x);++extern "C" void ggml_vec_max_f32_amd_avx512bf16(const int n, float * s, const float * x);+extern "C" void ggml_vec_max_f32_amd_avx512(const int n, float * s, const float * x);+extern "C" void ggml_vec_max_f32_amd_avx2(const int n, float * s, const float * x);+extern "C" void ggml_vec_max_f32_amd_f16c(const int n, float * s, const float * x);+extern "C" void ggml_vec_max_f32_amd_fma(const int n, float * s, const float * x);+extern "C" void ggml_vec_max_f32_amd_avx(const int n, float * s, const float * x);+extern "C" void ggml_vec_max_f32_arm82(const int n, float * s, const float * x);+extern "C" void ggml_vec_max_f32_arm80(const int n, float * s, const float * x);++extern "C" void ggml_vec_norm_inv_f32_amd_avx512bf16(const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_inv_f32_amd_avx512(const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_inv_f32_amd_avx2(const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_inv_f32_amd_f16c(const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_inv_f32_amd_fma(const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_inv_f32_amd_avx(const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_inv_f32_arm82(const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_inv_f32_arm80(const int n, float * s, const float * x);++extern "C" void ggml_vec_argmax_f32_amd_avx512bf16(const int n, int * s, const float * x);+extern "C" void ggml_vec_argmax_f32_amd_avx512(const int n, int * s, const float * x);+extern "C" void ggml_vec_argmax_f32_amd_avx2(const int n, int * s, const float * x);+extern "C" void ggml_vec_argmax_f32_amd_f16c(const int n, int * s, const float * x);+extern "C" void ggml_vec_argmax_f32_amd_fma(const int n, int * s, const float * x);+extern "C" void ggml_vec_argmax_f32_amd_avx(const int n, int * s, const float * x);+extern "C" void ggml_vec_argmax_f32_arm82(const int n, int * s, const float * x);+extern "C" void ggml_vec_argmax_f32_arm80(const int n, int * s, const float * x);++static const struct VectorFuncs {+ typeof(ggml_fp16_to_fp32_row) *ptr_ggml_fp16_to_fp32_row;+ typeof(ggml_fp32_to_fp16_row) *ptr_ggml_fp32_to_fp16_row;+ typeof(ggml_bf16_to_fp32_row) *ptr_ggml_bf16_to_fp32_row;+ typeof(ggml_fp32_to_bf16_row) *ptr_ggml_fp32_to_bf16_row;+ typeof(ggml_vec_dot_f32) *ptr_ggml_vec_dot_f32;+ typeof(ggml_vec_dot_f16) *ptr_ggml_vec_dot_f16;+ typeof(ggml_vec_dot_bf16) *ptr_ggml_vec_dot_bf16;+ typeof(ggml_vec_dot_f16_unroll) *ptr_ggml_vec_dot_f16_unroll;+ typeof(ggml_vec_mad_f32_unroll) *ptr_ggml_vec_mad_f32_unroll;+ typeof(ggml_vec_set_i8) *ptr_ggml_vec_set_i8;+ typeof(ggml_vec_set_i16) *ptr_ggml_vec_set_i16;+ typeof(ggml_vec_set_i32) *ptr_ggml_vec_set_i32;+ typeof(ggml_vec_set_f16) *ptr_ggml_vec_set_f16;+ typeof(ggml_vec_set_bf16) *ptr_ggml_vec_set_bf16;+ typeof(ggml_vec_add_f32) *ptr_ggml_vec_add_f32;+ typeof(ggml_vec_add1_f32) *ptr_ggml_vec_add1_f32;+ typeof(ggml_vec_acc_f32) *ptr_ggml_vec_acc_f32;+ typeof(ggml_vec_acc1_f32) *ptr_ggml_vec_acc1_f32;+ typeof(ggml_vec_sub_f32) *ptr_ggml_vec_sub_f32;+ typeof(ggml_vec_set_f32) *ptr_ggml_vec_set_f32;+ typeof(ggml_vec_cpy_f32) *ptr_ggml_vec_cpy_f32;+ typeof(ggml_vec_neg_f32) *ptr_ggml_vec_neg_f32;+ typeof(ggml_vec_mul_f32) *ptr_ggml_vec_mul_f32;+ typeof(ggml_vec_div_f32) *ptr_ggml_vec_div_f32;+ typeof(ggml_vec_scale_f32) *ptr_ggml_vec_scale_f32;+ typeof(ggml_vec_scale_f16) *ptr_ggml_vec_scale_f16;+ typeof(ggml_vec_mad_f32) *ptr_ggml_vec_mad_f32;+ typeof(ggml_vec_mad_f16) *ptr_ggml_vec_mad_f16;+ typeof(ggml_vec_norm_f32) *ptr_ggml_vec_norm_f32;+ typeof(ggml_vec_sqr_f32) *ptr_ggml_vec_sqr_f32;+ typeof(ggml_vec_sqrt_f32) *ptr_ggml_vec_sqrt_f32;+ typeof(ggml_vec_log_f32) *ptr_ggml_vec_log_f32;+ typeof(ggml_vec_abs_f32) *ptr_ggml_vec_abs_f32;+ typeof(ggml_vec_sgn_f32) *ptr_ggml_vec_sgn_f32;+ typeof(ggml_vec_step_f32) *ptr_ggml_vec_step_f32;+ typeof(ggml_vec_tanh_f32) *ptr_ggml_vec_tanh_f32;+ typeof(ggml_vec_elu_f32) *ptr_ggml_vec_elu_f32;+ typeof(ggml_vec_relu_f32) *ptr_ggml_vec_relu_f32;+ typeof(ggml_vec_leaky_relu_f32) *ptr_ggml_vec_leaky_relu_f32;+ typeof(ggml_vec_hardswish_f32) *ptr_ggml_vec_hardswish_f32;+ typeof(ggml_vec_hardsigmoid_f32) *ptr_ggml_vec_hardsigmoid_f32;+ typeof(ggml_vec_gelu_f16) *ptr_ggml_vec_gelu_f16;+ typeof(ggml_vec_gelu_f32) *ptr_ggml_vec_gelu_f32;+ typeof(ggml_vec_gelu_quick_f32) *ptr_ggml_vec_gelu_quick_f32;+ typeof(ggml_vec_silu_f32) *ptr_ggml_vec_silu_f32;+ typeof(ggml_silu_backward_f32) *ptr_ggml_silu_backward_f32;+ typeof(ggml_vec_silu_backward_f32) *ptr_ggml_vec_silu_backward_f32;+ typeof(ggml_vec_sum_f32) *ptr_ggml_vec_sum_f32;+ typeof(ggml_vec_sum_f32_ggf) *ptr_ggml_vec_sum_f32_ggf;+ typeof(ggml_vec_sum_f16_ggf) *ptr_ggml_vec_sum_f16_ggf;+ typeof(ggml_vec_max_f32) *ptr_ggml_vec_max_f32;+ typeof(ggml_vec_norm_inv_f32) *ptr_ggml_vec_norm_inv_f32;+ typeof(ggml_vec_argmax_f32) *ptr_ggml_vec_argmax_f32;++ VectorFuncs() {+#ifdef __x86_64__+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2) && X86_HAVE(AVX512F) && X86_HAVE(AVX512VL) && X86_HAVE(AVX512_BF16)) {+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_avx512bf16;+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx512bf16;+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx512bf16;+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx512bf16;+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx512bf16;+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx512bf16;+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx512bf16;+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_amd_avx512bf16;+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_amd_avx512bf16;+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_amd_avx512bf16;+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_amd_avx512bf16;+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_amd_avx512bf16;+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_amd_avx512bf16;+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_amd_avx512bf16;+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_amd_avx512bf16;+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_amd_avx512bf16;+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_amd_avx512bf16;+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_amd_avx512bf16;+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_amd_avx512bf16;+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_amd_avx512bf16;+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_amd_avx512bf16;+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_amd_avx512bf16;+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_amd_avx512bf16;+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_amd_avx512bf16;+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_amd_avx512bf16;+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_amd_avx512bf16;+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_amd_avx512bf16;+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_amd_avx512bf16;+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_amd_avx512bf16;+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_amd_avx512bf16;+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_amd_avx512bf16;+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_amd_avx512bf16;+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_amd_avx512bf16;+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_amd_avx512bf16;+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_amd_avx512bf16;+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_amd_avx512bf16;+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_amd_avx512bf16;+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_amd_avx512bf16;+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_amd_avx512bf16;+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_amd_avx512bf16;+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_amd_avx512bf16;+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_amd_avx512bf16;+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_amd_avx512bf16;+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_amd_avx512bf16;+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_amd_avx512bf16;+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_amd_avx512bf16;+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_amd_avx512bf16;+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_avx512bf16;+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_avx512bf16;+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_avx512bf16;+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx512bf16;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx512bf16;+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx512bf16;+ return;+ }+#endif+#ifdef __x86_64__+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2) && X86_HAVE(AVX512F)) {+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_avx512;+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx512;+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx512;+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx512;+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx512;+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx512;+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx512;+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_amd_avx512;+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_amd_avx512;+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_amd_avx512;+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_amd_avx512;+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_amd_avx512;+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_amd_avx512;+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_amd_avx512;+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_amd_avx512;+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_amd_avx512;+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_amd_avx512;+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_amd_avx512;+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_amd_avx512;+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_amd_avx512;+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_amd_avx512;+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_amd_avx512;+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_amd_avx512;+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_amd_avx512;+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_amd_avx512;+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_amd_avx512;+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_amd_avx512;+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_amd_avx512;+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_amd_avx512;+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_amd_avx512;+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_amd_avx512;+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_amd_avx512;+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_amd_avx512;+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_amd_avx512;+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_amd_avx512;+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_amd_avx512;+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_amd_avx512;+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_amd_avx512;+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_amd_avx512;+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_amd_avx512;+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_amd_avx512;+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_amd_avx512;+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_amd_avx512;+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_amd_avx512;+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_amd_avx512;+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_amd_avx512;+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_amd_avx512;+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_avx512;+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_avx512;+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_avx512;+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx512;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx512;+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx512;+ return;+ }+#endif+#ifdef __x86_64__+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2)) {+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_avx2;+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx2;+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx2;+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx2;+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx2;+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx2;+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx2;+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_amd_avx2;+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_amd_avx2;+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_amd_avx2;+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_amd_avx2;+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_amd_avx2;+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_amd_avx2;+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_amd_avx2;+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_amd_avx2;+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_amd_avx2;+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_amd_avx2;+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_amd_avx2;+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_amd_avx2;+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_amd_avx2;+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_amd_avx2;+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_amd_avx2;+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_amd_avx2;+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_amd_avx2;+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_amd_avx2;+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_amd_avx2;+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_amd_avx2;+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_amd_avx2;+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_amd_avx2;+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_amd_avx2;+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_amd_avx2;+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_amd_avx2;+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_amd_avx2;+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_amd_avx2;+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_amd_avx2;+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_amd_avx2;+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_amd_avx2;+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_amd_avx2;+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_amd_avx2;+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_amd_avx2;+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_amd_avx2;+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_amd_avx2;+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_amd_avx2;+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_amd_avx2;+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_amd_avx2;+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_amd_avx2;+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_amd_avx2;+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_avx2;+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_avx2;+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_avx2;+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx2;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx2;+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx2;+ return;+ }+#endif+#ifdef __x86_64__+ if (X86_HAVE(F16C)) {+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_f16c;+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_f16c;+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_f16c;+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_f16c;+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_f16c;+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_f16c;+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_f16c;+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_amd_f16c;+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_amd_f16c;+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_amd_f16c;+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_amd_f16c;+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_amd_f16c;+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_amd_f16c;+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_amd_f16c;+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_amd_f16c;+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_amd_f16c;+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_amd_f16c;+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_amd_f16c;+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_amd_f16c;+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_amd_f16c;+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_amd_f16c;+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_amd_f16c;+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_amd_f16c;+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_amd_f16c;+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_amd_f16c;+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_amd_f16c;+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_amd_f16c;+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_amd_f16c;+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_amd_f16c;+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_amd_f16c;+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_amd_f16c;+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_amd_f16c;+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_amd_f16c;+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_amd_f16c;+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_amd_f16c;+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_amd_f16c;+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_amd_f16c;+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_amd_f16c;+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_amd_f16c;+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_amd_f16c;+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_amd_f16c;+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_amd_f16c;+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_amd_f16c;+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_amd_f16c;+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_amd_f16c;+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_amd_f16c;+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_amd_f16c;+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_f16c;+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_f16c;+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_f16c;+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_f16c;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_f16c;+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_f16c;+ return;+ }+#endif+#ifdef __x86_64__+ if (X86_HAVE(FMA)) {+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_fma;+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_fma;+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_fma;+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_fma;+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_fma;+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_fma;+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_fma;+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_amd_fma;+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_amd_fma;+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_amd_fma;+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_amd_fma;+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_amd_fma;+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_amd_fma;+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_amd_fma;+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_amd_fma;+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_amd_fma;+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_amd_fma;+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_amd_fma;+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_amd_fma;+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_amd_fma;+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_amd_fma;+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_amd_fma;+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_amd_fma;+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_amd_fma;+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_amd_fma;+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_amd_fma;+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_amd_fma;+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_amd_fma;+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_amd_fma;+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_amd_fma;+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_amd_fma;+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_amd_fma;+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_amd_fma;+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_amd_fma;+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_amd_fma;+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_amd_fma;+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_amd_fma;+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_amd_fma;+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_amd_fma;+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_amd_fma;+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_amd_fma;+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_amd_fma;+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_amd_fma;+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_amd_fma;+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_amd_fma;+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_amd_fma;+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_amd_fma;+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_fma;+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_fma;+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_fma;+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_fma;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_fma;+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_fma;+ return;+ }+#endif+#ifdef __x86_64__+ if (1) {+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_avx;+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx;+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx;+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx;+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx;+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx;+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx;+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_amd_avx;+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_amd_avx;+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_amd_avx;+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_amd_avx;+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_amd_avx;+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_amd_avx;+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_amd_avx;+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_amd_avx;+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_amd_avx;+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_amd_avx;+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_amd_avx;+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_amd_avx;+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_amd_avx;+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_amd_avx;+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_amd_avx;+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_amd_avx;+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_amd_avx;+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_amd_avx;+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_amd_avx;+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_amd_avx;+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_amd_avx;+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_amd_avx;+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_amd_avx;+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_amd_avx;+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_amd_avx;+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_amd_avx;+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_amd_avx;+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_amd_avx;+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_amd_avx;+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_amd_avx;+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_amd_avx;+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_amd_avx;+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_amd_avx;+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_amd_avx;+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_amd_avx;+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_amd_avx;+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_amd_avx;+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_amd_avx;+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_amd_avx;+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_amd_avx;+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_avx;+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_avx;+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_avx;+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx;+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx;+ return;+ }+#endif+#ifdef __aarch64__+ if ((getauxval(AT_HWCAP) & HWCAP_FPHP) && (getauxval(AT_HWCAP) & HWCAP_ASIMDHP)) {+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_arm82;+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_arm82;+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_arm82;+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_arm82;+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_arm82;+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_arm82;+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_arm82;+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_arm82;+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_arm82;+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_arm82;+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_arm82;+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_arm82;+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_arm82;+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_arm82;+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_arm82;+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_arm82;+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_arm82;+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_arm82;+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_arm82;+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_arm82;+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_arm82;+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_arm82;+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_arm82;+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_arm82;+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_arm82;+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_arm82;+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_arm82;+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_arm82;+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_arm82;+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_arm82;+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_arm82;+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_arm82;+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_arm82;+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_arm82;+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_arm82;+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_arm82;+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_arm82;+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_arm82;+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_arm82;+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_arm82;+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_arm82;+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_arm82;+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_arm82;+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_arm82;+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_arm82;+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_arm82;+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_arm82;+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_arm82;+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_arm82;+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_arm82;+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_arm82;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_arm82;+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_arm82;+ return;+ }+#endif+#ifdef __aarch64__+ if (1) {+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_arm80;+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_arm80;+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_arm80;+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_arm80;+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_arm80;+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_arm80;+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_arm80;+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_arm80;+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_arm80;+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_arm80;+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_arm80;+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_arm80;+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_arm80;+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_arm80;+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_arm80;+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_arm80;+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_arm80;+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_arm80;+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_arm80;+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_arm80;+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_arm80;+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_arm80;+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_arm80;+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_arm80;+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_arm80;+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_arm80;+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_arm80;+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_arm80;+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_arm80;+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_arm80;+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_arm80;+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_arm80;+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_arm80;+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_arm80;+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_arm80;+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_arm80;+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_arm80;+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_arm80;+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_arm80;+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_arm80;+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_arm80;+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_arm80;+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_arm80;+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_arm80;+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_arm80;+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_arm80;+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_arm80;+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_arm80;+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_arm80;+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_arm80;+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_arm80;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_arm80;+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_arm80;+ return;+ }+#endif+ }+} funcs;++void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) {+ return funcs.ptr_ggml_fp16_to_fp32_row(x, y, n);+}++void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) {+ return funcs.ptr_ggml_fp32_to_fp16_row(x, y, n);+}++void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {+ return funcs.ptr_ggml_bf16_to_fp32_row(x, y, n);+}++void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {+ return funcs.ptr_ggml_fp32_to_bf16_row(x, y, n);+}++void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_f32(n, s, bs, x, bx, y, by, nrc);+}++void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_f16(n, s, bs, x, bx, y, by, nrc);+}++void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc) {+ return funcs.ptr_ggml_vec_dot_bf16(n, s, bs, x, bx, y, by, nrc);+}++void ggml_vec_dot_f16_unroll(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y) {+ return funcs.ptr_ggml_vec_dot_f16_unroll(n, xs, s, xv, y);+}++void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv) {+ return funcs.ptr_ggml_vec_mad_f32_unroll(n, xs, vs, y, xv, vv);+}++void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v) {+ return funcs.ptr_ggml_vec_set_i8(n, x, v);+}++void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v) {+ return funcs.ptr_ggml_vec_set_i16(n, x, v);+}++void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v) {+ return funcs.ptr_ggml_vec_set_i32(n, x, v);+}++void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const int32_t v) {+ return funcs.ptr_ggml_vec_set_f16(n, x, v);+}++void ggml_vec_set_bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v) {+ return funcs.ptr_ggml_vec_set_bf16(n, x, v);+}++void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) {+ return funcs.ptr_ggml_vec_add_f32(n, z, x, y);+}++void ggml_vec_add1_f32(const int n, float * z, const float * x, const float v) {+ return funcs.ptr_ggml_vec_add1_f32(n, z, x, v);+}++void ggml_vec_acc_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_acc_f32(n, y, x);+}++void ggml_vec_acc1_f32(const int n, float * y, const float v) {+ return funcs.ptr_ggml_vec_acc1_f32(n, y, v);+}++void ggml_vec_sub_f32 (const int n, float * z, const float * x, const float * y) {+ return funcs.ptr_ggml_vec_sub_f32(n, z, x, y);+}++void ggml_vec_set_f32 (const int n, float * x, const float v) {+ return funcs.ptr_ggml_vec_set_f32(n, x, v);+}++void ggml_vec_cpy_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_cpy_f32(n, y, x);+}++void ggml_vec_neg_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_neg_f32(n, y, x);+}++void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y) {+ return funcs.ptr_ggml_vec_mul_f32(n, z, x, y);+}++void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) {+ return funcs.ptr_ggml_vec_div_f32(n, z, x, y);+}++void ggml_vec_scale_f32(const int n, float * y, const float v) {+ return funcs.ptr_ggml_vec_scale_f32(n, y, v);+}++void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {+ return funcs.ptr_ggml_vec_scale_f16(n, y, v);+}++void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v) {+ return funcs.ptr_ggml_vec_mad_f32(n, y, x, v);+}++void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v) {+ return funcs.ptr_ggml_vec_mad_f16(n, y, x, v);+}++void ggml_vec_norm_f32 (const int n, float * s, const float * x) {+ return funcs.ptr_ggml_vec_norm_f32(n, s, x);+}++void ggml_vec_sqr_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_sqr_f32(n, y, x);+}++void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_sqrt_f32(n, y, x);+}++void ggml_vec_log_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_log_f32(n, y, x);+}++void ggml_vec_abs_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_abs_f32(n, y, x);+}++void ggml_vec_sgn_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_sgn_f32(n, y, x);+}++void ggml_vec_step_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_step_f32(n, y, x);+}++void ggml_vec_tanh_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_tanh_f32(n, y, x);+}++void ggml_vec_elu_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_elu_f32(n, y, x);+}++void ggml_vec_relu_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_relu_f32(n, y, x);+}++void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) {+ return funcs.ptr_ggml_vec_leaky_relu_f32(n, y, x, ns);+}++void ggml_vec_hardswish_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_hardswish_f32(n, y, x);+}++void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_hardsigmoid_f32(n, y, x);+}++void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ return funcs.ptr_ggml_vec_gelu_f16(n, y, x);+}++void ggml_vec_gelu_f32(const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_gelu_f32(n, y, x);+}++void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_gelu_quick_f32(n, y, x);+}++void ggml_vec_silu_f32(const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_silu_f32(n, y, x);+}++float ggml_silu_backward_f32(float x, float dy) {+ return funcs.ptr_ggml_silu_backward_f32(x, dy);+}++void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, const float * dy) {+ return funcs.ptr_ggml_vec_silu_backward_f32(n, dx, x, dy);+}++void ggml_vec_sum_f32(const int n, float * s, const float * x) {+ return funcs.ptr_ggml_vec_sum_f32(n, s, x);+}++void ggml_vec_sum_f32_ggf(const int n, ggml_float * s, const float * x) {+ return funcs.ptr_ggml_vec_sum_f32_ggf(n, s, x);+}++void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x) {+ return funcs.ptr_ggml_vec_sum_f16_ggf(n, s, x);+}++void ggml_vec_max_f32(const int n, float * s, const float * x) {+ return funcs.ptr_ggml_vec_max_f32(n, s, x);+}++void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) {+ return funcs.ptr_ggml_vec_norm_inv_f32(n, s, x);+}++void ggml_vec_argmax_f32(const int n, int * s, const float * x) {+ return funcs.ptr_ggml_vec_argmax_f32(n, s, x);+}+diff --git a/llama.cpp/ggml-vector.h b/llama.cpp/ggml-vector.hnew file mode 100644index 00000000..cdaff979--- /dev/null+++ b/llama.cpp/ggml-vector.h@@ -0,0 +1,67 @@+#pragma once+#include "ggml.h"+#ifdef __cplusplus+extern "C" {+#endif++typedef double ggml_float;++extern float *ggml_table_gelu_f16;++void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n);+void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n);+void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n);+void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n);+void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);+void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);+void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);+void ggml_vec_dot_f16_unroll(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);+void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);+void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v);+void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v);+void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v);+void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const int32_t v);+void ggml_vec_set_bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v);+void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y);+void ggml_vec_add1_f32(const int n, float * z, const float * x, const float v);+void ggml_vec_acc_f32 (const int n, float * y, const float * x);+void ggml_vec_acc1_f32(const int n, float * y, const float v);+void ggml_vec_sub_f32 (const int n, float * z, const float * x, const float * y);+void ggml_vec_set_f32 (const int n, float * x, const float v);+void ggml_vec_cpy_f32 (const int n, float * y, const float * x);+void ggml_vec_neg_f32 (const int n, float * y, const float * x);+void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y);+void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y);+void ggml_vec_scale_f32(const int n, float * y, const float v);+void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v);+void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v);+void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);+void ggml_vec_norm_f32 (const int n, float * s, const float * x);+void ggml_vec_sqr_f32 (const int n, float * y, const float * x);+void ggml_vec_sqrt_f32 (const int n, float * y, const float * x);+void ggml_vec_log_f32 (const int n, float * y, const float * x);+void ggml_vec_abs_f32 (const int n, float * y, const float * x);+void ggml_vec_sgn_f32 (const int n, float * y, const float * x);+void ggml_vec_step_f32 (const int n, float * y, const float * x);+void ggml_vec_tanh_f32 (const int n, float * y, const float * x);+void ggml_vec_elu_f32 (const int n, float * y, const float * x);+void ggml_vec_relu_f32 (const int n, float * y, const float * x);+void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns);+void ggml_vec_hardswish_f32 (const int n, float * y, const float * x);+void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x);+void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);+void ggml_vec_gelu_f32(const int n, float * y, const float * x);+void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x);+void ggml_vec_silu_f32(const int n, float * y, const float * x);+float ggml_silu_backward_f32(float x, float dy);+void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, const float * dy);+void ggml_vec_sum_f32(const int n, float * s, const float * x);+void ggml_vec_sum_f32_ggf(const int n, ggml_float * s, const float * x);+void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x);+void ggml_vec_max_f32(const int n, float * s, const float * x);+void ggml_vec_norm_inv_f32(const int n, float * s, const float * x);+void ggml_vec_argmax_f32(const int n, int * s, const float * x);++#ifdef __cplusplus+}+#endifdiff --git a/llama.cpp/ggml-vector.inc b/llama.cpp/ggml-vector.incnew file mode 100644index 00000000..244c63c2--- /dev/null+++ b/llama.cpp/ggml-vector.inc@@ -0,0 +1,1256 @@+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi++#include "ggml-vector.h"+#include "ggml-impl.h"+#include <stdatomic.h>++void ggml_once(atomic_uint *, void (*)(void));++#define UNUSED GGML_UNUSED++#define GGML_SOFT_MAX_UNROLL 4+#define GGML_VEC_DOT_UNROLL 2+#define GGML_VEC_MAD_UNROLL 32++#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)++#define GGML_SIMD++// F32 NEON++#define GGML_F32_STEP 16+#define GGML_F32_EPR 4++#define GGML_F32x4 float32x4_t+#define GGML_F32x4_ZERO vdupq_n_f32(0.0f)+#define GGML_F32x4_SET1(x) vdupq_n_f32(x)+#define GGML_F32x4_LOAD vld1q_f32+#define GGML_F32x4_STORE vst1q_f32+#define GGML_F32x4_FMA(a, b, c) vfmaq_f32(a, b, c)+#define GGML_F32x4_ADD vaddq_f32+#define GGML_F32x4_MUL vmulq_f32+#define GGML_F32x4_REDUCE_ONE(x) vaddvq_f32(x)+#define GGML_F32x4_REDUCE(res, x) \+{ \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vaddq_f32(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vaddq_f32(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vaddq_f32(x[i], x[offset+i]); \+ } \+ res = GGML_F32x4_REDUCE_ONE(x[0]); \+}++#define GGML_F32_VEC GGML_F32x4+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD+#define GGML_F32_VEC_STORE GGML_F32x4_STORE+#define GGML_F32_VEC_FMA GGML_F32x4_FMA+#define GGML_F32_VEC_ADD GGML_F32x4_ADD+#define GGML_F32_VEC_MUL GGML_F32x4_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE++// F16 NEON++#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)+ #define GGML_F16_STEP 32+ #define GGML_F16_EPR 8++ #define GGML_F16x8 float16x8_t+ #define GGML_F16x8_ZERO vdupq_n_f16(0.0f)+ #define GGML_F16x8_SET1(x) vdupq_n_f16(x)+ #define GGML_F16x8_LOAD(x) vld1q_f16((const ggml_fp16_internal_t *)(x))+ #define GGML_F16x8_STORE vst1q_f16+ #define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c)+ #define GGML_F16x8_ADD vaddq_f16+ #define GGML_F16x8_MUL vmulq_f16+ #define GGML_F16x8_REDUCE(res, x) \+ do { \+ int offset = GGML_F16_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vaddq_f16(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vaddq_f16(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vaddq_f16(x[i], x[offset+i]); \+ } \+ const float32x4_t t0 = vcvt_f32_f16(vget_low_f16 (x[0])); \+ const float32x4_t t1 = vcvt_f32_f16(vget_high_f16(x[0])); \+ res = (ggml_float) vaddvq_f32(vaddq_f32(t0, t1)); \+ } while (0)++ #define GGML_F16_VEC GGML_F16x8+ #define GGML_F16_VEC_ZERO GGML_F16x8_ZERO+ #define GGML_F16_VEC_SET1 GGML_F16x8_SET1+ #define GGML_F16_VEC_LOAD(p, i) GGML_F16x8_LOAD(p)+ #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((ggml_fp16_internal_t *)(p), r[i])+ #define GGML_F16_VEC_FMA GGML_F16x8_FMA+ #define GGML_F16_VEC_ADD GGML_F16x8_ADD+ #define GGML_F16_VEC_MUL GGML_F16x8_MUL+ #define GGML_F16_VEC_REDUCE GGML_F16x8_REDUCE+#else+ // if FP16 vector arithmetic is not supported, we use FP32 instead+ // and take advantage of the vcvt_ functions to convert to/from FP16++ #define GGML_F16_STEP 16+ #define GGML_F16_EPR 4++ #define GGML_F32Cx4 float32x4_t+ #define GGML_F32Cx4_ZERO vdupq_n_f32(0.0f)+ #define GGML_F32Cx4_SET1(x) vdupq_n_f32(x)+ #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16((const ggml_fp16_internal_t *)(x)))+ #define GGML_F32Cx4_STORE(x, y) vst1_f16(x, vcvt_f16_f32(y))+ #define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c)+ #define GGML_F32Cx4_ADD vaddq_f32+ #define GGML_F32Cx4_MUL vmulq_f32+ #define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE++ #define GGML_F16_VEC GGML_F32Cx4+ #define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO+ #define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1+ #define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)+ #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((ggml_fp16_internal_t *)(p), r[i])+ #define GGML_F16_VEC_FMA GGML_F32Cx4_FMA+ #define GGML_F16_VEC_ADD GGML_F32Cx4_ADD+ #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL+ #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE+#endif++#elif defined(__AVX512F__)++#define GGML_SIMD++// F32 AVX512++#define GGML_F32_STEP 64+#define GGML_F32_EPR 16++#define GGML_F32x16 __m512+#define GGML_F32x16_ZERO _mm512_setzero_ps()+#define GGML_F32x16_SET1(x) _mm512_set1_ps(x)+#define GGML_F32x16_LOAD _mm512_loadu_ps+#define GGML_F32x16_STORE _mm512_storeu_ps+// _mm512_fmadd_ps is defined in AVX512F so no guard is required+#define GGML_F32x16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)+#define GGML_F32x16_ADD _mm512_add_ps+#define GGML_F32x16_MUL _mm512_mul_ps+#define GGML_F32x16_REDUCE(res, x) \+do { \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ res = _mm512_reduce_add_ps(x[0]); \+} while (0)++// TODO: is this optimal ?++#define GGML_F32_VEC GGML_F32x16+#define GGML_F32_VEC_ZERO GGML_F32x16_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x16_SET1+#define GGML_F32_VEC_LOAD GGML_F32x16_LOAD+#define GGML_F32_VEC_STORE GGML_F32x16_STORE+#define GGML_F32_VEC_FMA GGML_F32x16_FMA+#define GGML_F32_VEC_ADD GGML_F32x16_ADD+#define GGML_F32_VEC_MUL GGML_F32x16_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x16_REDUCE++// F16 AVX512++// F16 AVX++#define GGML_F16_STEP 64+#define GGML_F16_EPR 16++// AVX512 has FP16 extension (AVX512_FP16) but I don't have it on my machine so I use FP32 instead++#define GGML_F32Cx16 __m512+#define GGML_F32Cx16_ZERO _mm512_setzero_ps()+#define GGML_F32Cx16_SET1(x) _mm512_set1_ps(x)++// unlike _mm256_cvt intrinsics that require F16C, _mm512_cvt is defined in AVX512F+// so F16C guard isn't required+#define GGML_F32Cx16_LOAD(x) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x)))+#define GGML_F32Cx16_STORE(x, y) _mm256_storeu_si256((__m256i *)(x), _mm512_cvtps_ph(y, 0))++#define GGML_F32Cx16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)+#define GGML_F32Cx16_ADD _mm512_add_ps+#define GGML_F32Cx16_MUL _mm512_mul_ps+#define GGML_F32Cx16_REDUCE(res, x) \+do { \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \+ } \+ res = _mm512_reduce_add_ps(x[0]); \+} while (0)++#define GGML_F16_VEC GGML_F32Cx16+#define GGML_F16_VEC_ZERO GGML_F32Cx16_ZERO+#define GGML_F16_VEC_SET1 GGML_F32Cx16_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx16_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx16_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F32Cx16_FMA+#define GGML_F16_VEC_ADD GGML_F32Cx16_ADD+#define GGML_F16_VEC_MUL GGML_F32Cx16_MUL+#define GGML_F16_VEC_REDUCE GGML_F32Cx16_REDUCE++#elif defined(__AVX__)++#define GGML_SIMD++// F32 AVX++#define GGML_F32_STEP 32+#define GGML_F32_EPR 8++#define GGML_F32x8 __m256+#define GGML_F32x8_ZERO _mm256_setzero_ps()+#define GGML_F32x8_SET1(x) _mm256_set1_ps(x)+#define GGML_F32x8_LOAD _mm256_loadu_ps+#define GGML_F32x8_STORE _mm256_storeu_ps+#if defined(__FMA__)+ #define GGML_F32x8_FMA(a, b, c) _mm256_fmadd_ps(b, c, a)+#else+ #define GGML_F32x8_FMA(a, b, c) _mm256_add_ps(_mm256_mul_ps(b, c), a)+#endif+#define GGML_F32x8_ADD _mm256_add_ps+#define GGML_F32x8_MUL _mm256_mul_ps+#define GGML_F32x8_REDUCE(res, x) \+do { \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \+ } \+ const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), \+ _mm256_extractf128_ps(x[0], 1)); \+ const __m128 t1 = _mm_hadd_ps(t0, t0); \+ res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); \+} while (0)+// TODO: is this optimal ?++#define GGML_F32_VEC GGML_F32x8+#define GGML_F32_VEC_ZERO GGML_F32x8_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x8_SET1+#define GGML_F32_VEC_LOAD GGML_F32x8_LOAD+#define GGML_F32_VEC_STORE GGML_F32x8_STORE+#define GGML_F32_VEC_FMA GGML_F32x8_FMA+#define GGML_F32_VEC_ADD GGML_F32x8_ADD+#define GGML_F32_VEC_MUL GGML_F32x8_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE++// F16 AVX++#define GGML_F16_STEP 32+#define GGML_F16_EPR 8++// F16 arithmetic is not supported by AVX, so we use F32 instead++#define GGML_F32Cx8 __m256+#define GGML_F32Cx8_ZERO _mm256_setzero_ps()+#define GGML_F32Cx8_SET1(x) _mm256_set1_ps(x)++#if defined(__F16C__)+// the _mm256_cvt intrinsics require F16C+#define GGML_F32Cx8_LOAD(x) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x)))+#define GGML_F32Cx8_STORE(x, y) _mm_storeu_si128((__m128i *)(x), _mm256_cvtps_ph(y, 0))+#else+static inline __m256 __avx_f32cx8_load(ggml_fp16_t *x) {+ float tmp[8];++ for (int i = 0; i < 8; i++) {+ tmp[i] = GGML_FP16_TO_FP32(x[i]);+ }++ return _mm256_loadu_ps(tmp);+}+static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {+ float arr[8];++ _mm256_storeu_ps(arr, y);++ for (int i = 0; i < 8; i++)+ x[i] = GGML_FP32_TO_FP16(arr[i]);+}+#define GGML_F32Cx8_LOAD(x) __avx_f32cx8_load((void *)(x))+#define GGML_F32Cx8_STORE(x, y) __avx_f32cx8_store(x, y)+#endif++#define GGML_F32Cx8_FMA GGML_F32x8_FMA+#define GGML_F32Cx8_ADD _mm256_add_ps+#define GGML_F32Cx8_MUL _mm256_mul_ps+#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE++#define GGML_F16_VEC GGML_F32Cx8+#define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO+#define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx8_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA+#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD+#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL+#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE++#elif defined(__POWER9_VECTOR__)++#define GGML_SIMD++// F32 POWER9++#define GGML_F32_STEP 32+#define GGML_F32_EPR 4++#define GGML_F32x4 vector float+#define GGML_F32x4_ZERO 0.0f+#define GGML_F32x4_SET1 vec_splats+#define GGML_F32x4_LOAD(p) vec_xl(0, p)+#define GGML_F32x4_STORE(p, r) vec_xst(r, 0, p)+#define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a)+#define GGML_F32x4_ADD vec_add+#define GGML_F32x4_MUL vec_mul+#define GGML_F32x4_REDUCE(res, x) \+{ \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vec_add(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vec_add(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = vec_add(x[i], x[offset+i]); \+ } \+ res = vec_extract(x[0], 0) + \+ vec_extract(x[0], 1) + \+ vec_extract(x[0], 2) + \+ vec_extract(x[0], 3); \+}++#define GGML_F32_VEC GGML_F32x4+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD+#define GGML_F32_VEC_STORE GGML_F32x4_STORE+#define GGML_F32_VEC_FMA GGML_F32x4_FMA+#define GGML_F32_VEC_ADD GGML_F32x4_ADD+#define GGML_F32_VEC_MUL GGML_F32x4_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE++// F16 POWER9+#define GGML_F16_STEP GGML_F32_STEP+#define GGML_F16_EPR GGML_F32_EPR+#define GGML_F16_VEC GGML_F32x4+#define GGML_F16_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F16_VEC_SET1 GGML_F32x4_SET1+#define GGML_F16_VEC_FMA GGML_F32x4_FMA+#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE+// Use vec_xl, not vec_ld, in case the load address is not aligned.+#define GGML_F16_VEC_LOAD(p, i) (i & 0x1) ? \+ vec_extract_fp32_from_shorth(vec_xl(0, p - GGML_F16_EPR)) : \+ vec_extract_fp32_from_shortl(vec_xl(0, p))+#define GGML_ENDIAN_BYTE(i) ((unsigned char *)&(uint16_t){1})[i]+#define GGML_F16_VEC_STORE(p, r, i) \+ if (i & 0x1) \+ vec_xst(vec_pack_to_short_fp32(r[i - GGML_ENDIAN_BYTE(1)], \+ r[i - GGML_ENDIAN_BYTE(0)]), \+ 0, p - GGML_F16_EPR)++#elif defined(__wasm_simd128__)++#define GGML_SIMD++// F32 WASM++#define GGML_F32_STEP 16+#define GGML_F32_EPR 4++#define GGML_F32x4 v128_t+#define GGML_F32x4_ZERO wasm_f32x4_splat(0.0f)+#define GGML_F32x4_SET1(x) wasm_f32x4_splat(x)+#define GGML_F32x4_LOAD wasm_v128_load+#define GGML_F32x4_STORE wasm_v128_store+#define GGML_F32x4_FMA(a, b, c) wasm_f32x4_add(wasm_f32x4_mul(b, c), a)+#define GGML_F32x4_ADD wasm_f32x4_add+#define GGML_F32x4_MUL wasm_f32x4_mul+#define GGML_F32x4_REDUCE(res, x) \+{ \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \+ } \+ res = wasm_f32x4_extract_lane(x[0], 0) + \+ wasm_f32x4_extract_lane(x[0], 1) + \+ wasm_f32x4_extract_lane(x[0], 2) + \+ wasm_f32x4_extract_lane(x[0], 3); \+}++#define GGML_F32_VEC GGML_F32x4+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD+#define GGML_F32_VEC_STORE GGML_F32x4_STORE+#define GGML_F32_VEC_FMA GGML_F32x4_FMA+#define GGML_F32_VEC_ADD GGML_F32x4_ADD+#define GGML_F32_VEC_MUL GGML_F32x4_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE++// F16 WASM++#define GGML_F16_STEP 16+#define GGML_F16_EPR 4++inline static v128_t __wasm_f16x4_load(const ggml_fp16_t * p) {+ float tmp[4];++ tmp[0] = GGML_FP16_TO_FP32(p[0]);+ tmp[1] = GGML_FP16_TO_FP32(p[1]);+ tmp[2] = GGML_FP16_TO_FP32(p[2]);+ tmp[3] = GGML_FP16_TO_FP32(p[3]);++ return wasm_v128_load(tmp);+}++inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {+ float tmp[4];++ wasm_v128_store(tmp, x);++ p[0] = GGML_FP32_TO_FP16(tmp[0]);+ p[1] = GGML_FP32_TO_FP16(tmp[1]);+ p[2] = GGML_FP32_TO_FP16(tmp[2]);+ p[3] = GGML_FP32_TO_FP16(tmp[3]);+}++#define GGML_F16x4 v128_t+#define GGML_F16x4_ZERO wasm_f32x4_splat(0.0f)+#define GGML_F16x4_SET1(x) wasm_f32x4_splat(x)+#define GGML_F16x4_LOAD(x) __wasm_f16x4_load(x)+#define GGML_F16x4_STORE(x, y) __wasm_f16x4_store(x, y)+#define GGML_F16x4_FMA GGML_F32x4_FMA+#define GGML_F16x4_ADD wasm_f32x4_add+#define GGML_F16x4_MUL wasm_f32x4_mul+#define GGML_F16x4_REDUCE(res, x) \+{ \+ int offset = GGML_F16_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \+ } \+ res = wasm_f32x4_extract_lane(x[0], 0) + \+ wasm_f32x4_extract_lane(x[0], 1) + \+ wasm_f32x4_extract_lane(x[0], 2) + \+ wasm_f32x4_extract_lane(x[0], 3); \+}++#define GGML_F16_VEC GGML_F16x4+#define GGML_F16_VEC_ZERO GGML_F16x4_ZERO+#define GGML_F16_VEC_SET1 GGML_F16x4_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F16x4_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F16x4_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F16x4_FMA+#define GGML_F16_VEC_ADD GGML_F16x4_ADD+#define GGML_F16_VEC_MUL GGML_F16x4_MUL+#define GGML_F16_VEC_REDUCE GGML_F16x4_REDUCE++#elif defined(__SSE3__)++#define GGML_SIMD++// F32 SSE++#define GGML_F32_STEP 32+#define GGML_F32_EPR 4++#define GGML_F32x4 __m128+#define GGML_F32x4_ZERO _mm_setzero_ps()+#define GGML_F32x4_SET1(x) _mm_set1_ps(x)+#define GGML_F32x4_LOAD _mm_loadu_ps+#define GGML_F32x4_STORE _mm_storeu_ps+#if defined(__FMA__)+ // TODO: Does this work?+ #define GGML_F32x4_FMA(a, b, c) _mm_fmadd_ps(b, c, a)+#else+ #define GGML_F32x4_FMA(a, b, c) _mm_add_ps(_mm_mul_ps(b, c), a)+#endif+#define GGML_F32x4_ADD _mm_add_ps+#define GGML_F32x4_MUL _mm_mul_ps+#define GGML_F32x4_REDUCE(res, x) \+{ \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm_add_ps(x[i], x[offset+i]); \+ } \+ const __m128 t0 = _mm_hadd_ps(x[0], x[0]); \+ res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t0, t0)); \+}+// TODO: is this optimal ?++#define GGML_F32_VEC GGML_F32x4+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD+#define GGML_F32_VEC_STORE GGML_F32x4_STORE+#define GGML_F32_VEC_FMA GGML_F32x4_FMA+#define GGML_F32_VEC_ADD GGML_F32x4_ADD+#define GGML_F32_VEC_MUL GGML_F32x4_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE++// F16 SSE++#define GGML_F16_STEP 32+#define GGML_F16_EPR 4++static inline __m128 __sse_f16x4_load(ggml_fp16_t *x) {+ float tmp[4];++ tmp[0] = GGML_FP16_TO_FP32(x[0]);+ tmp[1] = GGML_FP16_TO_FP32(x[1]);+ tmp[2] = GGML_FP16_TO_FP32(x[2]);+ tmp[3] = GGML_FP16_TO_FP32(x[3]);++ return _mm_loadu_ps(tmp);+}++static inline void __sse_f16x4_store(ggml_fp16_t *x, __m128 y) {+ float arr[4];++ _mm_storeu_ps(arr, y);++ x[0] = GGML_FP32_TO_FP16(arr[0]);+ x[1] = GGML_FP32_TO_FP16(arr[1]);+ x[2] = GGML_FP32_TO_FP16(arr[2]);+ x[3] = GGML_FP32_TO_FP16(arr[3]);+}++#define GGML_F32Cx4 __m128+#define GGML_F32Cx4_ZERO _mm_setzero_ps()+#define GGML_F32Cx4_SET1(x) _mm_set1_ps(x)+#define GGML_F32Cx4_LOAD(x) __sse_f16x4_load(x)+#define GGML_F32Cx4_STORE(x, y) __sse_f16x4_store(x, y)+#define GGML_F32Cx4_FMA GGML_F32x4_FMA+#define GGML_F32Cx4_ADD _mm_add_ps+#define GGML_F32Cx4_MUL _mm_mul_ps+#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE++#define GGML_F16_VEC GGML_F32Cx4+#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO+#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA+#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD+#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL+#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE++#endif++// GGML_F32_ARR / GGML_F16_ARR+// number of registers to use per step+#ifdef GGML_SIMD+#define GGML_F32_ARR (GGML_F32_STEP/GGML_F32_EPR)+#define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR)+#endif++void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) {+ for (int64_t i = 0; i < n; i++) {+ y[i] = GGML_FP16_TO_FP32(x[i]);+ }+}++void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) {+ int64_t i = 0;+#if defined(__F16C__)+ for (; i + 7 < n; i += 8) {+ __m256 x_vec = _mm256_loadu_ps(x + i);+ __m128i y_vec = _mm256_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);+ _mm_storeu_si128((__m128i *)(y + i), y_vec);+ }+ for(; i + 3 < n; i += 4) {+ __m128 x_vec = _mm_loadu_ps(x + i);+ __m128i y_vec = _mm_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);+ _mm_storel_epi64((__m128i *)(y + i), y_vec);+ }+#endif+ for (; i < n; i++) {+ y[i] = GGML_FP32_TO_FP16(x[i]);+ }+}++void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {+ int64_t i = 0;+#if defined(__AVX512F__)+ for (; i + 16 <= n; i += 16) {+ _mm512_storeu_ps(y + i,+ _mm512_castsi512_ps(+ _mm512_slli_epi32(+ _mm512_cvtepu16_epi32(+ _mm256_loadu_si256(+ (const __m256i *)(x + i))),+ 16)));+ }+#elif defined(__AVX2__)+ for (; i + 8 <= n; i += 8) {+ _mm256_storeu_ps(y + i,+ _mm256_castsi256_ps(+ _mm256_slli_epi32(+ _mm256_cvtepu16_epi32(+ _mm_loadu_si128(+ (const __m128i *)(x + i))),+ 16)));+ }+#endif+ for (; i < n; i++) {+ y[i] = GGML_BF16_TO_FP32(x[i]);+ }+}++void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {+ int i = 0;+#if defined(__AVX512BF16__)+ for (; i + 32 <= n; i += 32) {+ _mm512_storeu_ps(+ (__m512 *)(y + i),+ (__m512)_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16),+ _mm512_loadu_ps(x + i)));+ }+#endif+ for (; i < n; i++) {+ y[i] = GGML_FP32_TO_BF16(x[i]);+ }+}++//+// fundamental operations+//++void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v) { for (int i = 0; i < n; ++i) x[i] = v; }++void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }++void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }++void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }++void ggml_vec_set_bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }++void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] + y[i]; }+void ggml_vec_add1_f32(const int n, float * z, const float * x, const float v) { for (int i = 0; i < n; ++i) z[i] = x[i] + v; }+void ggml_vec_acc_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] += x[i]; }+void ggml_vec_acc1_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] += v; }+void ggml_vec_sub_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] - y[i]; }+void ggml_vec_set_f32 (const int n, float * x, const float v) { for (int i = 0; i < n; ++i) x[i] = v; }+void ggml_vec_cpy_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]; }+void ggml_vec_neg_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = -x[i]; }+void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]*y[i]; }+void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]/y[i]; }++void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++#if defined(GGML_SIMD)+ float sumf = 0.0f;+ const int np = (n & ~(GGML_F32_STEP - 1));++ GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };++ GGML_F32_VEC ax[GGML_F32_ARR];+ GGML_F32_VEC ay[GGML_F32_ARR];++ for (int i = 0; i < np; i += GGML_F32_STEP) {+ for (int j = 0; j < GGML_F32_ARR; j++) {+ ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);++ sum[j] = GGML_F32_VEC_FMA(sum[j], ax[j], ay[j]);+ }+ }++ // reduce sum0..sum3 to sum0+ GGML_F32_VEC_REDUCE(sumf, sum);++ // leftovers+ for (int i = np; i < n; ++i) {+ sumf += x[i]*y[i];+ }+#else+ // scalar+ ggml_float sumf = 0.0;+ for (int i = 0; i < n; ++i) {+ sumf += (ggml_float)(x[i]*y[i]);+ }+#endif++ *s = sumf;+}++void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);+ int i = 0;+ ggml_float sumf = 0;++#if defined(__AVX512BF16__)+ __m512 c1 = _mm512_setzero_ps();+ __m512 c2 = _mm512_setzero_ps();+ for (; i + 64 <= n; i += 64) {+ c1 = _mm512_dpbf16_ps(c1, (__m512bh)_mm512_loadu_ps((const float *)(x + i)),+ (__m512bh)_mm512_loadu_ps((const float *)(y + i)));+ c2 = _mm512_dpbf16_ps(c2, (__m512bh)_mm512_loadu_ps((const float *)(x + i + 32)),+ (__m512bh)_mm512_loadu_ps((const float *)(y + i + 32)));+ }+ sumf += (ggml_float)_mm512_reduce_add_ps(c1);+ sumf += (ggml_float)_mm512_reduce_add_ps(c2);++#elif defined(__AVX512F__)+#define LOAD(p) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)(p))), 16))+ __m512 c1 = _mm512_setzero_ps();+ __m512 c2 = _mm512_setzero_ps();+ for (; i + 32 <= n; i += 32) {+ c1 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i), LOAD(y + i)), c1);+ c2 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c2);+ }+ sumf += (ggml_float)_mm512_reduce_add_ps(c1);+ sumf += (ggml_float)_mm512_reduce_add_ps(c2);++#undef LOAD+#elif defined(__AVX2__)+#define LOAD(p) _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)(p))), 16))+ __m256 c1 = _mm256_setzero_ps();+ __m256 c2 = _mm256_setzero_ps();+ __m256 c3 = _mm256_setzero_ps();+ __m256 c4 = _mm256_setzero_ps();+ for (; i + 32 <= n; i += 32) {+ c1 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i), LOAD(y + i)), c1);+ c2 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 8), LOAD(y + i + 8)), c2);+ c3 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c3);+ c4 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 24), LOAD(y + i + 24)), c4);+ }+ __m128 g;+ c1 = _mm256_add_ps(_mm256_add_ps(c1, c3),+ _mm256_add_ps(c2, c4));+ g = _mm_add_ps(_mm256_extractf128_ps(c1, 1),+ _mm256_castps256_ps128(c1));+ g = _mm_add_ps(g, _mm_movehl_ps(g, g));+ g = _mm_add_ss(g, _mm_movehdup_ps(g));+ sumf += (ggml_float)_mm_cvtss_f32(g);++#undef LOAD+#endif++ for (; i < n; ++i) {+ sumf += (ggml_float)(GGML_BF16_TO_FP32(x[i]) *+ GGML_BF16_TO_FP32(y[i]));+ }+ *s = sumf;+}++void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ ggml_float sumf = 0.0;++#if defined(GGML_SIMD)+ const int np = (n & ~(GGML_F16_STEP - 1));++ GGML_F16_VEC sum[GGML_F16_ARR] = { GGML_F16_VEC_ZERO };++ GGML_F16_VEC ax[GGML_F16_ARR];+ GGML_F16_VEC ay[GGML_F16_ARR];++ for (int i = 0; i < np; i += GGML_F16_STEP) {+ for (int j = 0; j < GGML_F16_ARR; j++) {+ ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);++ sum[j] = GGML_F16_VEC_FMA(sum[j], ax[j], ay[j]);+ }+ }++ // reduce sum0..sum3 to sum0+ GGML_F16_VEC_REDUCE(sumf, sum);++ // leftovers+ for (int i = np; i < n; ++i) {+ sumf += (ggml_float)(GGML_FP16_TO_FP32(x[i])*GGML_FP16_TO_FP32(y[i]));+ }+#else+ for (int i = 0; i < n; ++i) {+ sumf += (ggml_float)(GGML_FP16_TO_FP32(x[i])*GGML_FP16_TO_FP32(y[i]));+ }+#endif++ *s = sumf;+}++// compute GGML_VEC_DOT_UNROLL dot products at once+// xs - x row stride in bytes+void ggml_vec_dot_f16_unroll(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y) {+ ggml_float sumf[GGML_VEC_DOT_UNROLL] = { 0.0 };++ ggml_fp16_t * restrict x[GGML_VEC_DOT_UNROLL];++ for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {+ x[i] = (ggml_fp16_t *) ((char *) xv + i*xs);+ }++#if defined(GGML_SIMD)+ const int np = (n & ~(GGML_F16_STEP - 1));++ GGML_F16_VEC sum[GGML_VEC_DOT_UNROLL][GGML_F16_ARR] = { { GGML_F16_VEC_ZERO } };++ GGML_F16_VEC ax[GGML_F16_ARR];+ GGML_F16_VEC ay[GGML_F16_ARR];++ for (int i = 0; i < np; i += GGML_F16_STEP) {+ for (int j = 0; j < GGML_F16_ARR; j++) {+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);++ for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {+ ax[j] = GGML_F16_VEC_LOAD(x[k] + i + j*GGML_F16_EPR, j);++ sum[k][j] = GGML_F16_VEC_FMA(sum[k][j], ax[j], ay[j]);+ }+ }+ }++ // reduce sum0..sum3 to sum0+ for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {+ GGML_F16_VEC_REDUCE(sumf[k], sum[k]);+ }++ // leftovers+ for (int i = np; i < n; ++i) {+ for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {+ sumf[j] += (ggml_float)(GGML_FP16_TO_FP32(x[j][i])*GGML_FP16_TO_FP32(y[i]));+ }+ }+#else+ for (int i = 0; i < n; ++i) {+ for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {+ sumf[j] += (ggml_float)(GGML_FP16_TO_FP32(x[j][i])*GGML_FP16_TO_FP32(y[i]));+ }+ }+#endif++ for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {+ s[i] = sumf[i];+ }+}++void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v) {+#if defined(GGML_SIMD)+ const int np = (n & ~(GGML_F32_STEP - 1));++ GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);++ GGML_F32_VEC ax[GGML_F32_ARR];+ GGML_F32_VEC ay[GGML_F32_ARR];++ for (int i = 0; i < np; i += GGML_F32_STEP) {+ for (int j = 0; j < GGML_F32_ARR; j++) {+ ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);+ ay[j] = GGML_F32_VEC_FMA(ay[j], ax[j], vx);++ GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);+ }+ }++ // leftovers+ for (int i = np; i < n; ++i) {+ y[i] += x[i]*v;+ }+#else+ // scalar+ for (int i = 0; i < n; ++i) {+ y[i] += x[i]*v;+ }+#endif+}++void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v) {+#if defined(GGML_SIMD)+ const int np = (n & ~(GGML_F16_STEP - 1));++ GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);++ GGML_F16_VEC ax[GGML_F16_ARR];+ GGML_F16_VEC ay[GGML_F16_ARR];++ for (int i = 0; i < np; i += GGML_F16_STEP) {+ for (int j = 0; j < GGML_F16_ARR; j++) {+ ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);+ ay[j] = GGML_F16_VEC_FMA(ay[j], ax[j], vx);++ GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);+ }+ }++ // leftovers+ for (int i = np; i < n; ++i) {+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i]) + GGML_FP16_TO_FP32(x[i])*v);+ }+#else+ // scalar+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i]) + GGML_FP16_TO_FP32(x[i])*v);+ }+#endif+}++// xs and vs are byte strides of x and v+void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv) {++ const float * restrict x[GGML_VEC_MAD_UNROLL];+ const float * restrict v[GGML_VEC_MAD_UNROLL];++ for (int i = 0; i < GGML_VEC_MAD_UNROLL; ++i) {+ x[i] = (const float *) ((const char *) xv + i*xs);+ v[i] = (const float *) ((const char *) vv + i*vs);+ }++#if defined(GGML_SIMD)+ const int np = (n & ~(GGML_F32_STEP - 1));++ GGML_F32_VEC vx[GGML_VEC_MAD_UNROLL];++ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {+ vx[k] = GGML_F32_VEC_SET1(v[k][0]);+ }++ GGML_F32_VEC ax[GGML_VEC_MAD_UNROLL][GGML_F32_ARR];+ GGML_F32_VEC ay[GGML_F32_ARR];++ for (int i = 0; i < np; i += GGML_F32_STEP) {+ for (int j = 0; j < GGML_F32_ARR; j++) {+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);++ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {+ ax[k][j] = GGML_F32_VEC_LOAD(x[k] + i + j*GGML_F32_EPR);+ ay[j] = GGML_F32_VEC_FMA(ay[j], ax[k][j], vx[k]);+ }++ GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);+ }+ }++ // leftovers+ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {+ for (int i = np; i < n; ++i) {+ y[i] += x[k][i]*v[k][0];+ }+ }+#else+ // scalar+ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {+ for (int i = 0; i < n; ++i) {+ y[i] += x[k][i]*v[k][0];+ }+ }+#endif+}++//inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] *= v; }+void ggml_vec_scale_f32(const int n, float * y, const float v) {+#if defined(GGML_USE_ACCELERATE)+ vDSP_vsmul(y, 1, &v, y, 1, n);+#elif defined(GGML_SIMD)+ const int np = (n & ~(GGML_F32_STEP - 1));++ GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);++ GGML_F32_VEC ay[GGML_F32_ARR];++ for (int i = 0; i < np; i += GGML_F32_STEP) {+ for (int j = 0; j < GGML_F32_ARR; j++) {+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);+ ay[j] = GGML_F32_VEC_MUL(ay[j], vx);++ GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);+ }+ }++ // leftovers+ for (int i = np; i < n; ++i) {+ y[i] *= v;+ }+#else+ // scalar+ for (int i = 0; i < n; ++i) {+ y[i] *= v;+ }+#endif+}++void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {+#if defined(GGML_SIMD)+ const int np = (n & ~(GGML_F16_STEP - 1));++ GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);++ GGML_F16_VEC ay[GGML_F16_ARR];++ for (int i = 0; i < np; i += GGML_F16_STEP) {+ for (int j = 0; j < GGML_F16_ARR; j++) {+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);+ ay[j] = GGML_F16_VEC_MUL(ay[j], vx);++ GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);+ }+ }++ // leftovers+ for (int i = np; i < n; ++i) {+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i])*v);+ }+#else+ // scalar+ for (int i = 0; i < n; ++i) {+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i])*v);+ }+#endif+}++void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, 0, x, 0, x, 0, 1); *s = sqrtf(*s); }+void ggml_vec_sqr_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i]; }+void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrtf(x[i]); }+void ggml_vec_log_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = logf(x[i]); }+void ggml_vec_abs_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fabsf(x[i]); }+void ggml_vec_sgn_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : ((x[i] < 0.f) ? -1.f : 0.f); }+void ggml_vec_step_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : 0.f; }+void ggml_vec_tanh_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = tanhf(x[i]); }+void ggml_vec_elu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : expf(x[i])-1; }+void ggml_vec_relu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.f; }+void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); }+// TODO: optimize performance+void ggml_vec_hardswish_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }+void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }++// ACTIVATION FUNCTIONS+//+// 0.3 +--------------------------------------------------------------------------------------++// | + + + + + + + * % |+// | x>0?x:0.01 **%**** |+// | x/(1+exp(-x)) #%##### |+// | .5*x*(1+erf(x/sqrt(2)))*$%$$$$$ |+// 0.2 |-+ x*(1/(1+exp(-1.702*x)))*%%%%%%%-|+// | *%# |+// | *% |+// | *% |+// | *% |+// 0.1 |-+ *% +-|+// | *% |+// | % |+// | *% |+// | % |+// 0 |***************************************************************************% +-|+// |%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%$$$$$ % |+// | ########## %%%%%$$$$ % |+// | ##### %%%%$$ % |+// | #### %%$$ % |+// | #### %%%$ % |+// -0.1 |-+ ### %%% %# +-|+// | ### % %# |+// | ### %%% %%# |+// | ## $%%%%$ # |+// | ### # |+// -0.2 |-+ ## # +-|+// | ### # |+// | ## # |+// | #### ### |+// | + + + + + #####+ + |+// -0.3 +--------------------------------------------------------------------------------------++// -7 -6 -5 -4 -3 -2 -1 0 1+//++inline static float ggml_silu_f32(float x) {+ // SiLU is the favored by LLaMA, Mistral, Phi, Rocket, etc.+ return x/(1.f + expf(-x));+}++inline static float ggml_gelu_f32(float x) {+ // GeLU approximation that goes slower and we seem to be stuck with.+ return .5f * x * (1.f + tanhf(sqrtf(M_2_PI) * (x + .044715f * x * x * x)));+}++inline static float ggml_gelu_quick_f32(float x) {+ // "Quick" GeLU is used by LLaVA's CLIP vision model.+ return x*(1.f/(1.f+expf(-1.702f*x)));+}++static void ggml_vec_gelu_f16_init(void) {+ ggml_table_gelu_f16 = malloc(sizeof(float) * 65536);+ for (int i = 0; i < 65536; ++i) {+ union {+ unsigned short i;+ ggml_fp16_t f;+ } u = {i};+ ggml_table_gelu_f16[i] = ggml_gelu_f32(u.f);+ }+}++void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {+ static atomic_uint once;+ ggml_once(&once, ggml_vec_gelu_f16_init);+ const uint16_t * i16 = (const uint16_t *) x;+ for (int i = 0; i < n; ++i) {+ y[i] = ggml_table_gelu_f16[i16[i]];+ }+}++void ggml_vec_gelu_f32(const int n, float * y, const float * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = ggml_gelu_f32(x[i]);+ }+}++void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = ggml_gelu_quick_f32(x[i]);+ }+}++void ggml_vec_silu_f32(const int n, float * y, const float * x) {+ for (int i = 0; i < n; ++i) {+ y[i] = ggml_silu_f32(x[i]);+ }+}++float ggml_silu_backward_f32(float x, float dy) {+ const float s = 1.0f/(1.0f + expf(-x));+ return dy*s*(1.0f + x*(1.0f - s));+}++void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, const float * dy) {+ for (int i = 0; i < n; ++i) {+ dx[i] = ggml_silu_backward_f32(x[i], dy[i]);+ }+}++void ggml_vec_sum_f32(const int n, float * s, const float * x) {+#ifndef GGML_USE_ACCELERATE+ ggml_float sum = 0.0;+ for (int i = 0; i < n; ++i) {+ sum += (ggml_float)x[i];+ }+ *s = sum;+#else+ vDSP_sve(x, 1, s, n);+#endif+}++void ggml_vec_sum_f32_ggf(const int n, ggml_float * s, const float * x) {+ ggml_float sum = 0.0;+ for (int i = 0; i < n; ++i) {+ sum += (ggml_float)x[i];+ }+ *s = sum;+}++void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x) {+ float sum = 0.0f;+ for (int i = 0; i < n; ++i) {+ sum += GGML_FP16_TO_FP32(x[i]);+ }+ *s = sum;+}++void ggml_vec_max_f32(const int n, float * s, const float * x) {+#ifndef GGML_USE_ACCELERATE+ float max = -INFINITY;+ for (int i = 0; i < n; ++i) {+ max = MAX(max, x[i]);+ }+ *s = max;+#else+ vDSP_maxv(x, 1, s, n);+#endif+}++void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) {+ ggml_vec_norm_f32(n, s, x);+ *s = 1.f/(*s);+}++void ggml_vec_argmax_f32(const int n, int * s, const float * x) {+ float max = -INFINITY;+ int idx = 0;+ for (int i = 0; i < n; ++i) {+ max = MAX(max, x[i]);+ if (max == x[i]) { idx = i; }+ }+ *s = idx;+}diff --git a/llama.cpp/ggml-vector.py b/llama.cpp/ggml-vector.pynew file mode 100644index 00000000..528bf60d--- /dev/null+++ b/llama.cpp/ggml-vector.py@@ -0,0 +1,67 @@+import re++with open('llama.cpp/ggml-vector.h') as f:+ prototypes = f.read()+prototypes = [line.replace(';', '') for line in prototypes.split('\n')+ if line.endswith(';') and not line.startswith('//')]+FUNCS = [(re.search(r'(?<= )\w+', proto).group(0), proto)+ for proto in prototypes]++# BEGIN SPECIAL FUNCTIONS+# FUNCS.append(('ggml_validate_row_data', 'bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes)'))+# END SPECIAL FUNCTIONS++ARCHS = (+ ('amd_avx512bf16', '__x86_64__', ('X86_HAVE(FMA)', 'X86_HAVE(F16C)', 'X86_HAVE(AVX2)', 'X86_HAVE(AVX512F)', 'X86_HAVE(AVX512VL)', 'X86_HAVE(AVX512_BF16)')),+ ('amd_avx512', '__x86_64__', ('X86_HAVE(FMA)', 'X86_HAVE(F16C)', 'X86_HAVE(AVX2)', 'X86_HAVE(AVX512F)')),+ ('amd_avx2', '__x86_64__', ('X86_HAVE(FMA)', 'X86_HAVE(F16C)', 'X86_HAVE(AVX2)')),+ ('amd_f16c', '__x86_64__', ('X86_HAVE(F16C)',)),+ ('amd_fma', '__x86_64__', ('X86_HAVE(FMA)',)),+ ('amd_avx', '__x86_64__', ()),+ ('arm82', '__aarch64__', ('(getauxval(AT_HWCAP) & HWCAP_FPHP)', '(getauxval(AT_HWCAP) & HWCAP_ASIMDHP)')),+ ('arm80', '__aarch64__', ()),+)++for arch, mac, needs in ARCHS:+ path = 'llama.cpp/ggml-vector-%s.c' % (arch.replace('_', '-'))+ with open(path, 'w') as f:+ f.write('#ifdef %s\n' % (mac))+ for func, proto in FUNCS:+ f.write('#define %s %s_%s\n' % (func, func, arch))+ f.write('#define GGML_VECTOR\n')+ f.write('#include "ggml-vector.inc"\n')+ f.write('#endif // %s\n' % (mac))++with open('llama.cpp/ggml-vector.cpp', 'w') as f:+ f.write('#include <cosmo.h>\n')+ f.write('#include <sys/auxv.h>\n')+ f.write('#include <libc/sysv/consts/hwcap.h>\n')+ f.write('#include "ggml-vector.h"\n')+ f.write('\n')+ for func, proto in FUNCS:+ for arch, mac, needs in ARCHS:+ f.write('extern "C" %s;\n' % (proto.replace(func, '%s_%s' % (func, arch))))+ f.write('\n')+ f.write('static const struct VectorFuncs {\n')+ for func, proto in FUNCS:+ f.write(' typeof(%s) *ptr_%s;\n' % (func, func))+ f.write('\n')+ f.write(' VectorFuncs() {\n')+ for arch, mac, needs in ARCHS:+ f.write('#ifdef %s\n' % (mac))+ f.write(' if (%s) {\n' % (' && '.join(needs) or '1'))+ for func, proto in FUNCS:+ f.write(' ptr_%s = %s_%s;\n' % (func, func, arch))+ f.write(' return;\n')+ f.write(' }\n')+ f.write('#endif\n')+ f.write(' }\n')+ f.write('} funcs;\n')+ f.write('\n')+ for func, proto in FUNCS:+ proto = proto.replace(';', '')+ args = [s.split(' ')[-1] for s in re.search(r'(?<=\().*(?=\))', proto).group(0).split(',')]+ f.write(proto + ' {\n')+ f.write(' return funcs.ptr_%s(%s);\n' % (func, ', '.join(args)))+ f.write('}\n')+ f.write('\n')diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 9970c813..f4012a25 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -35,6 +35,7 @@ SOFTWARE.");#include "ggml-quants.h"#include "ggml-metal.h"#include "ggml-cuda.h"+#include "ggml-vector.h"#include "llamafile/llamafile.h"#include "llamafile/log.h"@@ -117,11 +118,6 @@ void ggml_print_backtrace(void) {/*#define GGML_PERF*/#define GGML_DEBUG 0-#define GGML_KAHAN 0--#define GGML_SOFT_MAX_UNROLL 4-#define GGML_VEC_DOT_UNROLL 2-#define GGML_VEC_MAD_UNROLL 32//// logging@@ -246,7 +242,7 @@ typedef double ggml_float;// global data//-static float *ggml_table_gelu_f16;+float *ggml_table_gelu_f16;// precomputed f32 table for f16 (256 KB) (ggml-impl.h)float ggml_table_f32_f16[1 << 16];@@ -262,147 +258,24 @@ GGML_CALL const char * ggml_status_to_string(enum ggml_status status) {return "GGML status: unknown";}-// note: do not use these inside ggml.c-// these are meant to be used via the ggml.h APIfloat ggml_fp16_to_fp32(ggml_fp16_t x) {+#define ggml_fp16_to_fp32 do_not_use__ggml_fp16_to_fp32__in_ggmlreturn GGML_FP16_TO_FP32(x);}ggml_fp16_t ggml_fp32_to_fp16(float x) {+#define ggml_fp32_to_fp16 do_not_use__ggml_fp32_to_fp16__in_ggmlreturn GGML_FP32_TO_FP16(x);}-void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) {- for (int64_t i = 0; i < n; i++) {- y[i] = GGML_FP16_TO_FP32(x[i]);- }-}--#ifdef __x86_64__-#pragma GCC push_options-#pragma GCC target("avx512f")-static void ggml_bf16_to_fp32_row_avx512f(const ggml_bf16_t * x, float * y, int64_t n) {- int64_t i = 0;- for (; i + 16 <= n; i += 16) {- _mm512_storeu_ps(y + i,- _mm512_castsi512_ps(- _mm512_slli_epi32(- _mm512_cvtepu16_epi32(- _mm256_loadu_si256(- (const __m256i *)(x + i))),- 16)));- }- for (; i < n; i++) {- y[i] = GGML_BF16_TO_FP32(x[i]);- }-}-#pragma GCC pop_options-#endif--#ifdef __x86_64__-#pragma GCC push_options-#pragma GCC target("avx2")-static void ggml_bf16_to_fp32_row_avx2(const ggml_bf16_t * x, float * y, int64_t n) {- int64_t i = 0;- for (; i + 8 <= n; i += 8) {- _mm256_storeu_ps(y + i,- _mm256_castsi256_ps(- _mm256_slli_epi32(- _mm256_cvtepu16_epi32(- _mm_loadu_si128(- (const __m128i *)(x + i))),- 16)));- }- for (; i < n; i++) {- y[i] = GGML_BF16_TO_FP32(x[i]);- }-}-#pragma GCC pop_options-#endif--void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {-#ifdef __x86_64__- if (X86_HAVE(AVX512F))- return ggml_bf16_to_fp32_row_avx512f(x, y, n);- if (X86_HAVE(AVX2))- return ggml_bf16_to_fp32_row_avx2(x, y, n);-#endif- for (int i = 0; i < n; i++) {- y[i] = GGML_BF16_TO_FP32(x[i]);- }-}--#ifdef __x86_64__-#pragma GCC push_options-#pragma GCC target("f16c")-static void ggml_fp32_to_fp16_row_f16c(const float * x, ggml_fp16_t * y, int64_t n) {- int64_t i = 0;- for (; i + 7 < n; i += 8) {- __m256 x_vec = _mm256_loadu_ps(x + i);- __m128i y_vec = _mm256_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);- _mm_storeu_si128((__m128i *)(y + i), y_vec);- }- for(; i + 3 < n; i += 4) {- __m128 x_vec = _mm_loadu_ps(x + i);- __m128i y_vec = _mm_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);- _mm_storel_epi64((__m128i *)(y + i), y_vec);- }- for (; i < n; i++) {- y[i] = GGML_FP32_TO_FP16(x[i]);- }-}-#pragma GCC pop_options-#endif--#ifdef __aarch64__-static void ggml_fp32_to_fp16_row_neon(const float *x, ggml_fp16_t *y, int64_t n) {- int64_t i = 0;- for (; i + 3 < n; i += 4) {- vst1_f16((ggml_fp16_t *)(y + i), vcvt_f16_f32(vld1q_f32(x + i)));- }- for (; i < n; i++) {- y[i] = GGML_FP32_TO_FP16(x[i]);- }-}-#endif--void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) {-#ifdef __x86_64__- if (X86_HAVE(F16C)) {- return ggml_fp32_to_fp16_row_f16c(x, y, n);- }-#elif defined(__aarch64__)- return ggml_fp32_to_fp16_row_neon(x, y, n);-#endif- for (int64_t i = 0; i < n; i++) {- y[i] = GGML_FP32_TO_FP16(x[i]);- }+float ggml_bf16_to_fp32(ggml_bf16_t x) {+#define ggml_bf16_to_fp32 do_not_use__ggml_bf16_to_fp32__in_ggml+ return GGML_BF16_TO_FP32(x); // it just left shifts}-#ifdef __x86_64__-#pragma GCC push_options-#pragma GCC target("avx512bf16")-static void ggml_fp32_to_bf16_row_avx512bf16(const float * x, ggml_bf16_t * y, int64_t n) {- int64_t i = 0;- for (; i + 32 <= n; i += 32)- _mm512_storeu_ps(- (__m512 *)(y + i),- (__m512)_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16),- _mm512_loadu_ps(x + i)));- for (; i < n; i++)- y[i] = GGML_FP32_TO_BF16(x[i]);-}-#pragma GCC pop_options-#endif--void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {-#ifdef __x86_64__- if (X86_HAVE(AVX512_BF16))- return ggml_fp32_to_bf16_row_avx512bf16(x, y, n);-#endif- for (int64_t i = 0; i < n; i++) {- y[i] = GGML_FP32_TO_BF16(x[i]);- }+ggml_bf16_t ggml_fp32_to_bf16(float x) {+#define ggml_fp32_to_bf16 do_not_use__ggml_fp32_to_bf16__in_ggml+ return GGML_FP32_TO_BF16(x);}GGML_CALL bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b) {@@ -471,10 +344,6 @@ FILE * ggml_fopen(const char * fname, const char * mode) {static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float);-static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);-static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);-static void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);-static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {[GGML_TYPE_I8] = {.type_name = "i8",@@ -852,25 +721,9 @@ ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type) {// number of elements to fit in a single register//-#define GGML_F32x8_REDUCE_AVX(res, x) \-do { \- int offset = (16/4) >> 1; \- for (int i = 0; i < offset; ++i) { \- x[i] = _mm256_add_ps(x[i], x[offset+i]); \- } \- offset >>= 1; \- for (int i = 0; i < offset; ++i) { \- x[i] = _mm256_add_ps(x[i], x[offset+i]); \- } \- offset >>= 1; \- for (int i = 0; i < offset; ++i) { \- x[i] = _mm256_add_ps(x[i], x[offset+i]); \- } \- const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), \- _mm256_extractf128_ps(x[0], 1)); \- const __m128 t1 = _mm_hadd_ps(t0, t0); \- res = _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); \-} while (0)+#define GGML_SOFT_MAX_UNROLL 4+#define GGML_VEC_DOT_UNROLL 2+#define GGML_VEC_MAD_UNROLL 32#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)@@ -888,7 +741,6 @@ do { \#define GGML_F32x4_STORE vst1q_f32#define GGML_F32x4_FMA(a, b, c) vfmaq_f32(a, b, c)#define GGML_F32x4_ADD vaddq_f32-#define GGML_F32x4_SUB vsubq_f32#define GGML_F32x4_MUL vmulq_f32#define GGML_F32x4_REDUCE_ONE(x) vaddvq_f32(x)#define GGML_F32x4_REDUCE(res, x) \@@ -915,7 +767,6 @@ do { \#define GGML_F32_VEC_STORE GGML_F32x4_STORE#define GGML_F32_VEC_FMA GGML_F32x4_FMA#define GGML_F32_VEC_ADD GGML_F32x4_ADD-#define GGML_F32_VEC_SUB GGML_F32x4_SUB#define GGML_F32_VEC_MUL GGML_F32x4_MUL#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE@@ -932,7 +783,6 @@ do { \#define GGML_F16x8_STORE vst1q_f16#define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c)#define GGML_F16x8_ADD vaddq_f16- #define GGML_F16x8_SUB vsubq_f16#define GGML_F16x8_MUL vmulq_f16#define GGML_F16x8_REDUCE(res, x) \do { \@@ -960,7 +810,6 @@ do { \#define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((ggml_fp16_internal_t *)(p), r[i])#define GGML_F16_VEC_FMA GGML_F16x8_FMA#define GGML_F16_VEC_ADD GGML_F16x8_ADD- #define GGML_F16_VEC_SUB GGML_F16x8_SUB#define GGML_F16_VEC_MUL GGML_F16x8_MUL#define GGML_F16_VEC_REDUCE GGML_F16x8_REDUCE#else@@ -977,7 +826,6 @@ do { \#define GGML_F32Cx4_STORE(x, y) vst1_f16(x, vcvt_f16_f32(y))#define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c)#define GGML_F32Cx4_ADD vaddq_f32- #define GGML_F32Cx4_SUB vsubq_f32#define GGML_F32Cx4_MUL vmulq_f32#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE@@ -988,7 +836,6 @@ do { \#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((ggml_fp16_internal_t *)(p), r[i])#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD- #define GGML_F16_VEC_SUB GGML_F32Cx4_SUB#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE#endif@@ -1010,7 +857,6 @@ do { \// _mm512_fmadd_ps is defined in AVX512F so no guard is required#define GGML_F32x16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)#define GGML_F32x16_ADD _mm512_add_ps-#define GGML_F32x16_SUB _mm512_sub_ps#define GGML_F32x16_MUL _mm512_mul_ps#define GGML_F32x16_REDUCE(res, x) \do { \@@ -1038,7 +884,6 @@ do { \#define GGML_F32_VEC_STORE GGML_F32x16_STORE#define GGML_F32_VEC_FMA GGML_F32x16_FMA#define GGML_F32_VEC_ADD GGML_F32x16_ADD-#define GGML_F32_VEC_SUB GGML_F32x16_SUB#define GGML_F32_VEC_MUL GGML_F32x16_MUL#define GGML_F32_VEC_REDUCE GGML_F32x16_REDUCE@@ -1062,7 +907,6 @@ do { \#define GGML_F32Cx16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)#define GGML_F32Cx16_ADD _mm512_add_ps-#define GGML_F32Cx16_SUB _mm512_sub_ps#define GGML_F32Cx16_MUL _mm512_mul_ps#define GGML_F32Cx16_REDUCE(res, x) \do { \@@ -1088,7 +932,6 @@ do { \#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx16_STORE(p, r[i])#define GGML_F16_VEC_FMA GGML_F32Cx16_FMA#define GGML_F16_VEC_ADD GGML_F32Cx16_ADD-#define GGML_F16_VEC_SUB GGML_F32Cx16_SUB#define GGML_F16_VEC_MUL GGML_F32Cx16_MUL#define GGML_F16_VEC_REDUCE GGML_F32Cx16_REDUCE@@ -1112,9 +955,26 @@ do { \#define GGML_F32x8_FMA(a, b, c) _mm256_add_ps(_mm256_mul_ps(b, c), a)#endif#define GGML_F32x8_ADD _mm256_add_ps-#define GGML_F32x8_SUB _mm256_sub_ps#define GGML_F32x8_MUL _mm256_mul_ps-#define GGML_F32x8_REDUCE(res, x) GGML_F32x8_REDUCE_AVX(res, x)+#define GGML_F32x8_REDUCE(res, x) \+do { \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \+ } \+ const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), \+ _mm256_extractf128_ps(x[0], 1)); \+ const __m128 t1 = _mm_hadd_ps(t0, t0); \+ res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); \+} while (0)// TODO: is this optimal ?#define GGML_F32_VEC GGML_F32x8@@ -1124,7 +984,6 @@ do { \#define GGML_F32_VEC_STORE GGML_F32x8_STORE#define GGML_F32_VEC_FMA GGML_F32x8_FMA#define GGML_F32_VEC_ADD GGML_F32x8_ADD-#define GGML_F32_VEC_SUB GGML_F32x8_SUB#define GGML_F32_VEC_MUL GGML_F32x8_MUL#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE@@ -1144,7 +1003,7 @@ do { \#define GGML_F32Cx8_LOAD(x) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x)))#define GGML_F32Cx8_STORE(x, y) _mm_storeu_si128((__m128i *)(x), _mm256_cvtps_ph(y, 0))#else-static inline __m256 __avx_f32cx8_load(const ggml_fp16_t *x) {+static inline __m256 __avx_f32cx8_load(ggml_fp16_t *x) {float tmp[8];for (int i = 0; i < 8; i++) {@@ -1167,7 +1026,6 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {#define GGML_F32Cx8_FMA GGML_F32x8_FMA#define GGML_F32Cx8_ADD _mm256_add_ps-#define GGML_F32Cx8_SUB _mm256_sub_ps#define GGML_F32Cx8_MUL _mm256_mul_ps#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE@@ -1178,7 +1036,6 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i])#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD-#define GGML_F16_VEC_SUB GGML_F32Cx8_SUB#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE@@ -1198,7 +1055,6 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {#define GGML_F32x4_STORE(p, r) vec_xst(r, 0, p)#define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a)#define GGML_F32x4_ADD vec_add-#define GGML_F32x4_SUB vec_sub#define GGML_F32x4_MUL vec_mul#define GGML_F32x4_REDUCE(res, x) \{ \@@ -1227,7 +1083,6 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {#define GGML_F32_VEC_STORE GGML_F32x4_STORE#define GGML_F32_VEC_FMA GGML_F32x4_FMA#define GGML_F32_VEC_ADD GGML_F32x4_ADD-#define GGML_F32_VEC_SUB GGML_F32x4_SUB#define GGML_F32_VEC_MUL GGML_F32x4_MUL#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE@@ -1266,7 +1121,6 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {#define GGML_F32x4_STORE wasm_v128_store#define GGML_F32x4_FMA(a, b, c) wasm_f32x4_add(wasm_f32x4_mul(b, c), a)#define GGML_F32x4_ADD wasm_f32x4_add-#define GGML_F32x4_SUB wasm_f32x4_sub#define GGML_F32x4_MUL wasm_f32x4_mul#define GGML_F32x4_REDUCE(res, x) \{ \@@ -1295,7 +1149,6 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {#define GGML_F32_VEC_STORE GGML_F32x4_STORE#define GGML_F32_VEC_FMA GGML_F32x4_FMA#define GGML_F32_VEC_ADD GGML_F32x4_ADD-#define GGML_F32_VEC_SUB GGML_F32x4_SUB#define GGML_F32_VEC_MUL GGML_F32x4_MUL#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE@@ -1333,7 +1186,6 @@ inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {#define GGML_F16x4_STORE(x, y) __wasm_f16x4_store(x, y)#define GGML_F16x4_FMA GGML_F32x4_FMA#define GGML_F16x4_ADD wasm_f32x4_add-#define GGML_F16x4_SUB wasm_f32x4_sub#define GGML_F16x4_MUL wasm_f32x4_mul#define GGML_F16x4_REDUCE(res, x) \{ \@@ -1362,7 +1214,6 @@ inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {#define GGML_F16_VEC_STORE(p, r, i) GGML_F16x4_STORE(p, r[i])#define GGML_F16_VEC_FMA GGML_F16x4_FMA#define GGML_F16_VEC_ADD GGML_F16x4_ADD-#define GGML_F16_VEC_SUB GGML_F16x4_SUB#define GGML_F16_VEC_MUL GGML_F16x4_MUL#define GGML_F16_VEC_REDUCE GGML_F16x4_REDUCE@@ -1387,7 +1238,6 @@ inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {#define GGML_F32x4_FMA(a, b, c) _mm_add_ps(_mm_mul_ps(b, c), a)#endif#define GGML_F32x4_ADD _mm_add_ps-#define GGML_F32x4_SUB _mm_sub_ps#define GGML_F32x4_MUL _mm_mul_ps#define GGML_F32x4_REDUCE(res, x) \{ \@@ -1415,7 +1265,6 @@ inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {#define GGML_F32_VEC_STORE GGML_F32x4_STORE#define GGML_F32_VEC_FMA GGML_F32x4_FMA#define GGML_F32_VEC_ADD GGML_F32x4_ADD-#define GGML_F32_VEC_SUB GGML_F32x4_SUB#define GGML_F32_VEC_MUL GGML_F32x4_MUL#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE@@ -1453,7 +1302,6 @@ static inline void __sse_f16x4_store(ggml_fp16_t *x, __m128 y) {#define GGML_F32Cx4_STORE(x, y) __sse_f16x4_store(x, y)#define GGML_F32Cx4_FMA GGML_F32x4_FMA#define GGML_F32Cx4_ADD _mm_add_ps-#define GGML_F32Cx4_SUB _mm_sub_ps#define GGML_F32Cx4_MUL _mm_mul_ps#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE@@ -1464,7 +1312,6 @@ static inline void __sse_f16x4_store(ggml_fp16_t *x, __m128 y) {#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i])#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD-#define GGML_F16_VEC_SUB GGML_F32Cx4_SUB#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE@@ -1477,423 +1324,6 @@ static inline void __sse_f16x4_store(ggml_fp16_t *x, __m128 y) {#define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR)#endif-//-// fundamental operations-//--inline static void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v) { for (int i = 0; i < n; ++i) x[i] = v; }--inline static void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }--inline static void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }--inline static void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }--inline static void ggml_vec_set_bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }--inline static void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] + y[i]; }-inline static void ggml_vec_add1_f32(const int n, float * z, const float * x, const float v) { for (int i = 0; i < n; ++i) z[i] = x[i] + v; }-inline static void ggml_vec_acc_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] += x[i]; }-inline static void ggml_vec_acc1_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] += v; }-inline static void ggml_vec_sub_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] - y[i]; }-inline static void ggml_vec_set_f32 (const int n, float * x, const float v) { for (int i = 0; i < n; ++i) x[i] = v; }-inline static void ggml_vec_cpy_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]; }-inline static void ggml_vec_neg_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = -x[i]; }-inline static void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]*y[i]; }-inline static void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]/y[i]; }--static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc) {- assert(nrc == 1);- UNUSED(nrc);- UNUSED(bx);- UNUSED(by);- UNUSED(bs);--#ifdef GGML_SIMD- float sumf = 0.0f;- const int np = (n & ~(GGML_F32_STEP - 1));-- GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };-- GGML_F32_VEC ax[GGML_F32_ARR];- GGML_F32_VEC ay[GGML_F32_ARR];-- for (int i = 0; i < np; i += GGML_F32_STEP) {- for (int j = 0; j < GGML_F32_ARR; j++) {- ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);- ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);-- sum[j] = GGML_F32_VEC_FMA(sum[j], ax[j], ay[j]);- }- }-- // reduce sum0..sum3 to sum0- GGML_F32_VEC_REDUCE(sumf, sum);-- // leftovers- for (int i = np; i < n; ++i) {- sumf += x[i]*y[i];- }-#else- // scalar- ggml_float sumf = 0.0;- for (int i = 0; i < n; ++i) {- sumf += (ggml_float)(x[i]*y[i]);- }-#endif-- *s = sumf;-}--static void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc) {- assert(nrc == 1);- UNUSED(nrc);- UNUSED(bx);- UNUSED(by);- UNUSED(bs);- ggml_float sumf = 0;- for (int i = 0; i < n; ++i) {- sumf += GGML_BF16_TO_FP32(x[i]) * GGML_BF16_TO_FP32(y[i]);- }- *s = sumf;-}--#ifdef __x86_64__-#pragma GCC push_options-#pragma GCC target("avx")-#pragma GCC target("f16c")-#pragma GCC target("fma")-static void ggml_vec_dot_f16_avx(const int n, float * restrict s, ggml_fp16_t * restrict x, ggml_fp16_t * restrict y) {- ggml_float sumf = 0.0;- int np = (n & ~(32 - 1));- __m256 ax[(32/8)], ay[(32/8)], sum[(32/8)] = { _mm256_setzero_ps() };- for (int i = 0; i < np; i += 32) {- for (int j = 0; j < (32/8); j++) {- ax[j] = _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(x + i + j*8)));- ay[j] = _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(y + i + j*8)));- sum[j] = _mm256_fmadd_ps(ax[j], ay[j], sum[j]);- }- }- // reduce sum0..sum3 to sum0- GGML_F32x8_REDUCE_AVX(sumf, sum);- for (int i = np; i < n; ++i) {- sumf += (ggml_float)(ggml_lookup_fp16_to_fp32(x[i]) *- ggml_lookup_fp16_to_fp32(y[i]));- }- *s = sumf;-}-#pragma GCC pop_options-#endif /* __x86_64__ */--static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc) {- assert(nrc == 1);- UNUSED(nrc);- UNUSED(bx);- UNUSED(by);- UNUSED(bs);--#ifdef __x86_64__- if (X86_HAVE(AVX) && X86_HAVE(F16C) && X86_HAVE(FMA)) {- return ggml_vec_dot_f16_avx(n, s, x, y);- }-#endif-- ggml_float sumf = 0.0;--#if defined(GGML_SIMD)- const int np = (n & ~(GGML_F16_STEP - 1));-- GGML_F16_VEC sum[GGML_F16_ARR] = { GGML_F16_VEC_ZERO };- GGML_F16_VEC err[GGML_F16_ARR] = { GGML_F16_VEC_ZERO };-- GGML_F16_VEC ax[GGML_F16_ARR];- GGML_F16_VEC ay[GGML_F16_ARR];-- for (int i = 0; i < np; i += GGML_F16_STEP) {- for (int j = 0; j < GGML_F16_ARR; j++) {- ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);- ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);-- if (GGML_KAHAN) {- GGML_F16_VEC ky = GGML_F16_VEC_SUB(GGML_F16_VEC_MUL(ax[j], ay[j]), err[j]);- GGML_F16_VEC kt = GGML_F16_VEC_ADD(sum[j], ky);- err[j] = GGML_F16_VEC_SUB(GGML_F16_VEC_SUB(kt, sum[j]), ky);- sum[j] = kt;- } else {- sum[j] = GGML_F16_VEC_FMA(sum[j], ax[j], ay[j]);- }- }- }-- // reduce sum0..sum3 to sum0- GGML_F16_VEC_REDUCE(sumf, sum);-- // leftovers- for (int i = np; i < n; ++i) {- sumf += (ggml_float)(GGML_FP16_TO_FP32(x[i])*GGML_FP16_TO_FP32(y[i]));- }-#else- for (int i = 0; i < n; ++i) {- sumf += (ggml_float)(GGML_FP16_TO_FP32(x[i])*GGML_FP16_TO_FP32(y[i]));- }-#endif-- *s = sumf;-}--// compute GGML_VEC_DOT_UNROLL dot products at once-// xs - x row stride in bytes-inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y) {- ggml_float sumf[GGML_VEC_DOT_UNROLL] = { 0.0 };-- ggml_fp16_t * restrict x[GGML_VEC_DOT_UNROLL];-- for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {- x[i] = (ggml_fp16_t *) ((char *) xv + i*xs);- }--#if defined(GGML_SIMD)- const int np = (n & ~(GGML_F16_STEP - 1));-- GGML_F16_VEC sum[GGML_VEC_DOT_UNROLL][GGML_F16_ARR] = { { GGML_F16_VEC_ZERO } };- GGML_F16_VEC err[GGML_VEC_DOT_UNROLL][GGML_F16_ARR] = { { GGML_F16_VEC_ZERO } };-- GGML_F16_VEC ax[GGML_F16_ARR];- GGML_F16_VEC ay[GGML_F16_ARR];-- for (int i = 0; i < np; i += GGML_F16_STEP) {- for (int j = 0; j < GGML_F16_ARR; j++) {- ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);-- for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {- ax[j] = GGML_F16_VEC_LOAD(x[k] + i + j*GGML_F16_EPR, j);-- if (GGML_KAHAN) {- GGML_F16_VEC ky = GGML_F16_VEC_SUB(GGML_F16_VEC_MUL(ax[j], ay[j]), err[k][j]);- GGML_F16_VEC kt = GGML_F16_VEC_ADD(sum[k][j], ky);- err[k][j] = GGML_F16_VEC_SUB(GGML_F16_VEC_SUB(kt, sum[k][j]), ky);- sum[k][j] = kt;- } else {- sum[k][j] = GGML_F16_VEC_FMA(sum[k][j], ax[j], ay[j]);- }- }- }- }-- // reduce sum0..sum3 to sum0- for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {- GGML_F16_VEC_REDUCE(sumf[k], sum[k]);- }-- // leftovers- for (int i = np; i < n; ++i) {- for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {- sumf[j] += (ggml_float)(GGML_FP16_TO_FP32(x[j][i])*GGML_FP16_TO_FP32(y[i]));- }- }-#else- for (int i = 0; i < n; ++i) {- for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {- sumf[j] += (ggml_float)(GGML_FP16_TO_FP32(x[j][i])*GGML_FP16_TO_FP32(y[i]));- }- }-#endif-- for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {- s[i] = sumf[i];- }-}--inline static void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v) {-#if defined(GGML_SIMD)- const int np = (n & ~(GGML_F32_STEP - 1));-- GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);-- GGML_F32_VEC ax[GGML_F32_ARR];- GGML_F32_VEC ay[GGML_F32_ARR];-- for (int i = 0; i < np; i += GGML_F32_STEP) {- for (int j = 0; j < GGML_F32_ARR; j++) {- ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);- ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);- ay[j] = GGML_F32_VEC_FMA(ay[j], ax[j], vx);-- GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);- }- }-- // leftovers- for (int i = np; i < n; ++i) {- y[i] += x[i]*v;- }-#else- // scalar- for (int i = 0; i < n; ++i) {- y[i] += x[i]*v;- }-#endif-}--inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v) {-#if defined(GGML_SIMD)- const int np = (n & ~(GGML_F16_STEP - 1));-- GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);-- GGML_F16_VEC ax[GGML_F16_ARR];- GGML_F16_VEC ay[GGML_F16_ARR];-- for (int i = 0; i < np; i += GGML_F16_STEP) {- for (int j = 0; j < GGML_F16_ARR; j++) {- ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);- ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);- ay[j] = GGML_F16_VEC_FMA(ay[j], ax[j], vx);-- GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);- }- }-- // leftovers- for (int i = np; i < n; ++i) {- y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i]) + GGML_FP16_TO_FP32(x[i])*v);- }-#else- // scalar- for (int i = 0; i < n; ++i) {- y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i]) + GGML_FP16_TO_FP32(x[i])*v);- }-#endif-}--// xs and vs are byte strides of x and v-inline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv) {-- const float * restrict x[GGML_VEC_MAD_UNROLL];- const float * restrict v[GGML_VEC_MAD_UNROLL];-- for (int i = 0; i < GGML_VEC_MAD_UNROLL; ++i) {- x[i] = (const float *) ((const char *) xv + i*xs);- v[i] = (const float *) ((const char *) vv + i*vs);- }--#if defined(GGML_SIMD)- const int np = (n & ~(GGML_F32_STEP - 1));-- GGML_F32_VEC vx[GGML_VEC_MAD_UNROLL];-- for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {- vx[k] = GGML_F32_VEC_SET1(v[k][0]);- }-- GGML_F32_VEC ax[GGML_VEC_MAD_UNROLL][GGML_F32_ARR];- GGML_F32_VEC ay[GGML_F32_ARR];-- for (int i = 0; i < np; i += GGML_F32_STEP) {- for (int j = 0; j < GGML_F32_ARR; j++) {- ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);-- for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {- ax[k][j] = GGML_F32_VEC_LOAD(x[k] + i + j*GGML_F32_EPR);- ay[j] = GGML_F32_VEC_FMA(ay[j], ax[k][j], vx[k]);- }-- GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);- }- }-- // leftovers- for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {- for (int i = np; i < n; ++i) {- y[i] += x[k][i]*v[k][0];- }- }-#else- // scalar- for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {- for (int i = 0; i < n; ++i) {- y[i] += x[k][i]*v[k][0];- }- }-#endif-}--//inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] *= v; }-inline static void ggml_vec_scale_f32(const int n, float * y, const float v) {-#if defined(GGML_USE_ACCELERATE)- vDSP_vsmul(y, 1, &v, y, 1, n);-#elif defined(GGML_SIMD)- const int np = (n & ~(GGML_F32_STEP - 1));-- GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);-- GGML_F32_VEC ay[GGML_F32_ARR];-- for (int i = 0; i < np; i += GGML_F32_STEP) {- for (int j = 0; j < GGML_F32_ARR; j++) {- ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);- ay[j] = GGML_F32_VEC_MUL(ay[j], vx);-- GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);- }- }-- // leftovers- for (int i = np; i < n; ++i) {- y[i] *= v;- }-#else- // scalar- for (int i = 0; i < n; ++i) {- y[i] *= v;- }-#endif-}--inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {-#if defined(GGML_SIMD)- const int np = (n & ~(GGML_F16_STEP - 1));-- GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);-- GGML_F16_VEC ay[GGML_F16_ARR];-- for (int i = 0; i < np; i += GGML_F16_STEP) {- for (int j = 0; j < GGML_F16_ARR; j++) {- ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);- ay[j] = GGML_F16_VEC_MUL(ay[j], vx);-- GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);- }- }-- // leftovers- for (int i = np; i < n; ++i) {- y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i])*v);- }-#else- // scalar- for (int i = 0; i < n; ++i) {- y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i])*v);- }-#endif-}--inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, 0, x, 0, x, 0, 1); *s = sqrtf(*s); }-inline static void ggml_vec_sqr_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i]; }-inline static void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrtf(x[i]); }-inline static void ggml_vec_log_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = logf(x[i]); }-inline static void ggml_vec_abs_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fabsf(x[i]); }-inline static void ggml_vec_sgn_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : ((x[i] < 0.f) ? -1.f : 0.f); }-inline static void ggml_vec_step_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : 0.f; }-inline static void ggml_vec_tanh_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = tanhf(x[i]); }-inline static void ggml_vec_elu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : expf(x[i])-1; }-inline static void ggml_vec_relu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.f; }-inline static void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); }-// TODO: optimize performance-inline static void ggml_vec_hardswish_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }-inline static void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }-////////////////////////////////////////////////////////////////////////////////// SYNCHRONIZATION PRIMITIVES@@ -1911,162 +1341,6 @@ void ggml_once(atomic_uint * once, void init(void)) {}}-// ACTIVATION FUNCTIONS-//-// 0.3 +--------------------------------------------------------------------------------------+-// | + + + + + + + * % |-// | x>0?x:0.01 **%**** |-// | x/(1+exp(-x)) #%##### |-// | .5*x*(1+erf(x/sqrt(2)))*$%$$$$$ |-// 0.2 |-+ x*(1/(1+exp(-1.702*x)))*%%%%%%%-|-// | *%# |-// | *% |-// | *% |-// | *% |-// 0.1 |-+ *% +-|-// | *% |-// | % |-// | *% |-// | % |-// 0 |***************************************************************************% +-|-// |%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%$$$$$ % |-// | ########## %%%%%$$$$ % |-// | ##### %%%%$$ % |-// | #### %%$$ % |-// | #### %%%$ % |-// -0.1 |-+ ### %%% %# +-|-// | ### % %# |-// | ### %%% %%# |-// | ## $%%%%$ # |-// | ### # |-// -0.2 |-+ ## # +-|-// | ### # |-// | ## # |-// | #### ### |-// | + + + + + #####+ + |-// -0.3 +--------------------------------------------------------------------------------------+-// -7 -6 -5 -4 -3 -2 -1 0 1-//--inline static float ggml_silu_f32(float x) {- // SiLU is the favored by LLaMA, Mistral, Phi, Rocket, etc.- return x/(1.f + expf(-x));-}--inline static float ggml_gelu_f32(float x) {- // GeLU approximation that goes slower and we seem to be stuck with.- return .5f * x * (1.f + tanhf(sqrtf(M_2_PI) * (x + .044715f * x * x * x)));-}--inline static float ggml_gelu_quick_f32(float x) {- // "Quick" GeLU is used by LLaVA's CLIP vision model.- return x*(1.f/(1.f+expf(-1.702f*x)));-}--static void ggml_vec_gelu_f16_init(void) {- ggml_table_gelu_f16 = malloc(sizeof(float) * 65536);- for (int i = 0; i < 65536; ++i) {- union {- unsigned short i;- ggml_fp16_t f;- } u = {i};- ggml_table_gelu_f16[i] = ggml_gelu_f32(u.f);- }-}--inline static void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {- static atomic_uint once;- ggml_once(&once, ggml_vec_gelu_f16_init);- const uint16_t * i16 = (const uint16_t *) x;- for (int i = 0; i < n; ++i) {- y[i] = ggml_table_gelu_f16[i16[i]];- }-}--inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {- for (int i = 0; i < n; ++i) {- y[i] = ggml_gelu_f32(x[i]);- }-}--inline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {- for (int i = 0; i < n; ++i) {- y[i] = ggml_gelu_quick_f32(x[i]);- }-}--inline static void ggml_vec_silu_f32(const int n, float * y, const float * x) {- for (int i = 0; i < n; ++i) {- y[i] = ggml_silu_f32(x[i]);- }-}--inline static float ggml_silu_backward_f32(float x, float dy) {- const float s = 1.0f/(1.0f + expf(-x));- return dy*s*(1.0f + x*(1.0f - s));-}--inline static void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, const float * dy) {- for (int i = 0; i < n; ++i) {- dx[i] = ggml_silu_backward_f32(x[i], dy[i]);- }-}--inline static void ggml_vec_sum_f32(const int n, float * s, const float * x) {-#ifndef GGML_USE_ACCELERATE- ggml_float sum = 0.0;- for (int i = 0; i < n; ++i) {- sum += (ggml_float)x[i];- }- *s = sum;-#else- vDSP_sve(x, 1, s, n);-#endif-}--inline static void ggml_vec_sum_f32_ggf(const int n, ggml_float * s, const float * x) {- ggml_float sum = 0.0;- for (int i = 0; i < n; ++i) {- sum += (ggml_float)x[i];- }- *s = sum;-}--inline static void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x) {- float sum = 0.0f;- for (int i = 0; i < n; ++i) {- sum += GGML_FP16_TO_FP32(x[i]);- }- *s = sum;-}--inline static void ggml_vec_max_f32(const int n, float * s, const float * x) {-#ifndef GGML_USE_ACCELERATE- float max = -INFINITY;- for (int i = 0; i < n; ++i) {- max = MAX(max, x[i]);- }- *s = max;-#else- vDSP_maxv(x, 1, s, n);-#endif-}--inline static void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) {- ggml_vec_norm_f32(n, s, x);- *s = 1.f/(*s);-}--inline static void ggml_vec_argmax_f32(const int n, int * s, const float * x) {- float max = -INFINITY;- int idx = 0;- for (int i = 0; i < n; ++i) {- max = MAX(max, x[i]);- if (max == x[i]) { idx = i; }- }- *s = idx;-}-//// data types//@@ -11184,6 +10458,7 @@ UseGgmlGemm2:;}#if 0+ if (ne11 > 1)fprintf(stderr, "MULMAT %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) * %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) -> %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) vdt=%s src1_cont=%d\n",ggml_type_name(src0->type), ne00, ne01, ne02, nb00, nb01, nb02,ggml_type_name(src1->type), ne10, ne11, ne12, nb10, nb11, nb12,@@ -16908,6 +16183,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggmGGML_ASSERT(false);} break;}++#ifdef LLAMAFILE_DEBUG+ llamafile_trapping_restore();+#endif}////////////////////////////////////////////////////////////////////////////////@@ -18855,6 +18134,72 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {else {ggml_graph_compute_thread_sync_task(&task_phase, state, false);}++#if 1+ // expensive integrity check+ // not useful to test mt-safety+ if (!state->ith) {+ int64_t nans = 0;+ int64_t infs = 0;+ if (node->type == GGML_TYPE_F32)+ for (int i3 = 0; i3 < node->ne[3]; ++i3)+ for (int i2 = 0; i2 < node->ne[2]; ++i2)+ for (int i1 = 0; i1 < node->ne[1]; ++i1)+ for (int i0 = 0; i0 < node->ne[0]; ++i0) {+ float x = *(const float *)((const char *)node->data ++ i3 * node->nb[3] ++ i2 * node->nb[2] ++ i1 * node->nb[1] ++ i0 * node->nb[0]);+ if (isnan(x))+ ++nans;+ if (isinf(x))+ ++infs;+ }+ else if (node->type == GGML_TYPE_F16)+ for (int i3 = 0; i3 < node->ne[3]; ++i3)+ for (int i2 = 0; i2 < node->ne[2]; ++i2)+ for (int i1 = 0; i1 < node->ne[1]; ++i1)+ for (int i0 = 0; i0 < node->ne[0]; ++i0) {+ float x = GGML_FP16_TO_FP32(*(const ggml_fp16_t *)((const char *)node->data ++ i3 * node->nb[3] ++ i2 * node->nb[2] ++ i1 * node->nb[1] ++ i0 * node->nb[0]));+ if (isnan(x))+ ++nans;+ if (isinf(x))+ ++infs;+ }+ else if (node->type == GGML_TYPE_BF16)+ for (int i3 = 0; i3 < node->ne[3]; ++i3)+ for (int i2 = 0; i2 < node->ne[2]; ++i2)+ for (int i1 = 0; i1 < node->ne[1]; ++i1)+ for (int i0 = 0; i0 < node->ne[0]; ++i0) {+ float x = GGML_BF16_TO_FP32(*(const ggml_bf16_t *)((const char *)node->data ++ i3 * node->nb[3] ++ i2 * node->nb[2] ++ i1 * node->nb[1] ++ i0 * node->nb[0]));+ if (isnan(x))+ ++nans;+ if (isinf(x))+ ++infs;+ }+ else+ continue;+ if (nans || infs) {+ flockfile(stderr);+ fprintf(stderr, "ERROR: node #%d produced %" PRId64 " NaNs and %" PRId64 " infinities in sequence %s -> %s\n",+ node_n, nans, infs, node_n ? ggml_op_name(cgraph->nodes[node_n - 1]->op) : "n/a",+ ggml_op_name(node->op));+ for (int i = 0; i < GGML_MAX_SRC && node->src[i]; ++i)+ fprintf(stderr, "\t- src[%d] is %s\n", i, ggml_op_name(node->src[i]->op));+ exit(1);+ }+ }+#endif+}return 0;diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex b45c29ba..478865c5 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -328,14 +328,20 @@ extern "C" {// get ggml_status name stringGGML_API GGML_CALL const char * ggml_status_to_string(enum ggml_status status);+ // ieee 754-2008 half-precision float16+ // todo: make this not an integral typetypedef uint16_t ggml_fp16_t;-- // convert FP16 <-> FP32- GGML_API float ggml_fp16_to_fp32(ggml_fp16_t x);- GGML_API ggml_fp16_t ggml_fp32_to_fp16(float x);-- GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n);- GGML_API void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n);+ GGML_API float ggml_fp16_to_fp32(ggml_fp16_t);+ GGML_API ggml_fp16_t ggml_fp32_to_fp16(float);+ GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t);+ GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t);++ // google brain half-precision bfloat16+ typedef struct { uint16_t bits; } ggml_bf16_t;+ GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);+ GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16+ GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);+ GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);struct ggml_object;struct ggml_context;@@ -2394,90 +2400,6 @@ extern "C" {GGML_API int ggml_cpu_has_vsx (void);GGML_API int ggml_cpu_has_matmul_int8(void);- /**- * Google Brain 16-bit floating point number.- *- * ┌sign- * │- * │ ┌exponent- * │ │- * │ │ ┌mantissa- * │ │ │- * │┌──┴───┐┌─┴───┐- * 0b0000000000000000 bfloat16- *- * Since bf16 has the same number of exponent bits as a 32bit float,- * encoding and decoding numbers becomes relatively straightforward.- *- * ┌sign- * │- * │ ┌exponent- * │ │- * │ │ ┌mantissa- * │ │ │- * │┌──┴───┐┌─┴───────────────────┐- * 0b00000000000000000000000000000000 IEEE binary32- *- * For comparison, the standard fp16 format has fewer exponent bits.- *- * ┌sign- * │- * │ ┌exponent- * │ │- * │ │ ┌mantissa- * │ │ │- * │┌─┴─┐┌─┴──────┐- * 0b0000000000000000 IEEE binary16- *- * So be warned that converting between them yields a ten bit float.- *- * @see IEEE 754-2008- */- typedef struct {- uint16_t x;- } ggml_bf16_t;-- /**- * Converts bfloat16 to float32.- */- static inline float ggml_bf16_to_fp32(ggml_bf16_t h) {- union {- float f;- uint32_t i;- } u;- u.i = (uint32_t)h.x << 16;- return u.f;- }-- /**- * Converts float32 to bfloat16.- *- * This function is binary identical to AMD Zen4 VCVTNEPS2BF16.- * Subnormals shall be flushed to zero, and NANs will be quiet.- * This code should vectorize nicely if using modern compilers.- */- static inline ggml_bf16_t ggml_fp32_to_bf16(float s) {- ggml_bf16_t h;- union {- float f;- uint32_t i;- } u;- u.f = s;- if ((u.i & 0x7fffffff) > 0x7f800000) { /* nan */- h.x = (u.i >> 16) | 64; /* force to quiet */- return h;- }- if (!(u.i & 0x7f800000)) { /* subnormal */- h.x = (u.i & 0x80000000) >> 16; /* flush to zero */- return h;- }- h.x = (u.i + (0x7fff + ((u.i >> 16) & 1))) >> 16;- return h;- }-- GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n);- GGML_API void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n);-//// Internal types and functions exposed for tests and benchmarks//diff --git a/llama.cpp/llava/clip.cpp b/llama.cpp/llava/clip.cppindex ec8d3654..abf8872e 100644--- a/llama.cpp/llava/clip.cpp+++ b/llama.cpp/llava/clip.cpp@@ -908,7 +908,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {LOG_TEE("%s: n_tensors: %d\n", __func__, n_tensors);LOG_TEE("%s: n_kv: %d\n", __func__, n_kv);LOG_TEE("%s: ftype: %s\n", __func__, ftype_str.c_str());- LOG_TEE("\n");+ // LOG_TEE("\n"); // [jart] do not want}const int n_tensors = gguf_get_n_tensors(ctx);@@ -1157,7 +1157,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {for (int i = 0; i < 32 && (hparams.image_grid_pinpoints[i] != 0); ++i) {LOG_TEE("%d ", hparams.image_grid_pinpoints[i]);}- LOG_TEE("\n");+ // LOG_TEE("\n"); // [jart] do not wantLOG_TEE("v_mm_patch_merge_type: %s\n", hparams.mm_patch_merge_type);}diff --git a/llama.cpp/llava/llava.cpp b/llama.cpp/llava/llava.cppindex bdd50471..280a7ec7 100644--- a/llama.cpp/llava/llava.cpp+++ b/llama.cpp/llava/llava.cpp@@ -297,7 +297,8 @@ static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const cliconst int64_t t_img_enc_end_us = ggml_time_us();float t_img_enc_ms = (t_img_enc_end_us - t_img_enc_start_us) / 1000.0;- LOG_TEE("\n%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / *n_img_pos);+ // [jart] get rid of sneak newline+ LOG_TEE("%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / *n_img_pos);return true;}diff --git a/llama.cpp/log.h b/llama.cpp/log.hindex 3c2d077c..db1ab355 100644--- a/llama.cpp/log.h+++ b/llama.cpp/log.h@@ -1,4 +1,6 @@#pragma once+#define LOG_NO_TIMESTAMPS+#define LOG_NO_FILE_LINE_FUNCTION#include <chrono>#include <cstring>diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex e0b5ceec..989295d8 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -89,13 +89,20 @@ o/$(MODE)/llamafile: \#o/$(MODE)/llamafile/sgemm.o: private CXXFLAGS += -Os-o/$(MODE)/llamafile/tinyblas_cpu_amd_avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge -Xx86_64-mf16c-o/$(MODE)/llamafile/tinyblas_cpu_amd_fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mf16c -Xx86_64-mfma-o/$(MODE)/llamafile/tinyblas_cpu_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mfma-o/$(MODE)/llamafile/tinyblas_cpu_amd_avxvnni.o: private TARGET_ARCH += -Xx86_64-mtune=alderlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavxvnni-o/$(MODE)/llamafile/tinyblas_cpu_amd_avx512f.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f-o/$(MODE)/llamafile/tinyblas_cpu_amd_zen4.o: private TARGET_ARCH += -Xx86_64-mtune=znver4 -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512vl -Xx86_64-mavx512vnni -Xx86_64-mavx512bf16-o/$(MODE)/llamafile/tinyblas_cpu_arm82.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+dotprod+fp16+o/$(MODE)/llamafile/tinyblas_cpu_sgemm_amd_avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge -Xx86_64-mf16c+o/$(MODE)/llamafile/tinyblas_cpu_mixmul_amd_avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge -Xx86_64-mf16c+o/$(MODE)/llamafile/tinyblas_cpu_sgemm_amd_fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mf16c -Xx86_64-mfma+o/$(MODE)/llamafile/tinyblas_cpu_mixmul_amd_fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mf16c -Xx86_64-mfma+o/$(MODE)/llamafile/tinyblas_cpu_sgemm_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mfma+o/$(MODE)/llamafile/tinyblas_cpu_mixmul_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mfma+o/$(MODE)/llamafile/tinyblas_cpu_sgemm_amd_avxvnni.o: private TARGET_ARCH += -Xx86_64-mtune=alderlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavxvnni+o/$(MODE)/llamafile/tinyblas_cpu_mixmul_amd_avxvnni.o: private TARGET_ARCH += -Xx86_64-mtune=alderlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavxvnni+o/$(MODE)/llamafile/tinyblas_cpu_sgemm_amd_avx512f.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f+o/$(MODE)/llamafile/tinyblas_cpu_mixmul_amd_avx512f.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f+o/$(MODE)/llamafile/tinyblas_cpu_sgemm_amd_zen4.o: private TARGET_ARCH += -Xx86_64-mtune=znver4 -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512vl -Xx86_64-mavx512vnni -Xx86_64-mavx512bf16+o/$(MODE)/llamafile/tinyblas_cpu_mixmul_amd_zen4.o: private TARGET_ARCH += -Xx86_64-mtune=znver4 -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512vl -Xx86_64-mavx512vnni -Xx86_64-mavx512bf16+o/$(MODE)/llamafile/tinyblas_cpu_sgemm_arm82.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+dotprod+fp16+o/$(MODE)/llamafile/tinyblas_cpu_mixmul_arm82.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+dotprod+fp16################################################################################# testingdiff --git a/llamafile/debug.cpp b/llamafile/debug.cppindex e8c9732a..15512354 100644--- a/llamafile/debug.cpp+++ b/llamafile/debug.cpp@@ -16,9 +16,11 @@// limitations under the License.#include "llamafile.h"+#include "log.h"#include <cosmo.h>#include <fenv.h>+#include <libc/calls/struct/aarch64.internal.h>#include <libc/calls/struct/ucontext.internal.h>#include <signal.h>#include <stdatomic.h>@@ -37,6 +39,7 @@ static int get_config(void) {}bool FLAG_trap;+static thread_local int g_enabled;thread_local int llamafile_debug_op_index;const struct ggml_cgraph *llamafile_debug_graph;@@ -105,45 +108,89 @@ static inline void spinlock(atomic_uint *lock) {}}+static inline void spunlock(atomic_uint *lock) {+ atomic_store_explicit(lock, 0, memory_order_release);+}+static void on_sigfpe(int sig, siginfo_t *si, void *arg) {- ucontext_t *ctx = (ucontext_t *)arg;+static atomic_uint lock;- spinlock(&lock); // first thread to crash wins- g_terminal_buddy.restore();- const char *s = "\nerror: ";+ spinlock(&lock);++ const char *issue;+ ucontext_t *ctx = (ucontext_t *)arg;if (si->si_code == FPE_INTDIV)- tinyprint(2, s, "integer divide by zero\n", NULL);+ issue = "integer divide by zero";else if (si->si_code == FPE_INTOVF)- tinyprint(2, s, "integer overflow\n", NULL);+ issue = "integer overflow";else if (si->si_code == FPE_FLTDIV)- tinyprint(2, s, "floating point divide by zero\n", NULL);+ issue = "floating point divide by zero";else if (si->si_code == FPE_FLTOVF)- tinyprint(2, s, "floating point overflow\n", NULL);+ issue = "floating point overflow";else if (si->si_code == FPE_FLTUND)- tinyprint(2, s, "floating point underflow\n", NULL);+ issue = "floating point underflow";else if (si->si_code == FPE_FLTRES)- tinyprint(2, s, "floating point inexact\n", NULL);+ issue = "floating point inexact";else if (si->si_code == FPE_FLTINV)- tinyprint(2, s, "invalid floating point operation\n", NULL);+ issue = "invalid floating point operation";else if (si->si_code == FPE_FLTSUB)- tinyprint(2, s, "subscript out of range\n", NULL);+ issue = "subscript out of range";else- tinyprint(2, s, "caught sigfpe\n", NULL);- struct StackFrame sf = {.next = (struct StackFrame *)ctx->uc_mcontext.BP,- .addr = ctx->uc_mcontext.PC};- ShowBacktrace(2, &sf);-#ifdef LLAMAFILE_DEBUG+ issue = "sigfpe";++ // show detailed information+ static bool once;+ char location[21];+ const char *path = "/tmp/cgraph.txt";+ int idx = llamafile_debug_op_index;const struct ggml_cgraph *g;if ((g = llamafile_debug_graph)) {- int idx = llamafile_debug_op_index;- const char *path = "/tmp/cgraph.txt";- fprintf(stderr, "while executing %s at index #%d of %s\n", //- ggml_op_name(g->nodes[idx]->op), idx, path);- FILE *f = fopen(path, "w");- print_graph(f, g);- fclose(f);+ FormatInt32(location, idx);+ tinyprint(2, "error: ", issue, " at ", ggml_op_name(g->nodes[idx]->op), " operation in ",+ path, " at index #", location, "\n", NULL);+ } else {+ FormatHex64(location, ctx->uc_mcontext.PC, 2);+ tinyprint(2, "error: ", issue, " at pc ", location, "\n", NULL);+ }+ if (!once) {+ struct StackFrame sf = {.next = (struct StackFrame *)ctx->uc_mcontext.BP,+ .addr = ctx->uc_mcontext.PC};+ ShowBacktrace(2, &sf);+ const struct ggml_cgraph *g;+ if ((g = llamafile_debug_graph)) {+ FILE *f = fopen(path, "w");+ print_graph(f, g);+ fclose(f);+ }+ once = true;+ }++ // recover from trap so that execution may resume+ // without this the same signal will just keep getting raised+ int traps = get_config();+#ifdef __x86_64__+ if (ctx->uc_mcontext.fpregs) {+ ctx->uc_mcontext.fpregs->mxcsr |= traps << 7; // disable traps+ ctx->uc_mcontext.fpregs->mxcsr &= ~traps; // clear cages+ spunlock(&lock);+ return;+ }+#elif defined(__aarch64__)+ struct _aarch64_ctx *ac;+ for (ac = (struct _aarch64_ctx *)ctx->uc_mcontext.__reserved; ac->magic;+ ac = (struct _aarch64_ctx *)((char *)ac + ac->size)) {+ if (ac->magic == FPSIMD_MAGIC) {+ struct fpsimd_context *sm = (struct fpsimd_context *)ac;+ sm->fpcr &= ~(traps << 8); // disable traps+ sm->fpsr &= ~traps; // clear cages+ spunlock(&lock);+ return;+ }}#endif++ // time to die+ g_terminal_buddy.restore();_exit(128 + SIGFPE);}@@ -160,7 +207,6 @@ static void setup_sigfpe(void) {int llamafile_trapping_enabled(int delta) {static atomic_uint once;- static thread_local int g_enabled;bool was_enabled = g_enabled > 0;bool is_enabled = (g_enabled += delta) > 0;int config = get_config();@@ -174,3 +220,10 @@ int llamafile_trapping_enabled(int delta) {}return g_enabled;}++void llamafile_trapping_restore(void) {+ if (g_enabled > 0) {+ feclearexcept(FE_ALL_EXCEPT);+ feenableexcept(get_config());+ }+}diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex 0503676c..ddf056ce 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -69,6 +69,7 @@ struct ggml_cgraph;int feenableexcept(int);int fedisableexcept(int);int llamafile_trapping_enabled(int);+void llamafile_trapping_restore(void);void ShowBacktrace(int, const struct StackFrame *);extern const struct ggml_cgraph *llamafile_debug_graph;extern thread_local int llamafile_debug_op_index;diff --git a/llamafile/tinyblas_cpu.h b/llamafile/tinyblas_cpu.hindex 850ac507..e4b4ec8c 100644--- a/llamafile/tinyblas_cpu.h+++ b/llamafile/tinyblas_cpu.h@@ -43,6 +43,7 @@#include "llama.cpp/ggml-impl.h"#include "llama.cpp/ggml-quants.h"#include "llamafile.h"+#include "log.h"#include "sgemm.h"#include <cosmo.h>@@ -65,8 +66,20 @@#define VECTOR_REGISTERS 16#endif+#if 0+#define NOT_SUPPORTED tinyBLAS_not_supported(__FILE__, __LINE__)+#else+#define NOT_SUPPORTED false+#endif+#define WANT_QUANTIZATION false+namespace {+bool tinyBLAS_not_supported(const char *file, int line) {+ tinylogf("%s:%d: tinyBLAS not supported\n", file, line);+ return false;+}+inline float unhalf(ggml_fp16_t d) {return GGML_FP16_TO_FP32(d);}diff --git a/llamafile/tinyblas_cpu_mixmul.inc b/llamafile/tinyblas_cpu_mixmul.incindex bd35e4e1..87d4ba57 100644--- a/llamafile/tinyblas_cpu_mixmul.inc+++ b/llamafile/tinyblas_cpu_mixmul.inc@@ -148,6 +148,8 @@ class MixMul {switch (result->type) {case GGML_TYPE_F32:return mixmuler<float>();+ case GGML_TYPE_BF16:+ return mixmuler<ggml_bf16_t>();case GGML_TYPE_F16:return mixmuler<ggml_fp16_t>();default:diff --git a/llamafile/tinyblas_cpu_sgemm.inc b/llamafile/tinyblas_cpu_sgemm.incindex 27448e7e..84520fd1 100644--- a/llamafile/tinyblas_cpu_sgemm.inc+++ b/llamafile/tinyblas_cpu_sgemm.inc@@ -51,91 +51,95 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, constcase GGML_TYPE_F32: {if (Btype != GGML_TYPE_F32)- return false;+ return NOT_SUPPORTED;#if defined(__AVX512F__)if (k % 16)- return false;+ return NOT_SUPPORTED;tinyBLAS<0, 16, __m512, __m512, float, float, TC> tb{k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__AVX__) || defined(__AVX2__)if (k % 8)- return false;+ return NOT_SUPPORTED;tinyBLAS<0, 8, __m256, __m256, float, float, TC> tb{k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__ARM_NEON)if (k % 4)- return false;+ return NOT_SUPPORTED;tinyBLAS<0, 4, float32x4_t, float32x4_t, float, float, TC> tb{k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#else- return false;+ return NOT_SUPPORTED;#endif}case GGML_TYPE_BF16: {#if defined(__AVX512BF16__)if (k % 32)- return false;+ return NOT_SUPPORTED;if (Btype == GGML_TYPE_F32 && n < 2) {tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, float, TC> tb{k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;}+ if (Btype == GGML_TYPE_F32)+ return WANT_QUANTIZATION;if (Btype != GGML_TYPE_BF16)- return false;+ return NOT_SUPPORTED;tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, TC> tb{k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__AVX512F__)if (k % 16)- return false;+ return NOT_SUPPORTED;tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, TC> tb{k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__AVX2__)if (k % 8)- return false;+ return NOT_SUPPORTED;if (Btype != GGML_TYPE_F32)- return false;+ return NOT_SUPPORTED;tinyBLAS<0, 8, __m256, __m256, ggml_bf16_t, float, TC> tb{k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__ARM_NEON) && !defined(_MSC_VER)if (k % 4)- return false;+ return NOT_SUPPORTED;if (Btype != GGML_TYPE_F32)- return false;+ return NOT_SUPPORTED;tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_bf16_t, float, TC> tb{k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#else- return false;+ return NOT_SUPPORTED;#endif}case GGML_TYPE_F16: {#if defined(__AVX512F__)if (k % 16)- return false;+ return NOT_SUPPORTED;if (Btype == GGML_TYPE_F32 && n < 2) {tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, float, TC> tb{k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;}+ if (Btype == GGML_TYPE_F32)+ return WANT_QUANTIZATION;if (Btype != GGML_TYPE_F16)- return false;+ return NOT_SUPPORTED;tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, TC> tb{k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);@@ -143,40 +147,44 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)if (X86_HAVE(F16C)) {if (k % 8)- return false;+ return NOT_SUPPORTED;if (Btype == GGML_TYPE_F32 && n < 2) {tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, TC> tb{k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;}+ if (Btype == GGML_TYPE_F32)+ return WANT_QUANTIZATION;if (Btype != GGML_TYPE_F16)- return false;+ return NOT_SUPPORTED;tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, TC> tb{k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;} else {- return false;+ return NOT_SUPPORTED;}#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)if (n < 2)// TODO(jart): Why is ggml_vec_dot_f16_unroll() so fast at matvec?- return false;+ return NOT_SUPPORTED;if (precision == GGML_PREC_F32) {if (k % 4)- return false;+ return NOT_SUPPORTED;if (Btype != GGML_TYPE_F32)- return false;+ return NOT_SUPPORTED;tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, TC> tb{k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;} else {if (k % 8)- return false;+ return NOT_SUPPORTED;+ if (Btype == GGML_TYPE_F32)+ return WANT_QUANTIZATION;if (Btype != GGML_TYPE_F16)- return false;+ return NOT_SUPPORTED;tinyBLAS<0, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, TC> tb{k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);@@ -185,23 +193,25 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const#elif defined(__ARM_NEON) && !defined(_MSC_VER)if (n < 2)// TODO(jart): Why is ggml_vec_dot_f16_unroll() so fast at matvec?- return false;+ return NOT_SUPPORTED;if (k % 4)- return false;+ return NOT_SUPPORTED;if (Btype != GGML_TYPE_F32)- return false;+ return NOT_SUPPORTED;tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, TC> tb{k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#else- return false;+ return NOT_SUPPORTED;#endif}case GGML_TYPE_Q8_0: {+ if (Btype == GGML_TYPE_F32)+ return WANT_QUANTIZATION;if (Btype != GGML_TYPE_Q8_0)- return false;+ return NOT_SUPPORTED;#if defined(__AVX2__) || defined(__AVX512F__)tinyBLAS_Q0_AVX2<0, block_q8_0, block_q8_0, TC> tb{k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth};@@ -213,13 +223,15 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, consttb.matmul(m, n, task);return true;#else- return false;+ return NOT_SUPPORTED;#endif}case GGML_TYPE_Q4_0: {+ if (Btype == GGML_TYPE_F32)+ return WANT_QUANTIZATION;if (Btype != GGML_TYPE_Q8_0)- return false;+ return NOT_SUPPORTED;#if defined(__AVX2__) || defined(__AVX512F__)tinyBLAS_Q0_AVX2<0, block_q4_0, block_q8_0, TC> tb{k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth};@@ -231,12 +243,12 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, consttb.matmul(m, n, task);return true;#else- return false;+ return NOT_SUPPORTED;#endif}default:- return false;+ return NOT_SUPPORTED;}(void)m;@@ -305,10 +317,13 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidcase GGML_TYPE_F32:return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (float *)C, ldc, ith, nth, task, Atype,Btype, precision);+ case GGML_TYPE_BF16:+ return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (ggml_bf16_t *)C, ldc, ith, nth, task,+ Atype, Btype, precision);case GGML_TYPE_F16:return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (ggml_fp16_t *)C, ldc, ith, nth, task,Atype, Btype, precision);default:- return false;+ return NOT_SUPPORTED;}}