Skip to content

Distribute and run LLMs with a single file.

Make GGML vector ops go faster across hardware

Justine Tunneyc9d7393f82b9
diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk
index f09c18f5..22443b02 100644
--- a/llama.cpp/BUILD.mk
+++ b/llama.cpp/BUILD.mk
@@ -48,6 +48,15 @@ o/$(MODE)/llama.cpp/ggml-quants-amd-avx.o: private TARGET_ARCH += -Xx86_64-mtune
o/$(MODE)/llama.cpp/ggml-quants-amd-avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2
o/$(MODE)/llama.cpp/ggml-quants-amd-avx512.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f
+o/$(MODE)/llama.cpp/ggml-vector.o: private CXXFLAGS += -Os
+o/$(MODE)/llama.cpp/ggml-vector-amd-avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge
+o/$(MODE)/llama.cpp/ggml-vector-amd-fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mfma
+o/$(MODE)/llama.cpp/ggml-vector-amd-f16c.o: private TARGET_ARCH += -Xx86_64-mtune=ivybridge -Xx86_64-mf16c
+o/$(MODE)/llama.cpp/ggml-vector-amd-avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2
+o/$(MODE)/llama.cpp/ggml-vector-amd-avx512.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f
+o/$(MODE)/llama.cpp/ggml-vector-amd-avx512bf16.o: private TARGET_ARCH += -Xx86_64-mtune=znver4 -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512vl -Xx86_64-mavx512bf16
+o/$(MODE)/llama.cpp/ggml-vector-arm82.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+fp16
+
$(LLAMA_CPP_OBJS): llama.cpp/BUILD.mk
.PHONY: o/$(MODE)/llama.cpp
diff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.h
index 90870cb4..72df530a 100644
--- a/llama.cpp/ggml-impl.h
+++ b/llama.cpp/ggml-impl.h
@@ -19,6 +19,83 @@
#define MIN(a, b) ((a) < (b) ? (a) : (b))
#define MAX(a, b) ((a) > (b) ? (a) : (b))
+/**
+ * Converts brain16 to float32.
+ *
+ * The bfloat16 floating point format has the following structure:
+ *
+ * ┌sign
+ * │
+ * │ ┌exponent
+ * │ │
+ * │ │ ┌mantissa
+ * │ │ │
+ * │┌──┴───┐┌─┴───┐
+ * 0b0000000000000000 brain16
+ *
+ * Since bf16 has the same number of exponent bits as a 32bit float,
+ * encoding and decoding numbers becomes relatively straightforward.
+ *
+ * ┌sign
+ * │
+ * │ ┌exponent
+ * │ │
+ * │ │ ┌mantissa
+ * │ │ │
+ * │┌──┴───┐┌─┴───────────────────┐
+ * 0b00000000000000000000000000000000 IEEE binary32
+ *
+ * For comparison, the standard fp16 format has fewer exponent bits.
+ *
+ * ┌sign
+ * │
+ * │ ┌exponent
+ * │ │
+ * │ │ ┌mantissa
+ * │ │ │
+ * │┌─┴─┐┌─┴──────┐
+ * 0b0000000000000000 IEEE binary16
+ *
+ * @see IEEE 754-2008
+ */
+static inline float ggml_compute_bf16_to_fp32(ggml_bf16_t h) {
+ union {
+ float f;
+ uint32_t i;
+ } u;
+ u.i = (uint32_t)h.bits << 16;
+ return u.f;
+}
+
+/**
+ * Converts float32 to brain16.
+ *
+ * This function is binary identical to AMD Zen4 VCVTNEPS2BF16.
+ * Subnormals shall be flushed to zero, and NANs will be quiet.
+ * This code should vectorize nicely if using modern compilers.
+ */
+static inline ggml_bf16_t ggml_compute_fp32_to_bf16(float s) {
+ ggml_bf16_t h;
+ union {
+ float f;
+ uint32_t i;
+ } u;
+ u.f = s;
+ if ((u.i & 0x7fffffff) > 0x7f800000) { /* nan */
+ h.bits = (u.i >> 16) | 64; /* force to quiet */
+ return h;
+ }
+ if (!(u.i & 0x7f800000)) { /* subnormal */
+ h.bits = (u.i & 0x80000000) >> 16; /* flush to zero */
+ return h;
+ }
+ h.bits = (u.i + (0x7fff + ((u.i >> 16) & 1))) >> 16;
+ return h;
+}
+
+#define GGML_FP32_TO_BF16(x) ggml_compute_fp32_to_bf16(x)
+#define GGML_BF16_TO_FP32(x) ggml_compute_bf16_to_fp32(x)
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -520,9 +597,6 @@ size_t ggml_hash_insert ( struct ggml_hash_set hash_set, struct ggml
// return index, asserts if table is full
size_t ggml_hash_find_or_insert( struct ggml_hash_set hash_set, struct ggml_tensor * key);
-#define GGML_FP32_TO_BF16(x) ggml_fp32_to_bf16(x)
-#define GGML_BF16_TO_FP32(x) ggml_bf16_to_fp32(x)
-
#ifdef __cplusplus
}
#endif
diff --git a/llama.cpp/ggml-quants.py b/llama.cpp/ggml-quants.py
index c3d0a469..9a9d0fd6 100644
--- a/llama.cpp/ggml-quants.py
+++ b/llama.cpp/ggml-quants.py
@@ -42,36 +42,21 @@ with open('llama.cpp/ggml-quants.cpp', 'w') as f:
f.write(' typeof(%s) *ptr_%s;\n' % (func, func))
f.write('\n')
f.write(' QuantFuncs() {\n')
- f.write('#ifdef __x86_64__\n')
for arch, mac, needs in ARCHS:
- if mac == '__x86_64__':
- f.write(' if (%s) {\n' % (' && '.join('X86_HAVE(%s)' % (need) for need in needs) or '1'))
- for func, proto in FUNCS:
- f.write(' ptr_%s = %s_%s;\n' % (func, func, arch))
- f.write(' return;\n')
- f.write(' }\n')
- f.write('#else\n')
- for func, proto in FUNCS:
- f.write(' ptr_%s = %s_arm80;\n' % (func, func))
- f.write('#endif\n')
+ f.write('#ifdef %s\n' % (mac))
+ f.write(' if (%s) {\n' % (' && '.join('X86_HAVE(%s)' % (need) for need in needs) or '1'))
+ for func, proto in FUNCS:
+ f.write(' ptr_%s = %s_%s;\n' % (func, func, arch))
+ f.write(' return;\n')
+ f.write(' }\n')
+ f.write('#endif\n')
f.write(' }\n')
f.write('} funcs;\n')
f.write('\n')
for func, proto in FUNCS:
proto = proto.replace(';', '')
+ args = [s.split(' ')[-1] for s in re.search(r'(?<=\().*(?=\))', proto).group(0).split(',')]
f.write(proto + ' {\n')
- if 'imatrix' in proto:
- args = 'src, dst, nrows, n_per_row, imatrix'
- elif 'quantize' in proto:
- args = 'x, y, k'
- elif 'vec_dot' in proto:
- args = 'n, s, bs, vx, bx, vy, by, nrc'
- elif 'grid_size' in proto:
- args = 'grid_size'
- elif 'validate' in proto:
- args = 'type, data, nbytes'
- else:
- args = 'type'
f.write(' return funcs.ptr_%s(%s);\n' % (func, args))
f.write('}\n')
f.write('\n')
diff --git a/llama.cpp/ggml-vector-amd-avx.c b/llama.cpp/ggml-vector-amd-avx.c
new file mode 100644
index 00000000..a47d29b5
--- /dev/null
+++ b/llama.cpp/ggml-vector-amd-avx.c
@@ -0,0 +1,57 @@
+#ifdef __x86_64__
+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_amd_avx
+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx
+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx
+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx
+#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx
+#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx
+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx
+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_amd_avx
+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_amd_avx
+#define ggml_vec_set_i8 ggml_vec_set_i8_amd_avx
+#define ggml_vec_set_i16 ggml_vec_set_i16_amd_avx
+#define ggml_vec_set_i32 ggml_vec_set_i32_amd_avx
+#define ggml_vec_set_f16 ggml_vec_set_f16_amd_avx
+#define ggml_vec_set_bf16 ggml_vec_set_bf16_amd_avx
+#define ggml_vec_add_f32 ggml_vec_add_f32_amd_avx
+#define ggml_vec_add1_f32 ggml_vec_add1_f32_amd_avx
+#define ggml_vec_acc_f32 ggml_vec_acc_f32_amd_avx
+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_amd_avx
+#define ggml_vec_sub_f32 ggml_vec_sub_f32_amd_avx
+#define ggml_vec_set_f32 ggml_vec_set_f32_amd_avx
+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_amd_avx
+#define ggml_vec_neg_f32 ggml_vec_neg_f32_amd_avx
+#define ggml_vec_mul_f32 ggml_vec_mul_f32_amd_avx
+#define ggml_vec_div_f32 ggml_vec_div_f32_amd_avx
+#define ggml_vec_scale_f32 ggml_vec_scale_f32_amd_avx
+#define ggml_vec_scale_f16 ggml_vec_scale_f16_amd_avx
+#define ggml_vec_mad_f32 ggml_vec_mad_f32_amd_avx
+#define ggml_vec_mad_f16 ggml_vec_mad_f16_amd_avx
+#define ggml_vec_norm_f32 ggml_vec_norm_f32_amd_avx
+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_amd_avx
+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_amd_avx
+#define ggml_vec_log_f32 ggml_vec_log_f32_amd_avx
+#define ggml_vec_abs_f32 ggml_vec_abs_f32_amd_avx
+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_amd_avx
+#define ggml_vec_step_f32 ggml_vec_step_f32_amd_avx
+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_amd_avx
+#define ggml_vec_elu_f32 ggml_vec_elu_f32_amd_avx
+#define ggml_vec_relu_f32 ggml_vec_relu_f32_amd_avx
+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_amd_avx
+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_amd_avx
+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_amd_avx
+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_amd_avx
+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_amd_avx
+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_amd_avx
+#define ggml_vec_silu_f32 ggml_vec_silu_f32_amd_avx
+#define ggml_silu_backward_f32 ggml_silu_backward_f32_amd_avx
+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_amd_avx
+#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_avx
+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_avx
+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_avx
+#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx
+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx
+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx
+#define GGML_VECTOR
+#include "ggml-vector.inc"
+#endif // __x86_64__
diff --git a/llama.cpp/ggml-vector-amd-avx2.c b/llama.cpp/ggml-vector-amd-avx2.c
new file mode 100644
index 00000000..2daeb599
--- /dev/null
+++ b/llama.cpp/ggml-vector-amd-avx2.c
@@ -0,0 +1,57 @@
+#ifdef __x86_64__
+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_amd_avx2
+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx2
+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx2
+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx2
+#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx2
+#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx2
+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx2
+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_amd_avx2
+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_amd_avx2
+#define ggml_vec_set_i8 ggml_vec_set_i8_amd_avx2
+#define ggml_vec_set_i16 ggml_vec_set_i16_amd_avx2
+#define ggml_vec_set_i32 ggml_vec_set_i32_amd_avx2
+#define ggml_vec_set_f16 ggml_vec_set_f16_amd_avx2
+#define ggml_vec_set_bf16 ggml_vec_set_bf16_amd_avx2
+#define ggml_vec_add_f32 ggml_vec_add_f32_amd_avx2
+#define ggml_vec_add1_f32 ggml_vec_add1_f32_amd_avx2
+#define ggml_vec_acc_f32 ggml_vec_acc_f32_amd_avx2
+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_amd_avx2
+#define ggml_vec_sub_f32 ggml_vec_sub_f32_amd_avx2
+#define ggml_vec_set_f32 ggml_vec_set_f32_amd_avx2
+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_amd_avx2
+#define ggml_vec_neg_f32 ggml_vec_neg_f32_amd_avx2
+#define ggml_vec_mul_f32 ggml_vec_mul_f32_amd_avx2
+#define ggml_vec_div_f32 ggml_vec_div_f32_amd_avx2
+#define ggml_vec_scale_f32 ggml_vec_scale_f32_amd_avx2
+#define ggml_vec_scale_f16 ggml_vec_scale_f16_amd_avx2
+#define ggml_vec_mad_f32 ggml_vec_mad_f32_amd_avx2
+#define ggml_vec_mad_f16 ggml_vec_mad_f16_amd_avx2
+#define ggml_vec_norm_f32 ggml_vec_norm_f32_amd_avx2
+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_amd_avx2
+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_amd_avx2
+#define ggml_vec_log_f32 ggml_vec_log_f32_amd_avx2
+#define ggml_vec_abs_f32 ggml_vec_abs_f32_amd_avx2
+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_amd_avx2
+#define ggml_vec_step_f32 ggml_vec_step_f32_amd_avx2
+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_amd_avx2
+#define ggml_vec_elu_f32 ggml_vec_elu_f32_amd_avx2
+#define ggml_vec_relu_f32 ggml_vec_relu_f32_amd_avx2
+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_amd_avx2
+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_amd_avx2
+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_amd_avx2
+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_amd_avx2
+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_amd_avx2
+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_amd_avx2
+#define ggml_vec_silu_f32 ggml_vec_silu_f32_amd_avx2
+#define ggml_silu_backward_f32 ggml_silu_backward_f32_amd_avx2
+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_amd_avx2
+#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_avx2
+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_avx2
+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_avx2
+#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx2
+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx2
+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx2
+#define GGML_VECTOR
+#include "ggml-vector.inc"
+#endif // __x86_64__
diff --git a/llama.cpp/ggml-vector-amd-avx512.c b/llama.cpp/ggml-vector-amd-avx512.c
new file mode 100644
index 00000000..325e6bb3
--- /dev/null
+++ b/llama.cpp/ggml-vector-amd-avx512.c
@@ -0,0 +1,57 @@
+#ifdef __x86_64__
+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_amd_avx512
+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx512
+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx512
+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx512
+#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx512
+#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx512
+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx512
+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_amd_avx512
+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_amd_avx512
+#define ggml_vec_set_i8 ggml_vec_set_i8_amd_avx512
+#define ggml_vec_set_i16 ggml_vec_set_i16_amd_avx512
+#define ggml_vec_set_i32 ggml_vec_set_i32_amd_avx512
+#define ggml_vec_set_f16 ggml_vec_set_f16_amd_avx512
+#define ggml_vec_set_bf16 ggml_vec_set_bf16_amd_avx512
+#define ggml_vec_add_f32 ggml_vec_add_f32_amd_avx512
+#define ggml_vec_add1_f32 ggml_vec_add1_f32_amd_avx512
+#define ggml_vec_acc_f32 ggml_vec_acc_f32_amd_avx512
+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_amd_avx512
+#define ggml_vec_sub_f32 ggml_vec_sub_f32_amd_avx512
+#define ggml_vec_set_f32 ggml_vec_set_f32_amd_avx512
+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_amd_avx512
+#define ggml_vec_neg_f32 ggml_vec_neg_f32_amd_avx512
+#define ggml_vec_mul_f32 ggml_vec_mul_f32_amd_avx512
+#define ggml_vec_div_f32 ggml_vec_div_f32_amd_avx512
+#define ggml_vec_scale_f32 ggml_vec_scale_f32_amd_avx512
+#define ggml_vec_scale_f16 ggml_vec_scale_f16_amd_avx512
+#define ggml_vec_mad_f32 ggml_vec_mad_f32_amd_avx512
+#define ggml_vec_mad_f16 ggml_vec_mad_f16_amd_avx512
+#define ggml_vec_norm_f32 ggml_vec_norm_f32_amd_avx512
+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_amd_avx512
+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_amd_avx512
+#define ggml_vec_log_f32 ggml_vec_log_f32_amd_avx512
+#define ggml_vec_abs_f32 ggml_vec_abs_f32_amd_avx512
+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_amd_avx512
+#define ggml_vec_step_f32 ggml_vec_step_f32_amd_avx512
+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_amd_avx512
+#define ggml_vec_elu_f32 ggml_vec_elu_f32_amd_avx512
+#define ggml_vec_relu_f32 ggml_vec_relu_f32_amd_avx512
+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_amd_avx512
+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_amd_avx512
+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_amd_avx512
+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_amd_avx512
+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_amd_avx512
+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_amd_avx512
+#define ggml_vec_silu_f32 ggml_vec_silu_f32_amd_avx512
+#define ggml_silu_backward_f32 ggml_silu_backward_f32_amd_avx512
+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_amd_avx512
+#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_avx512
+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_avx512
+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_avx512
+#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx512
+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx512
+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx512
+#define GGML_VECTOR
+#include "ggml-vector.inc"
+#endif // __x86_64__
diff --git a/llama.cpp/ggml-vector-amd-avx512bf16.c b/llama.cpp/ggml-vector-amd-avx512bf16.c
new file mode 100644
index 00000000..5f7a3a65
--- /dev/null
+++ b/llama.cpp/ggml-vector-amd-avx512bf16.c
@@ -0,0 +1,57 @@
+#ifdef __x86_64__
+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_amd_avx512bf16
+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx512bf16
+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx512bf16
+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx512bf16
+#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx512bf16
+#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx512bf16
+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx512bf16
+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_amd_avx512bf16
+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_amd_avx512bf16
+#define ggml_vec_set_i8 ggml_vec_set_i8_amd_avx512bf16
+#define ggml_vec_set_i16 ggml_vec_set_i16_amd_avx512bf16
+#define ggml_vec_set_i32 ggml_vec_set_i32_amd_avx512bf16
+#define ggml_vec_set_f16 ggml_vec_set_f16_amd_avx512bf16
+#define ggml_vec_set_bf16 ggml_vec_set_bf16_amd_avx512bf16
+#define ggml_vec_add_f32 ggml_vec_add_f32_amd_avx512bf16
+#define ggml_vec_add1_f32 ggml_vec_add1_f32_amd_avx512bf16
+#define ggml_vec_acc_f32 ggml_vec_acc_f32_amd_avx512bf16
+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_amd_avx512bf16
+#define ggml_vec_sub_f32 ggml_vec_sub_f32_amd_avx512bf16
+#define ggml_vec_set_f32 ggml_vec_set_f32_amd_avx512bf16
+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_amd_avx512bf16
+#define ggml_vec_neg_f32 ggml_vec_neg_f32_amd_avx512bf16
+#define ggml_vec_mul_f32 ggml_vec_mul_f32_amd_avx512bf16
+#define ggml_vec_div_f32 ggml_vec_div_f32_amd_avx512bf16
+#define ggml_vec_scale_f32 ggml_vec_scale_f32_amd_avx512bf16
+#define ggml_vec_scale_f16 ggml_vec_scale_f16_amd_avx512bf16
+#define ggml_vec_mad_f32 ggml_vec_mad_f32_amd_avx512bf16
+#define ggml_vec_mad_f16 ggml_vec_mad_f16_amd_avx512bf16
+#define ggml_vec_norm_f32 ggml_vec_norm_f32_amd_avx512bf16
+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_amd_avx512bf16
+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_amd_avx512bf16
+#define ggml_vec_log_f32 ggml_vec_log_f32_amd_avx512bf16
+#define ggml_vec_abs_f32 ggml_vec_abs_f32_amd_avx512bf16
+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_amd_avx512bf16
+#define ggml_vec_step_f32 ggml_vec_step_f32_amd_avx512bf16
+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_amd_avx512bf16
+#define ggml_vec_elu_f32 ggml_vec_elu_f32_amd_avx512bf16
+#define ggml_vec_relu_f32 ggml_vec_relu_f32_amd_avx512bf16
+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_amd_avx512bf16
+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_amd_avx512bf16
+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_amd_avx512bf16
+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_amd_avx512bf16
+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_amd_avx512bf16
+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_amd_avx512bf16
+#define ggml_vec_silu_f32 ggml_vec_silu_f32_amd_avx512bf16
+#define ggml_silu_backward_f32 ggml_silu_backward_f32_amd_avx512bf16
+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_amd_avx512bf16
+#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_avx512bf16
+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_avx512bf16
+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_avx512bf16
+#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx512bf16
+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx512bf16
+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx512bf16
+#define GGML_VECTOR
+#include "ggml-vector.inc"
+#endif // __x86_64__
diff --git a/llama.cpp/ggml-vector-amd-f16c.c b/llama.cpp/ggml-vector-amd-f16c.c
new file mode 100644
index 00000000..5a0f2c9f
--- /dev/null
+++ b/llama.cpp/ggml-vector-amd-f16c.c
@@ -0,0 +1,57 @@
+#ifdef __x86_64__
+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_amd_f16c
+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_f16c
+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_f16c
+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_f16c
+#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_f16c
+#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_f16c
+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_f16c
+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_amd_f16c
+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_amd_f16c
+#define ggml_vec_set_i8 ggml_vec_set_i8_amd_f16c
+#define ggml_vec_set_i16 ggml_vec_set_i16_amd_f16c
+#define ggml_vec_set_i32 ggml_vec_set_i32_amd_f16c
+#define ggml_vec_set_f16 ggml_vec_set_f16_amd_f16c
+#define ggml_vec_set_bf16 ggml_vec_set_bf16_amd_f16c
+#define ggml_vec_add_f32 ggml_vec_add_f32_amd_f16c
+#define ggml_vec_add1_f32 ggml_vec_add1_f32_amd_f16c
+#define ggml_vec_acc_f32 ggml_vec_acc_f32_amd_f16c
+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_amd_f16c
+#define ggml_vec_sub_f32 ggml_vec_sub_f32_amd_f16c
+#define ggml_vec_set_f32 ggml_vec_set_f32_amd_f16c
+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_amd_f16c
+#define ggml_vec_neg_f32 ggml_vec_neg_f32_amd_f16c
+#define ggml_vec_mul_f32 ggml_vec_mul_f32_amd_f16c
+#define ggml_vec_div_f32 ggml_vec_div_f32_amd_f16c
+#define ggml_vec_scale_f32 ggml_vec_scale_f32_amd_f16c
+#define ggml_vec_scale_f16 ggml_vec_scale_f16_amd_f16c
+#define ggml_vec_mad_f32 ggml_vec_mad_f32_amd_f16c
+#define ggml_vec_mad_f16 ggml_vec_mad_f16_amd_f16c
+#define ggml_vec_norm_f32 ggml_vec_norm_f32_amd_f16c
+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_amd_f16c
+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_amd_f16c
+#define ggml_vec_log_f32 ggml_vec_log_f32_amd_f16c
+#define ggml_vec_abs_f32 ggml_vec_abs_f32_amd_f16c
+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_amd_f16c
+#define ggml_vec_step_f32 ggml_vec_step_f32_amd_f16c
+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_amd_f16c
+#define ggml_vec_elu_f32 ggml_vec_elu_f32_amd_f16c
+#define ggml_vec_relu_f32 ggml_vec_relu_f32_amd_f16c
+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_amd_f16c
+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_amd_f16c
+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_amd_f16c
+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_amd_f16c
+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_amd_f16c
+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_amd_f16c
+#define ggml_vec_silu_f32 ggml_vec_silu_f32_amd_f16c
+#define ggml_silu_backward_f32 ggml_silu_backward_f32_amd_f16c
+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_amd_f16c
+#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_f16c
+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_f16c
+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_f16c
+#define ggml_vec_max_f32 ggml_vec_max_f32_amd_f16c
+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_f16c
+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_f16c
+#define GGML_VECTOR
+#include "ggml-vector.inc"
+#endif // __x86_64__
diff --git a/llama.cpp/ggml-vector-amd-fma.c b/llama.cpp/ggml-vector-amd-fma.c
new file mode 100644
index 00000000..77e72ada
--- /dev/null
+++ b/llama.cpp/ggml-vector-amd-fma.c
@@ -0,0 +1,57 @@
+#ifdef __x86_64__
+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_amd_fma
+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_fma
+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_fma
+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_fma
+#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_fma
+#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_fma
+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_fma
+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_amd_fma
+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_amd_fma
+#define ggml_vec_set_i8 ggml_vec_set_i8_amd_fma
+#define ggml_vec_set_i16 ggml_vec_set_i16_amd_fma
+#define ggml_vec_set_i32 ggml_vec_set_i32_amd_fma
+#define ggml_vec_set_f16 ggml_vec_set_f16_amd_fma
+#define ggml_vec_set_bf16 ggml_vec_set_bf16_amd_fma
+#define ggml_vec_add_f32 ggml_vec_add_f32_amd_fma
+#define ggml_vec_add1_f32 ggml_vec_add1_f32_amd_fma
+#define ggml_vec_acc_f32 ggml_vec_acc_f32_amd_fma
+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_amd_fma
+#define ggml_vec_sub_f32 ggml_vec_sub_f32_amd_fma
+#define ggml_vec_set_f32 ggml_vec_set_f32_amd_fma
+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_amd_fma
+#define ggml_vec_neg_f32 ggml_vec_neg_f32_amd_fma
+#define ggml_vec_mul_f32 ggml_vec_mul_f32_amd_fma
+#define ggml_vec_div_f32 ggml_vec_div_f32_amd_fma
+#define ggml_vec_scale_f32 ggml_vec_scale_f32_amd_fma
+#define ggml_vec_scale_f16 ggml_vec_scale_f16_amd_fma
+#define ggml_vec_mad_f32 ggml_vec_mad_f32_amd_fma
+#define ggml_vec_mad_f16 ggml_vec_mad_f16_amd_fma
+#define ggml_vec_norm_f32 ggml_vec_norm_f32_amd_fma
+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_amd_fma
+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_amd_fma
+#define ggml_vec_log_f32 ggml_vec_log_f32_amd_fma
+#define ggml_vec_abs_f32 ggml_vec_abs_f32_amd_fma
+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_amd_fma
+#define ggml_vec_step_f32 ggml_vec_step_f32_amd_fma
+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_amd_fma
+#define ggml_vec_elu_f32 ggml_vec_elu_f32_amd_fma
+#define ggml_vec_relu_f32 ggml_vec_relu_f32_amd_fma
+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_amd_fma
+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_amd_fma
+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_amd_fma
+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_amd_fma
+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_amd_fma
+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_amd_fma
+#define ggml_vec_silu_f32 ggml_vec_silu_f32_amd_fma
+#define ggml_silu_backward_f32 ggml_silu_backward_f32_amd_fma
+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_amd_fma
+#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_fma
+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_fma
+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_fma
+#define ggml_vec_max_f32 ggml_vec_max_f32_amd_fma
+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_fma
+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_fma
+#define GGML_VECTOR
+#include "ggml-vector.inc"
+#endif // __x86_64__
diff --git a/llama.cpp/ggml-vector-arm80.c b/llama.cpp/ggml-vector-arm80.c
new file mode 100644
index 00000000..64d51d24
--- /dev/null
+++ b/llama.cpp/ggml-vector-arm80.c
@@ -0,0 +1,57 @@
+#ifdef __aarch64__
+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_arm80
+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_arm80
+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_arm80
+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_arm80
+#define ggml_vec_dot_f32 ggml_vec_dot_f32_arm80
+#define ggml_vec_dot_f16 ggml_vec_dot_f16_arm80
+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_arm80
+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_arm80
+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_arm80
+#define ggml_vec_set_i8 ggml_vec_set_i8_arm80
+#define ggml_vec_set_i16 ggml_vec_set_i16_arm80
+#define ggml_vec_set_i32 ggml_vec_set_i32_arm80
+#define ggml_vec_set_f16 ggml_vec_set_f16_arm80
+#define ggml_vec_set_bf16 ggml_vec_set_bf16_arm80
+#define ggml_vec_add_f32 ggml_vec_add_f32_arm80
+#define ggml_vec_add1_f32 ggml_vec_add1_f32_arm80
+#define ggml_vec_acc_f32 ggml_vec_acc_f32_arm80
+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_arm80
+#define ggml_vec_sub_f32 ggml_vec_sub_f32_arm80
+#define ggml_vec_set_f32 ggml_vec_set_f32_arm80
+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_arm80
+#define ggml_vec_neg_f32 ggml_vec_neg_f32_arm80
+#define ggml_vec_mul_f32 ggml_vec_mul_f32_arm80
+#define ggml_vec_div_f32 ggml_vec_div_f32_arm80
+#define ggml_vec_scale_f32 ggml_vec_scale_f32_arm80
+#define ggml_vec_scale_f16 ggml_vec_scale_f16_arm80
+#define ggml_vec_mad_f32 ggml_vec_mad_f32_arm80
+#define ggml_vec_mad_f16 ggml_vec_mad_f16_arm80
+#define ggml_vec_norm_f32 ggml_vec_norm_f32_arm80
+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_arm80
+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_arm80
+#define ggml_vec_log_f32 ggml_vec_log_f32_arm80
+#define ggml_vec_abs_f32 ggml_vec_abs_f32_arm80
+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_arm80
+#define ggml_vec_step_f32 ggml_vec_step_f32_arm80
+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_arm80
+#define ggml_vec_elu_f32 ggml_vec_elu_f32_arm80
+#define ggml_vec_relu_f32 ggml_vec_relu_f32_arm80
+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_arm80
+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_arm80
+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_arm80
+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_arm80
+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_arm80
+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_arm80
+#define ggml_vec_silu_f32 ggml_vec_silu_f32_arm80
+#define ggml_silu_backward_f32 ggml_silu_backward_f32_arm80
+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_arm80
+#define ggml_vec_sum_f32 ggml_vec_sum_f32_arm80
+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_arm80
+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_arm80
+#define ggml_vec_max_f32 ggml_vec_max_f32_arm80
+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_arm80
+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_arm80
+#define GGML_VECTOR
+#include "ggml-vector.inc"
+#endif // __aarch64__
diff --git a/llama.cpp/ggml-vector-arm82.c b/llama.cpp/ggml-vector-arm82.c
new file mode 100644
index 00000000..fa07a23f
--- /dev/null
+++ b/llama.cpp/ggml-vector-arm82.c
@@ -0,0 +1,57 @@
+#ifdef __aarch64__
+#define ggml_fp16_to_fp32_row ggml_fp16_to_fp32_row_arm82
+#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_arm82
+#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_arm82
+#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_arm82
+#define ggml_vec_dot_f32 ggml_vec_dot_f32_arm82
+#define ggml_vec_dot_f16 ggml_vec_dot_f16_arm82
+#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_arm82
+#define ggml_vec_dot_f16_unroll ggml_vec_dot_f16_unroll_arm82
+#define ggml_vec_mad_f32_unroll ggml_vec_mad_f32_unroll_arm82
+#define ggml_vec_set_i8 ggml_vec_set_i8_arm82
+#define ggml_vec_set_i16 ggml_vec_set_i16_arm82
+#define ggml_vec_set_i32 ggml_vec_set_i32_arm82
+#define ggml_vec_set_f16 ggml_vec_set_f16_arm82
+#define ggml_vec_set_bf16 ggml_vec_set_bf16_arm82
+#define ggml_vec_add_f32 ggml_vec_add_f32_arm82
+#define ggml_vec_add1_f32 ggml_vec_add1_f32_arm82
+#define ggml_vec_acc_f32 ggml_vec_acc_f32_arm82
+#define ggml_vec_acc1_f32 ggml_vec_acc1_f32_arm82
+#define ggml_vec_sub_f32 ggml_vec_sub_f32_arm82
+#define ggml_vec_set_f32 ggml_vec_set_f32_arm82
+#define ggml_vec_cpy_f32 ggml_vec_cpy_f32_arm82
+#define ggml_vec_neg_f32 ggml_vec_neg_f32_arm82
+#define ggml_vec_mul_f32 ggml_vec_mul_f32_arm82
+#define ggml_vec_div_f32 ggml_vec_div_f32_arm82
+#define ggml_vec_scale_f32 ggml_vec_scale_f32_arm82
+#define ggml_vec_scale_f16 ggml_vec_scale_f16_arm82
+#define ggml_vec_mad_f32 ggml_vec_mad_f32_arm82
+#define ggml_vec_mad_f16 ggml_vec_mad_f16_arm82
+#define ggml_vec_norm_f32 ggml_vec_norm_f32_arm82
+#define ggml_vec_sqr_f32 ggml_vec_sqr_f32_arm82
+#define ggml_vec_sqrt_f32 ggml_vec_sqrt_f32_arm82
+#define ggml_vec_log_f32 ggml_vec_log_f32_arm82
+#define ggml_vec_abs_f32 ggml_vec_abs_f32_arm82
+#define ggml_vec_sgn_f32 ggml_vec_sgn_f32_arm82
+#define ggml_vec_step_f32 ggml_vec_step_f32_arm82
+#define ggml_vec_tanh_f32 ggml_vec_tanh_f32_arm82
+#define ggml_vec_elu_f32 ggml_vec_elu_f32_arm82
+#define ggml_vec_relu_f32 ggml_vec_relu_f32_arm82
+#define ggml_vec_leaky_relu_f32 ggml_vec_leaky_relu_f32_arm82
+#define ggml_vec_hardswish_f32 ggml_vec_hardswish_f32_arm82
+#define ggml_vec_hardsigmoid_f32 ggml_vec_hardsigmoid_f32_arm82
+#define ggml_vec_gelu_f16 ggml_vec_gelu_f16_arm82
+#define ggml_vec_gelu_f32 ggml_vec_gelu_f32_arm82
+#define ggml_vec_gelu_quick_f32 ggml_vec_gelu_quick_f32_arm82
+#define ggml_vec_silu_f32 ggml_vec_silu_f32_arm82
+#define ggml_silu_backward_f32 ggml_silu_backward_f32_arm82
+#define ggml_vec_silu_backward_f32 ggml_vec_silu_backward_f32_arm82
+#define ggml_vec_sum_f32 ggml_vec_sum_f32_arm82
+#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_arm82
+#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_arm82
+#define ggml_vec_max_f32 ggml_vec_max_f32_arm82
+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_arm82
+#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_arm82
+#define GGML_VECTOR
+#include "ggml-vector.inc"
+#endif // __aarch64__
diff --git a/llama.cpp/ggml-vector.cpp b/llama.cpp/ggml-vector.cpp
new file mode 100644
index 00000000..a6aed0cb
--- /dev/null
+++ b/llama.cpp/ggml-vector.cpp
@@ -0,0 +1,1217 @@
+#include <cosmo.h>
+#include <sys/auxv.h>
+#include <libc/sysv/consts/hwcap.h>
+#include "ggml-vector.h"
+
+extern "C" void ggml_fp16_to_fp32_row_amd_avx512bf16(const ggml_fp16_t * x, float * y, int64_t n);
+extern "C" void ggml_fp16_to_fp32_row_amd_avx512(const ggml_fp16_t * x, float * y, int64_t n);
+extern "C" void ggml_fp16_to_fp32_row_amd_avx2(const ggml_fp16_t * x, float * y, int64_t n);
+extern "C" void ggml_fp16_to_fp32_row_amd_f16c(const ggml_fp16_t * x, float * y, int64_t n);
+extern "C" void ggml_fp16_to_fp32_row_amd_fma(const ggml_fp16_t * x, float * y, int64_t n);
+extern "C" void ggml_fp16_to_fp32_row_amd_avx(const ggml_fp16_t * x, float * y, int64_t n);
+extern "C" void ggml_fp16_to_fp32_row_arm82(const ggml_fp16_t * x, float * y, int64_t n);
+extern "C" void ggml_fp16_to_fp32_row_arm80(const ggml_fp16_t * x, float * y, int64_t n);
+
+extern "C" void ggml_fp32_to_fp16_row_amd_avx512bf16(const float * x, ggml_fp16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_fp16_row_amd_avx512(const float * x, ggml_fp16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_fp16_row_amd_avx2(const float * x, ggml_fp16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_fp16_row_amd_f16c(const float * x, ggml_fp16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_fp16_row_amd_fma(const float * x, ggml_fp16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_fp16_row_amd_avx(const float * x, ggml_fp16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_fp16_row_arm82(const float * x, ggml_fp16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_fp16_row_arm80(const float * x, ggml_fp16_t * y, int64_t n);
+
+extern "C" void ggml_bf16_to_fp32_row_amd_avx512bf16(const ggml_bf16_t * x, float * y, int64_t n);
+extern "C" void ggml_bf16_to_fp32_row_amd_avx512(const ggml_bf16_t * x, float * y, int64_t n);
+extern "C" void ggml_bf16_to_fp32_row_amd_avx2(const ggml_bf16_t * x, float * y, int64_t n);
+extern "C" void ggml_bf16_to_fp32_row_amd_f16c(const ggml_bf16_t * x, float * y, int64_t n);
+extern "C" void ggml_bf16_to_fp32_row_amd_fma(const ggml_bf16_t * x, float * y, int64_t n);
+extern "C" void ggml_bf16_to_fp32_row_amd_avx(const ggml_bf16_t * x, float * y, int64_t n);
+extern "C" void ggml_bf16_to_fp32_row_arm82(const ggml_bf16_t * x, float * y, int64_t n);
+extern "C" void ggml_bf16_to_fp32_row_arm80(const ggml_bf16_t * x, float * y, int64_t n);
+
+extern "C" void ggml_fp32_to_bf16_row_amd_avx512bf16(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_amd_avx512(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_amd_avx2(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_amd_f16c(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_amd_fma(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_amd_avx(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_arm82(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_arm80(const float * x, ggml_bf16_t * y, int64_t n);
+
+extern "C" void ggml_vec_dot_f32_amd_avx512bf16(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_amd_avx512(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_amd_avx2(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_amd_f16c(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_amd_fma(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_amd_avx(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_arm82(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f32_arm80(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_f16_amd_avx512bf16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_amd_avx512(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_amd_avx2(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_amd_f16c(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_amd_fma(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_amd_avx(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_arm82(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_f16_arm80(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_bf16_amd_avx512bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_amd_avx512(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_amd_avx2(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_amd_f16c(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_amd_fma(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_amd_avx(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_arm82(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
+extern "C" void ggml_vec_dot_bf16_arm80(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
+
+extern "C" void ggml_vec_dot_f16_unroll_amd_avx512bf16(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
+extern "C" void ggml_vec_dot_f16_unroll_amd_avx512(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
+extern "C" void ggml_vec_dot_f16_unroll_amd_avx2(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
+extern "C" void ggml_vec_dot_f16_unroll_amd_f16c(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
+extern "C" void ggml_vec_dot_f16_unroll_amd_fma(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
+extern "C" void ggml_vec_dot_f16_unroll_amd_avx(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
+extern "C" void ggml_vec_dot_f16_unroll_arm82(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
+extern "C" void ggml_vec_dot_f16_unroll_arm80(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
+
+extern "C" void ggml_vec_mad_f32_unroll_amd_avx512bf16(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
+extern "C" void ggml_vec_mad_f32_unroll_amd_avx512(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
+extern "C" void ggml_vec_mad_f32_unroll_amd_avx2(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
+extern "C" void ggml_vec_mad_f32_unroll_amd_f16c(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
+extern "C" void ggml_vec_mad_f32_unroll_amd_fma(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
+extern "C" void ggml_vec_mad_f32_unroll_amd_avx(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
+extern "C" void ggml_vec_mad_f32_unroll_arm82(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
+extern "C" void ggml_vec_mad_f32_unroll_arm80(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
+
+extern "C" void ggml_vec_set_i8_amd_avx512bf16(const int n, int8_t * x, const int8_t v);
+extern "C" void ggml_vec_set_i8_amd_avx512(const int n, int8_t * x, const int8_t v);
+extern "C" void ggml_vec_set_i8_amd_avx2(const int n, int8_t * x, const int8_t v);
+extern "C" void ggml_vec_set_i8_amd_f16c(const int n, int8_t * x, const int8_t v);
+extern "C" void ggml_vec_set_i8_amd_fma(const int n, int8_t * x, const int8_t v);
+extern "C" void ggml_vec_set_i8_amd_avx(const int n, int8_t * x, const int8_t v);
+extern "C" void ggml_vec_set_i8_arm82(const int n, int8_t * x, const int8_t v);
+extern "C" void ggml_vec_set_i8_arm80(const int n, int8_t * x, const int8_t v);
+
+extern "C" void ggml_vec_set_i16_amd_avx512bf16(const int n, int16_t * x, const int16_t v);
+extern "C" void ggml_vec_set_i16_amd_avx512(const int n, int16_t * x, const int16_t v);
+extern "C" void ggml_vec_set_i16_amd_avx2(const int n, int16_t * x, const int16_t v);
+extern "C" void ggml_vec_set_i16_amd_f16c(const int n, int16_t * x, const int16_t v);
+extern "C" void ggml_vec_set_i16_amd_fma(const int n, int16_t * x, const int16_t v);
+extern "C" void ggml_vec_set_i16_amd_avx(const int n, int16_t * x, const int16_t v);
+extern "C" void ggml_vec_set_i16_arm82(const int n, int16_t * x, const int16_t v);
+extern "C" void ggml_vec_set_i16_arm80(const int n, int16_t * x, const int16_t v);
+
+extern "C" void ggml_vec_set_i32_amd_avx512bf16(const int n, int32_t * x, const int32_t v);
+extern "C" void ggml_vec_set_i32_amd_avx512(const int n, int32_t * x, const int32_t v);
+extern "C" void ggml_vec_set_i32_amd_avx2(const int n, int32_t * x, const int32_t v);
+extern "C" void ggml_vec_set_i32_amd_f16c(const int n, int32_t * x, const int32_t v);
+extern "C" void ggml_vec_set_i32_amd_fma(const int n, int32_t * x, const int32_t v);
+extern "C" void ggml_vec_set_i32_amd_avx(const int n, int32_t * x, const int32_t v);
+extern "C" void ggml_vec_set_i32_arm82(const int n, int32_t * x, const int32_t v);
+extern "C" void ggml_vec_set_i32_arm80(const int n, int32_t * x, const int32_t v);
+
+extern "C" void ggml_vec_set_f16_amd_avx512bf16(const int n, ggml_fp16_t * x, const int32_t v);
+extern "C" void ggml_vec_set_f16_amd_avx512(const int n, ggml_fp16_t * x, const int32_t v);
+extern "C" void ggml_vec_set_f16_amd_avx2(const int n, ggml_fp16_t * x, const int32_t v);
+extern "C" void ggml_vec_set_f16_amd_f16c(const int n, ggml_fp16_t * x, const int32_t v);
+extern "C" void ggml_vec_set_f16_amd_fma(const int n, ggml_fp16_t * x, const int32_t v);
+extern "C" void ggml_vec_set_f16_amd_avx(const int n, ggml_fp16_t * x, const int32_t v);
+extern "C" void ggml_vec_set_f16_arm82(const int n, ggml_fp16_t * x, const int32_t v);
+extern "C" void ggml_vec_set_f16_arm80(const int n, ggml_fp16_t * x, const int32_t v);
+
+extern "C" void ggml_vec_set_bf16_amd_avx512bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
+extern "C" void ggml_vec_set_bf16_amd_avx512(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
+extern "C" void ggml_vec_set_bf16_amd_avx2(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
+extern "C" void ggml_vec_set_bf16_amd_f16c(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
+extern "C" void ggml_vec_set_bf16_amd_fma(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
+extern "C" void ggml_vec_set_bf16_amd_avx(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
+extern "C" void ggml_vec_set_bf16_arm82(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
+extern "C" void ggml_vec_set_bf16_arm80(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
+
+extern "C" void ggml_vec_add_f32_amd_avx512bf16 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_add_f32_amd_avx512 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_add_f32_amd_avx2 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_add_f32_amd_f16c (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_add_f32_amd_fma (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_add_f32_amd_avx (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_add_f32_arm82 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_add_f32_arm80 (const int n, float * z, const float * x, const float * y);
+
+extern "C" void ggml_vec_add1_f32_amd_avx512bf16(const int n, float * z, const float * x, const float v);
+extern "C" void ggml_vec_add1_f32_amd_avx512(const int n, float * z, const float * x, const float v);
+extern "C" void ggml_vec_add1_f32_amd_avx2(const int n, float * z, const float * x, const float v);
+extern "C" void ggml_vec_add1_f32_amd_f16c(const int n, float * z, const float * x, const float v);
+extern "C" void ggml_vec_add1_f32_amd_fma(const int n, float * z, const float * x, const float v);
+extern "C" void ggml_vec_add1_f32_amd_avx(const int n, float * z, const float * x, const float v);
+extern "C" void ggml_vec_add1_f32_arm82(const int n, float * z, const float * x, const float v);
+extern "C" void ggml_vec_add1_f32_arm80(const int n, float * z, const float * x, const float v);
+
+extern "C" void ggml_vec_acc_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_acc_f32_amd_avx512 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_acc_f32_amd_avx2 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_acc_f32_amd_f16c (const int n, float * y, const float * x);
+extern "C" void ggml_vec_acc_f32_amd_fma (const int n, float * y, const float * x);
+extern "C" void ggml_vec_acc_f32_amd_avx (const int n, float * y, const float * x);
+extern "C" void ggml_vec_acc_f32_arm82 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_acc_f32_arm80 (const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_acc1_f32_amd_avx512bf16(const int n, float * y, const float v);
+extern "C" void ggml_vec_acc1_f32_amd_avx512(const int n, float * y, const float v);
+extern "C" void ggml_vec_acc1_f32_amd_avx2(const int n, float * y, const float v);
+extern "C" void ggml_vec_acc1_f32_amd_f16c(const int n, float * y, const float v);
+extern "C" void ggml_vec_acc1_f32_amd_fma(const int n, float * y, const float v);
+extern "C" void ggml_vec_acc1_f32_amd_avx(const int n, float * y, const float v);
+extern "C" void ggml_vec_acc1_f32_arm82(const int n, float * y, const float v);
+extern "C" void ggml_vec_acc1_f32_arm80(const int n, float * y, const float v);
+
+extern "C" void ggml_vec_sub_f32_amd_avx512bf16 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_sub_f32_amd_avx512 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_sub_f32_amd_avx2 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_sub_f32_amd_f16c (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_sub_f32_amd_fma (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_sub_f32_amd_avx (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_sub_f32_arm82 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_sub_f32_arm80 (const int n, float * z, const float * x, const float * y);
+
+extern "C" void ggml_vec_set_f32_amd_avx512bf16 (const int n, float * x, const float v);
+extern "C" void ggml_vec_set_f32_amd_avx512 (const int n, float * x, const float v);
+extern "C" void ggml_vec_set_f32_amd_avx2 (const int n, float * x, const float v);
+extern "C" void ggml_vec_set_f32_amd_f16c (const int n, float * x, const float v);
+extern "C" void ggml_vec_set_f32_amd_fma (const int n, float * x, const float v);
+extern "C" void ggml_vec_set_f32_amd_avx (const int n, float * x, const float v);
+extern "C" void ggml_vec_set_f32_arm82 (const int n, float * x, const float v);
+extern "C" void ggml_vec_set_f32_arm80 (const int n, float * x, const float v);
+
+extern "C" void ggml_vec_cpy_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_cpy_f32_amd_avx512 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_cpy_f32_amd_avx2 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_cpy_f32_amd_f16c (const int n, float * y, const float * x);
+extern "C" void ggml_vec_cpy_f32_amd_fma (const int n, float * y, const float * x);
+extern "C" void ggml_vec_cpy_f32_amd_avx (const int n, float * y, const float * x);
+extern "C" void ggml_vec_cpy_f32_arm82 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_cpy_f32_arm80 (const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_neg_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_neg_f32_amd_avx512 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_neg_f32_amd_avx2 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_neg_f32_amd_f16c (const int n, float * y, const float * x);
+extern "C" void ggml_vec_neg_f32_amd_fma (const int n, float * y, const float * x);
+extern "C" void ggml_vec_neg_f32_amd_avx (const int n, float * y, const float * x);
+extern "C" void ggml_vec_neg_f32_arm82 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_neg_f32_arm80 (const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_mul_f32_amd_avx512bf16 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_mul_f32_amd_avx512 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_mul_f32_amd_avx2 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_mul_f32_amd_f16c (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_mul_f32_amd_fma (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_mul_f32_amd_avx (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_mul_f32_arm82 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_mul_f32_arm80 (const int n, float * z, const float * x, const float * y);
+
+extern "C" void ggml_vec_div_f32_amd_avx512bf16 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_div_f32_amd_avx512 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_div_f32_amd_avx2 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_div_f32_amd_f16c (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_div_f32_amd_fma (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_div_f32_amd_avx (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_div_f32_arm82 (const int n, float * z, const float * x, const float * y);
+extern "C" void ggml_vec_div_f32_arm80 (const int n, float * z, const float * x, const float * y);
+
+extern "C" void ggml_vec_scale_f32_amd_avx512bf16(const int n, float * y, const float v);
+extern "C" void ggml_vec_scale_f32_amd_avx512(const int n, float * y, const float v);
+extern "C" void ggml_vec_scale_f32_amd_avx2(const int n, float * y, const float v);
+extern "C" void ggml_vec_scale_f32_amd_f16c(const int n, float * y, const float v);
+extern "C" void ggml_vec_scale_f32_amd_fma(const int n, float * y, const float v);
+extern "C" void ggml_vec_scale_f32_amd_avx(const int n, float * y, const float v);
+extern "C" void ggml_vec_scale_f32_arm82(const int n, float * y, const float v);
+extern "C" void ggml_vec_scale_f32_arm80(const int n, float * y, const float v);
+
+extern "C" void ggml_vec_scale_f16_amd_avx512bf16(const int n, ggml_fp16_t * y, const float v);
+extern "C" void ggml_vec_scale_f16_amd_avx512(const int n, ggml_fp16_t * y, const float v);
+extern "C" void ggml_vec_scale_f16_amd_avx2(const int n, ggml_fp16_t * y, const float v);
+extern "C" void ggml_vec_scale_f16_amd_f16c(const int n, ggml_fp16_t * y, const float v);
+extern "C" void ggml_vec_scale_f16_amd_fma(const int n, ggml_fp16_t * y, const float v);
+extern "C" void ggml_vec_scale_f16_amd_avx(const int n, ggml_fp16_t * y, const float v);
+extern "C" void ggml_vec_scale_f16_arm82(const int n, ggml_fp16_t * y, const float v);
+extern "C" void ggml_vec_scale_f16_arm80(const int n, ggml_fp16_t * y, const float v);
+
+extern "C" void ggml_vec_mad_f32_amd_avx512bf16(const int n, float * restrict y, const float * restrict x, const float v);
+extern "C" void ggml_vec_mad_f32_amd_avx512(const int n, float * restrict y, const float * restrict x, const float v);
+extern "C" void ggml_vec_mad_f32_amd_avx2(const int n, float * restrict y, const float * restrict x, const float v);
+extern "C" void ggml_vec_mad_f32_amd_f16c(const int n, float * restrict y, const float * restrict x, const float v);
+extern "C" void ggml_vec_mad_f32_amd_fma(const int n, float * restrict y, const float * restrict x, const float v);
+extern "C" void ggml_vec_mad_f32_amd_avx(const int n, float * restrict y, const float * restrict x, const float v);
+extern "C" void ggml_vec_mad_f32_arm82(const int n, float * restrict y, const float * restrict x, const float v);
+extern "C" void ggml_vec_mad_f32_arm80(const int n, float * restrict y, const float * restrict x, const float v);
+
+extern "C" void ggml_vec_mad_f16_amd_avx512bf16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
+extern "C" void ggml_vec_mad_f16_amd_avx512(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
+extern "C" void ggml_vec_mad_f16_amd_avx2(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
+extern "C" void ggml_vec_mad_f16_amd_f16c(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
+extern "C" void ggml_vec_mad_f16_amd_fma(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
+extern "C" void ggml_vec_mad_f16_amd_avx(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
+extern "C" void ggml_vec_mad_f16_arm82(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
+extern "C" void ggml_vec_mad_f16_arm80(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
+
+extern "C" void ggml_vec_norm_f32_amd_avx512bf16 (const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_f32_amd_avx512 (const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_f32_amd_avx2 (const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_f32_amd_f16c (const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_f32_amd_fma (const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_f32_amd_avx (const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_f32_arm82 (const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_f32_arm80 (const int n, float * s, const float * x);
+
+extern "C" void ggml_vec_sqr_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqr_f32_amd_avx512 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqr_f32_amd_avx2 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqr_f32_amd_f16c (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqr_f32_amd_fma (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqr_f32_amd_avx (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqr_f32_arm82 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqr_f32_arm80 (const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_sqrt_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqrt_f32_amd_avx512 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqrt_f32_amd_avx2 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqrt_f32_amd_f16c (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqrt_f32_amd_fma (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqrt_f32_amd_avx (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqrt_f32_arm82 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sqrt_f32_arm80 (const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_log_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_log_f32_amd_avx512 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_log_f32_amd_avx2 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_log_f32_amd_f16c (const int n, float * y, const float * x);
+extern "C" void ggml_vec_log_f32_amd_fma (const int n, float * y, const float * x);
+extern "C" void ggml_vec_log_f32_amd_avx (const int n, float * y, const float * x);
+extern "C" void ggml_vec_log_f32_arm82 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_log_f32_arm80 (const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_abs_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_abs_f32_amd_avx512 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_abs_f32_amd_avx2 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_abs_f32_amd_f16c (const int n, float * y, const float * x);
+extern "C" void ggml_vec_abs_f32_amd_fma (const int n, float * y, const float * x);
+extern "C" void ggml_vec_abs_f32_amd_avx (const int n, float * y, const float * x);
+extern "C" void ggml_vec_abs_f32_arm82 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_abs_f32_arm80 (const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_sgn_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sgn_f32_amd_avx512 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sgn_f32_amd_avx2 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sgn_f32_amd_f16c (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sgn_f32_amd_fma (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sgn_f32_amd_avx (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sgn_f32_arm82 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_sgn_f32_arm80 (const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_step_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_step_f32_amd_avx512 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_step_f32_amd_avx2 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_step_f32_amd_f16c (const int n, float * y, const float * x);
+extern "C" void ggml_vec_step_f32_amd_fma (const int n, float * y, const float * x);
+extern "C" void ggml_vec_step_f32_amd_avx (const int n, float * y, const float * x);
+extern "C" void ggml_vec_step_f32_arm82 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_step_f32_arm80 (const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_tanh_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_tanh_f32_amd_avx512 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_tanh_f32_amd_avx2 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_tanh_f32_amd_f16c (const int n, float * y, const float * x);
+extern "C" void ggml_vec_tanh_f32_amd_fma (const int n, float * y, const float * x);
+extern "C" void ggml_vec_tanh_f32_amd_avx (const int n, float * y, const float * x);
+extern "C" void ggml_vec_tanh_f32_arm82 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_tanh_f32_arm80 (const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_elu_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_elu_f32_amd_avx512 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_elu_f32_amd_avx2 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_elu_f32_amd_f16c (const int n, float * y, const float * x);
+extern "C" void ggml_vec_elu_f32_amd_fma (const int n, float * y, const float * x);
+extern "C" void ggml_vec_elu_f32_amd_avx (const int n, float * y, const float * x);
+extern "C" void ggml_vec_elu_f32_arm82 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_elu_f32_arm80 (const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_relu_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_relu_f32_amd_avx512 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_relu_f32_amd_avx2 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_relu_f32_amd_f16c (const int n, float * y, const float * x);
+extern "C" void ggml_vec_relu_f32_amd_fma (const int n, float * y, const float * x);
+extern "C" void ggml_vec_relu_f32_amd_avx (const int n, float * y, const float * x);
+extern "C" void ggml_vec_relu_f32_arm82 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_relu_f32_arm80 (const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_leaky_relu_f32_amd_avx512bf16 (const int n, float * y, const float * x, const float ns);
+extern "C" void ggml_vec_leaky_relu_f32_amd_avx512 (const int n, float * y, const float * x, const float ns);
+extern "C" void ggml_vec_leaky_relu_f32_amd_avx2 (const int n, float * y, const float * x, const float ns);
+extern "C" void ggml_vec_leaky_relu_f32_amd_f16c (const int n, float * y, const float * x, const float ns);
+extern "C" void ggml_vec_leaky_relu_f32_amd_fma (const int n, float * y, const float * x, const float ns);
+extern "C" void ggml_vec_leaky_relu_f32_amd_avx (const int n, float * y, const float * x, const float ns);
+extern "C" void ggml_vec_leaky_relu_f32_arm82 (const int n, float * y, const float * x, const float ns);
+extern "C" void ggml_vec_leaky_relu_f32_arm80 (const int n, float * y, const float * x, const float ns);
+
+extern "C" void ggml_vec_hardswish_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardswish_f32_amd_avx512 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardswish_f32_amd_avx2 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardswish_f32_amd_f16c (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardswish_f32_amd_fma (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardswish_f32_amd_avx (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardswish_f32_arm82 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardswish_f32_arm80 (const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_hardsigmoid_f32_amd_avx512bf16 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardsigmoid_f32_amd_avx512 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardsigmoid_f32_amd_avx2 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardsigmoid_f32_amd_f16c (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardsigmoid_f32_amd_fma (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardsigmoid_f32_amd_avx (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardsigmoid_f32_arm82 (const int n, float * y, const float * x);
+extern "C" void ggml_vec_hardsigmoid_f32_arm80 (const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_gelu_f16_amd_avx512bf16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);
+extern "C" void ggml_vec_gelu_f16_amd_avx512(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);
+extern "C" void ggml_vec_gelu_f16_amd_avx2(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);
+extern "C" void ggml_vec_gelu_f16_amd_f16c(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);
+extern "C" void ggml_vec_gelu_f16_amd_fma(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);
+extern "C" void ggml_vec_gelu_f16_amd_avx(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);
+extern "C" void ggml_vec_gelu_f16_arm82(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);
+extern "C" void ggml_vec_gelu_f16_arm80(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);
+
+extern "C" void ggml_vec_gelu_f32_amd_avx512bf16(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_f32_amd_avx512(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_f32_amd_avx2(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_f32_amd_f16c(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_f32_amd_fma(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_f32_amd_avx(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_f32_arm82(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_f32_arm80(const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_gelu_quick_f32_amd_avx512bf16(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_quick_f32_amd_avx512(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_quick_f32_amd_avx2(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_quick_f32_amd_f16c(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_quick_f32_amd_fma(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_quick_f32_amd_avx(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_quick_f32_arm82(const int n, float * y, const float * x);
+extern "C" void ggml_vec_gelu_quick_f32_arm80(const int n, float * y, const float * x);
+
+extern "C" void ggml_vec_silu_f32_amd_avx512bf16(const int n, float * y, const float * x);
+extern "C" void ggml_vec_silu_f32_amd_avx512(const int n, float * y, const float * x);
+extern "C" void ggml_vec_silu_f32_amd_avx2(const int n, float * y, const float * x);
+extern "C" void ggml_vec_silu_f32_amd_f16c(const int n, float * y, const float * x);
+extern "C" void ggml_vec_silu_f32_amd_fma(const int n, float * y, const float * x);
+extern "C" void ggml_vec_silu_f32_amd_avx(const int n, float * y, const float * x);
+extern "C" void ggml_vec_silu_f32_arm82(const int n, float * y, const float * x);
+extern "C" void ggml_vec_silu_f32_arm80(const int n, float * y, const float * x);
+
+extern "C" float ggml_silu_backward_f32_amd_avx512bf16(float x, float dy);
+extern "C" float ggml_silu_backward_f32_amd_avx512(float x, float dy);
+extern "C" float ggml_silu_backward_f32_amd_avx2(float x, float dy);
+extern "C" float ggml_silu_backward_f32_amd_f16c(float x, float dy);
+extern "C" float ggml_silu_backward_f32_amd_fma(float x, float dy);
+extern "C" float ggml_silu_backward_f32_amd_avx(float x, float dy);
+extern "C" float ggml_silu_backward_f32_arm82(float x, float dy);
+extern "C" float ggml_silu_backward_f32_arm80(float x, float dy);
+
+extern "C" void ggml_vec_silu_backward_f32_amd_avx512bf16(const int n, float * dx, const float * x, const float * dy);
+extern "C" void ggml_vec_silu_backward_f32_amd_avx512(const int n, float * dx, const float * x, const float * dy);
+extern "C" void ggml_vec_silu_backward_f32_amd_avx2(const int n, float * dx, const float * x, const float * dy);
+extern "C" void ggml_vec_silu_backward_f32_amd_f16c(const int n, float * dx, const float * x, const float * dy);
+extern "C" void ggml_vec_silu_backward_f32_amd_fma(const int n, float * dx, const float * x, const float * dy);
+extern "C" void ggml_vec_silu_backward_f32_amd_avx(const int n, float * dx, const float * x, const float * dy);
+extern "C" void ggml_vec_silu_backward_f32_arm82(const int n, float * dx, const float * x, const float * dy);
+extern "C" void ggml_vec_silu_backward_f32_arm80(const int n, float * dx, const float * x, const float * dy);
+
+extern "C" void ggml_vec_sum_f32_amd_avx512bf16(const int n, float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_amd_avx512(const int n, float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_amd_avx2(const int n, float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_amd_f16c(const int n, float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_amd_fma(const int n, float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_amd_avx(const int n, float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_arm82(const int n, float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_arm80(const int n, float * s, const float * x);
+
+extern "C" void ggml_vec_sum_f32_ggf_amd_avx512bf16(const int n, ggml_float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_ggf_amd_avx512(const int n, ggml_float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_ggf_amd_avx2(const int n, ggml_float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_ggf_amd_f16c(const int n, ggml_float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_ggf_amd_fma(const int n, ggml_float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_ggf_amd_avx(const int n, ggml_float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_ggf_arm82(const int n, ggml_float * s, const float * x);
+extern "C" void ggml_vec_sum_f32_ggf_arm80(const int n, ggml_float * s, const float * x);
+
+extern "C" void ggml_vec_sum_f16_ggf_amd_avx512bf16(const int n, float * s, const ggml_fp16_t * x);
+extern "C" void ggml_vec_sum_f16_ggf_amd_avx512(const int n, float * s, const ggml_fp16_t * x);
+extern "C" void ggml_vec_sum_f16_ggf_amd_avx2(const int n, float * s, const ggml_fp16_t * x);
+extern "C" void ggml_vec_sum_f16_ggf_amd_f16c(const int n, float * s, const ggml_fp16_t * x);
+extern "C" void ggml_vec_sum_f16_ggf_amd_fma(const int n, float * s, const ggml_fp16_t * x);
+extern "C" void ggml_vec_sum_f16_ggf_amd_avx(const int n, float * s, const ggml_fp16_t * x);
+extern "C" void ggml_vec_sum_f16_ggf_arm82(const int n, float * s, const ggml_fp16_t * x);
+extern "C" void ggml_vec_sum_f16_ggf_arm80(const int n, float * s, const ggml_fp16_t * x);
+
+extern "C" void ggml_vec_max_f32_amd_avx512bf16(const int n, float * s, const float * x);
+extern "C" void ggml_vec_max_f32_amd_avx512(const int n, float * s, const float * x);
+extern "C" void ggml_vec_max_f32_amd_avx2(const int n, float * s, const float * x);
+extern "C" void ggml_vec_max_f32_amd_f16c(const int n, float * s, const float * x);
+extern "C" void ggml_vec_max_f32_amd_fma(const int n, float * s, const float * x);
+extern "C" void ggml_vec_max_f32_amd_avx(const int n, float * s, const float * x);
+extern "C" void ggml_vec_max_f32_arm82(const int n, float * s, const float * x);
+extern "C" void ggml_vec_max_f32_arm80(const int n, float * s, const float * x);
+
+extern "C" void ggml_vec_norm_inv_f32_amd_avx512bf16(const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_inv_f32_amd_avx512(const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_inv_f32_amd_avx2(const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_inv_f32_amd_f16c(const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_inv_f32_amd_fma(const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_inv_f32_amd_avx(const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_inv_f32_arm82(const int n, float * s, const float * x);
+extern "C" void ggml_vec_norm_inv_f32_arm80(const int n, float * s, const float * x);
+
+extern "C" void ggml_vec_argmax_f32_amd_avx512bf16(const int n, int * s, const float * x);
+extern "C" void ggml_vec_argmax_f32_amd_avx512(const int n, int * s, const float * x);
+extern "C" void ggml_vec_argmax_f32_amd_avx2(const int n, int * s, const float * x);
+extern "C" void ggml_vec_argmax_f32_amd_f16c(const int n, int * s, const float * x);
+extern "C" void ggml_vec_argmax_f32_amd_fma(const int n, int * s, const float * x);
+extern "C" void ggml_vec_argmax_f32_amd_avx(const int n, int * s, const float * x);
+extern "C" void ggml_vec_argmax_f32_arm82(const int n, int * s, const float * x);
+extern "C" void ggml_vec_argmax_f32_arm80(const int n, int * s, const float * x);
+
+static const struct VectorFuncs {
+ typeof(ggml_fp16_to_fp32_row) *ptr_ggml_fp16_to_fp32_row;
+ typeof(ggml_fp32_to_fp16_row) *ptr_ggml_fp32_to_fp16_row;
+ typeof(ggml_bf16_to_fp32_row) *ptr_ggml_bf16_to_fp32_row;
+ typeof(ggml_fp32_to_bf16_row) *ptr_ggml_fp32_to_bf16_row;
+ typeof(ggml_vec_dot_f32) *ptr_ggml_vec_dot_f32;
+ typeof(ggml_vec_dot_f16) *ptr_ggml_vec_dot_f16;
+ typeof(ggml_vec_dot_bf16) *ptr_ggml_vec_dot_bf16;
+ typeof(ggml_vec_dot_f16_unroll) *ptr_ggml_vec_dot_f16_unroll;
+ typeof(ggml_vec_mad_f32_unroll) *ptr_ggml_vec_mad_f32_unroll;
+ typeof(ggml_vec_set_i8) *ptr_ggml_vec_set_i8;
+ typeof(ggml_vec_set_i16) *ptr_ggml_vec_set_i16;
+ typeof(ggml_vec_set_i32) *ptr_ggml_vec_set_i32;
+ typeof(ggml_vec_set_f16) *ptr_ggml_vec_set_f16;
+ typeof(ggml_vec_set_bf16) *ptr_ggml_vec_set_bf16;
+ typeof(ggml_vec_add_f32) *ptr_ggml_vec_add_f32;
+ typeof(ggml_vec_add1_f32) *ptr_ggml_vec_add1_f32;
+ typeof(ggml_vec_acc_f32) *ptr_ggml_vec_acc_f32;
+ typeof(ggml_vec_acc1_f32) *ptr_ggml_vec_acc1_f32;
+ typeof(ggml_vec_sub_f32) *ptr_ggml_vec_sub_f32;
+ typeof(ggml_vec_set_f32) *ptr_ggml_vec_set_f32;
+ typeof(ggml_vec_cpy_f32) *ptr_ggml_vec_cpy_f32;
+ typeof(ggml_vec_neg_f32) *ptr_ggml_vec_neg_f32;
+ typeof(ggml_vec_mul_f32) *ptr_ggml_vec_mul_f32;
+ typeof(ggml_vec_div_f32) *ptr_ggml_vec_div_f32;
+ typeof(ggml_vec_scale_f32) *ptr_ggml_vec_scale_f32;
+ typeof(ggml_vec_scale_f16) *ptr_ggml_vec_scale_f16;
+ typeof(ggml_vec_mad_f32) *ptr_ggml_vec_mad_f32;
+ typeof(ggml_vec_mad_f16) *ptr_ggml_vec_mad_f16;
+ typeof(ggml_vec_norm_f32) *ptr_ggml_vec_norm_f32;
+ typeof(ggml_vec_sqr_f32) *ptr_ggml_vec_sqr_f32;
+ typeof(ggml_vec_sqrt_f32) *ptr_ggml_vec_sqrt_f32;
+ typeof(ggml_vec_log_f32) *ptr_ggml_vec_log_f32;
+ typeof(ggml_vec_abs_f32) *ptr_ggml_vec_abs_f32;
+ typeof(ggml_vec_sgn_f32) *ptr_ggml_vec_sgn_f32;
+ typeof(ggml_vec_step_f32) *ptr_ggml_vec_step_f32;
+ typeof(ggml_vec_tanh_f32) *ptr_ggml_vec_tanh_f32;
+ typeof(ggml_vec_elu_f32) *ptr_ggml_vec_elu_f32;
+ typeof(ggml_vec_relu_f32) *ptr_ggml_vec_relu_f32;
+ typeof(ggml_vec_leaky_relu_f32) *ptr_ggml_vec_leaky_relu_f32;
+ typeof(ggml_vec_hardswish_f32) *ptr_ggml_vec_hardswish_f32;
+ typeof(ggml_vec_hardsigmoid_f32) *ptr_ggml_vec_hardsigmoid_f32;
+ typeof(ggml_vec_gelu_f16) *ptr_ggml_vec_gelu_f16;
+ typeof(ggml_vec_gelu_f32) *ptr_ggml_vec_gelu_f32;
+ typeof(ggml_vec_gelu_quick_f32) *ptr_ggml_vec_gelu_quick_f32;
+ typeof(ggml_vec_silu_f32) *ptr_ggml_vec_silu_f32;
+ typeof(ggml_silu_backward_f32) *ptr_ggml_silu_backward_f32;
+ typeof(ggml_vec_silu_backward_f32) *ptr_ggml_vec_silu_backward_f32;
+ typeof(ggml_vec_sum_f32) *ptr_ggml_vec_sum_f32;
+ typeof(ggml_vec_sum_f32_ggf) *ptr_ggml_vec_sum_f32_ggf;
+ typeof(ggml_vec_sum_f16_ggf) *ptr_ggml_vec_sum_f16_ggf;
+ typeof(ggml_vec_max_f32) *ptr_ggml_vec_max_f32;
+ typeof(ggml_vec_norm_inv_f32) *ptr_ggml_vec_norm_inv_f32;
+ typeof(ggml_vec_argmax_f32) *ptr_ggml_vec_argmax_f32;
+
+ VectorFuncs() {
+#ifdef __x86_64__
+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2) && X86_HAVE(AVX512F) && X86_HAVE(AVX512VL) && X86_HAVE(AVX512_BF16)) {
+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_avx512bf16;
+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx512bf16;
+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx512bf16;
+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx512bf16;
+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx512bf16;
+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx512bf16;
+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx512bf16;
+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_amd_avx512bf16;
+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_amd_avx512bf16;
+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_amd_avx512bf16;
+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_amd_avx512bf16;
+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_amd_avx512bf16;
+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_amd_avx512bf16;
+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_amd_avx512bf16;
+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_amd_avx512bf16;
+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_amd_avx512bf16;
+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_amd_avx512bf16;
+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_amd_avx512bf16;
+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_amd_avx512bf16;
+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_amd_avx512bf16;
+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_amd_avx512bf16;
+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_amd_avx512bf16;
+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_amd_avx512bf16;
+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_amd_avx512bf16;
+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_amd_avx512bf16;
+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_amd_avx512bf16;
+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_amd_avx512bf16;
+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_amd_avx512bf16;
+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_amd_avx512bf16;
+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_amd_avx512bf16;
+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_amd_avx512bf16;
+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_amd_avx512bf16;
+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_amd_avx512bf16;
+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_amd_avx512bf16;
+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_amd_avx512bf16;
+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_amd_avx512bf16;
+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_amd_avx512bf16;
+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_amd_avx512bf16;
+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_amd_avx512bf16;
+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_amd_avx512bf16;
+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_amd_avx512bf16;
+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_amd_avx512bf16;
+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_amd_avx512bf16;
+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_amd_avx512bf16;
+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_amd_avx512bf16;
+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_amd_avx512bf16;
+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_amd_avx512bf16;
+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_avx512bf16;
+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_avx512bf16;
+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_avx512bf16;
+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx512bf16;
+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx512bf16;
+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx512bf16;
+ return;
+ }
+#endif
+#ifdef __x86_64__
+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2) && X86_HAVE(AVX512F)) {
+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_avx512;
+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx512;
+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx512;
+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx512;
+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx512;
+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx512;
+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx512;
+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_amd_avx512;
+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_amd_avx512;
+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_amd_avx512;
+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_amd_avx512;
+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_amd_avx512;
+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_amd_avx512;
+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_amd_avx512;
+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_amd_avx512;
+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_amd_avx512;
+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_amd_avx512;
+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_amd_avx512;
+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_amd_avx512;
+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_amd_avx512;
+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_amd_avx512;
+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_amd_avx512;
+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_amd_avx512;
+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_amd_avx512;
+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_amd_avx512;
+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_amd_avx512;
+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_amd_avx512;
+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_amd_avx512;
+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_amd_avx512;
+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_amd_avx512;
+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_amd_avx512;
+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_amd_avx512;
+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_amd_avx512;
+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_amd_avx512;
+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_amd_avx512;
+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_amd_avx512;
+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_amd_avx512;
+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_amd_avx512;
+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_amd_avx512;
+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_amd_avx512;
+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_amd_avx512;
+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_amd_avx512;
+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_amd_avx512;
+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_amd_avx512;
+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_amd_avx512;
+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_amd_avx512;
+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_amd_avx512;
+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_avx512;
+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_avx512;
+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_avx512;
+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx512;
+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx512;
+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx512;
+ return;
+ }
+#endif
+#ifdef __x86_64__
+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && X86_HAVE(AVX2)) {
+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_avx2;
+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx2;
+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx2;
+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx2;
+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx2;
+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx2;
+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx2;
+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_amd_avx2;
+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_amd_avx2;
+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_amd_avx2;
+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_amd_avx2;
+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_amd_avx2;
+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_amd_avx2;
+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_amd_avx2;
+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_amd_avx2;
+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_amd_avx2;
+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_amd_avx2;
+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_amd_avx2;
+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_amd_avx2;
+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_amd_avx2;
+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_amd_avx2;
+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_amd_avx2;
+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_amd_avx2;
+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_amd_avx2;
+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_amd_avx2;
+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_amd_avx2;
+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_amd_avx2;
+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_amd_avx2;
+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_amd_avx2;
+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_amd_avx2;
+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_amd_avx2;
+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_amd_avx2;
+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_amd_avx2;
+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_amd_avx2;
+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_amd_avx2;
+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_amd_avx2;
+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_amd_avx2;
+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_amd_avx2;
+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_amd_avx2;
+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_amd_avx2;
+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_amd_avx2;
+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_amd_avx2;
+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_amd_avx2;
+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_amd_avx2;
+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_amd_avx2;
+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_amd_avx2;
+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_amd_avx2;
+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_avx2;
+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_avx2;
+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_avx2;
+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx2;
+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx2;
+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx2;
+ return;
+ }
+#endif
+#ifdef __x86_64__
+ if (X86_HAVE(F16C)) {
+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_f16c;
+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_f16c;
+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_f16c;
+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_f16c;
+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_f16c;
+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_f16c;
+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_f16c;
+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_amd_f16c;
+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_amd_f16c;
+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_amd_f16c;
+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_amd_f16c;
+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_amd_f16c;
+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_amd_f16c;
+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_amd_f16c;
+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_amd_f16c;
+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_amd_f16c;
+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_amd_f16c;
+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_amd_f16c;
+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_amd_f16c;
+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_amd_f16c;
+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_amd_f16c;
+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_amd_f16c;
+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_amd_f16c;
+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_amd_f16c;
+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_amd_f16c;
+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_amd_f16c;
+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_amd_f16c;
+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_amd_f16c;
+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_amd_f16c;
+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_amd_f16c;
+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_amd_f16c;
+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_amd_f16c;
+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_amd_f16c;
+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_amd_f16c;
+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_amd_f16c;
+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_amd_f16c;
+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_amd_f16c;
+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_amd_f16c;
+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_amd_f16c;
+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_amd_f16c;
+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_amd_f16c;
+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_amd_f16c;
+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_amd_f16c;
+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_amd_f16c;
+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_amd_f16c;
+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_amd_f16c;
+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_amd_f16c;
+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_f16c;
+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_f16c;
+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_f16c;
+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_f16c;
+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_f16c;
+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_f16c;
+ return;
+ }
+#endif
+#ifdef __x86_64__
+ if (X86_HAVE(FMA)) {
+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_fma;
+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_fma;
+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_fma;
+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_fma;
+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_fma;
+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_fma;
+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_fma;
+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_amd_fma;
+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_amd_fma;
+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_amd_fma;
+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_amd_fma;
+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_amd_fma;
+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_amd_fma;
+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_amd_fma;
+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_amd_fma;
+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_amd_fma;
+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_amd_fma;
+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_amd_fma;
+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_amd_fma;
+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_amd_fma;
+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_amd_fma;
+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_amd_fma;
+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_amd_fma;
+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_amd_fma;
+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_amd_fma;
+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_amd_fma;
+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_amd_fma;
+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_amd_fma;
+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_amd_fma;
+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_amd_fma;
+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_amd_fma;
+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_amd_fma;
+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_amd_fma;
+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_amd_fma;
+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_amd_fma;
+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_amd_fma;
+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_amd_fma;
+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_amd_fma;
+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_amd_fma;
+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_amd_fma;
+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_amd_fma;
+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_amd_fma;
+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_amd_fma;
+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_amd_fma;
+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_amd_fma;
+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_amd_fma;
+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_amd_fma;
+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_fma;
+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_fma;
+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_fma;
+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_fma;
+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_fma;
+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_fma;
+ return;
+ }
+#endif
+#ifdef __x86_64__
+ if (1) {
+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_amd_avx;
+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx;
+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx;
+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx;
+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx;
+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx;
+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx;
+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_amd_avx;
+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_amd_avx;
+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_amd_avx;
+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_amd_avx;
+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_amd_avx;
+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_amd_avx;
+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_amd_avx;
+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_amd_avx;
+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_amd_avx;
+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_amd_avx;
+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_amd_avx;
+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_amd_avx;
+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_amd_avx;
+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_amd_avx;
+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_amd_avx;
+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_amd_avx;
+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_amd_avx;
+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_amd_avx;
+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_amd_avx;
+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_amd_avx;
+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_amd_avx;
+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_amd_avx;
+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_amd_avx;
+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_amd_avx;
+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_amd_avx;
+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_amd_avx;
+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_amd_avx;
+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_amd_avx;
+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_amd_avx;
+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_amd_avx;
+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_amd_avx;
+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_amd_avx;
+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_amd_avx;
+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_amd_avx;
+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_amd_avx;
+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_amd_avx;
+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_amd_avx;
+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_amd_avx;
+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_amd_avx;
+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_amd_avx;
+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_avx;
+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_avx;
+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_avx;
+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx;
+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx;
+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx;
+ return;
+ }
+#endif
+#ifdef __aarch64__
+ if ((getauxval(AT_HWCAP) & HWCAP_FPHP) && (getauxval(AT_HWCAP) & HWCAP_ASIMDHP)) {
+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_arm82;
+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_arm82;
+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_arm82;
+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_arm82;
+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_arm82;
+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_arm82;
+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_arm82;
+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_arm82;
+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_arm82;
+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_arm82;
+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_arm82;
+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_arm82;
+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_arm82;
+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_arm82;
+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_arm82;
+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_arm82;
+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_arm82;
+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_arm82;
+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_arm82;
+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_arm82;
+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_arm82;
+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_arm82;
+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_arm82;
+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_arm82;
+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_arm82;
+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_arm82;
+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_arm82;
+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_arm82;
+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_arm82;
+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_arm82;
+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_arm82;
+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_arm82;
+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_arm82;
+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_arm82;
+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_arm82;
+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_arm82;
+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_arm82;
+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_arm82;
+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_arm82;
+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_arm82;
+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_arm82;
+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_arm82;
+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_arm82;
+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_arm82;
+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_arm82;
+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_arm82;
+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_arm82;
+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_arm82;
+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_arm82;
+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_arm82;
+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_arm82;
+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_arm82;
+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_arm82;
+ return;
+ }
+#endif
+#ifdef __aarch64__
+ if (1) {
+ ptr_ggml_fp16_to_fp32_row = ggml_fp16_to_fp32_row_arm80;
+ ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_arm80;
+ ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_arm80;
+ ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_arm80;
+ ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_arm80;
+ ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_arm80;
+ ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_arm80;
+ ptr_ggml_vec_dot_f16_unroll = ggml_vec_dot_f16_unroll_arm80;
+ ptr_ggml_vec_mad_f32_unroll = ggml_vec_mad_f32_unroll_arm80;
+ ptr_ggml_vec_set_i8 = ggml_vec_set_i8_arm80;
+ ptr_ggml_vec_set_i16 = ggml_vec_set_i16_arm80;
+ ptr_ggml_vec_set_i32 = ggml_vec_set_i32_arm80;
+ ptr_ggml_vec_set_f16 = ggml_vec_set_f16_arm80;
+ ptr_ggml_vec_set_bf16 = ggml_vec_set_bf16_arm80;
+ ptr_ggml_vec_add_f32 = ggml_vec_add_f32_arm80;
+ ptr_ggml_vec_add1_f32 = ggml_vec_add1_f32_arm80;
+ ptr_ggml_vec_acc_f32 = ggml_vec_acc_f32_arm80;
+ ptr_ggml_vec_acc1_f32 = ggml_vec_acc1_f32_arm80;
+ ptr_ggml_vec_sub_f32 = ggml_vec_sub_f32_arm80;
+ ptr_ggml_vec_set_f32 = ggml_vec_set_f32_arm80;
+ ptr_ggml_vec_cpy_f32 = ggml_vec_cpy_f32_arm80;
+ ptr_ggml_vec_neg_f32 = ggml_vec_neg_f32_arm80;
+ ptr_ggml_vec_mul_f32 = ggml_vec_mul_f32_arm80;
+ ptr_ggml_vec_div_f32 = ggml_vec_div_f32_arm80;
+ ptr_ggml_vec_scale_f32 = ggml_vec_scale_f32_arm80;
+ ptr_ggml_vec_scale_f16 = ggml_vec_scale_f16_arm80;
+ ptr_ggml_vec_mad_f32 = ggml_vec_mad_f32_arm80;
+ ptr_ggml_vec_mad_f16 = ggml_vec_mad_f16_arm80;
+ ptr_ggml_vec_norm_f32 = ggml_vec_norm_f32_arm80;
+ ptr_ggml_vec_sqr_f32 = ggml_vec_sqr_f32_arm80;
+ ptr_ggml_vec_sqrt_f32 = ggml_vec_sqrt_f32_arm80;
+ ptr_ggml_vec_log_f32 = ggml_vec_log_f32_arm80;
+ ptr_ggml_vec_abs_f32 = ggml_vec_abs_f32_arm80;
+ ptr_ggml_vec_sgn_f32 = ggml_vec_sgn_f32_arm80;
+ ptr_ggml_vec_step_f32 = ggml_vec_step_f32_arm80;
+ ptr_ggml_vec_tanh_f32 = ggml_vec_tanh_f32_arm80;
+ ptr_ggml_vec_elu_f32 = ggml_vec_elu_f32_arm80;
+ ptr_ggml_vec_relu_f32 = ggml_vec_relu_f32_arm80;
+ ptr_ggml_vec_leaky_relu_f32 = ggml_vec_leaky_relu_f32_arm80;
+ ptr_ggml_vec_hardswish_f32 = ggml_vec_hardswish_f32_arm80;
+ ptr_ggml_vec_hardsigmoid_f32 = ggml_vec_hardsigmoid_f32_arm80;
+ ptr_ggml_vec_gelu_f16 = ggml_vec_gelu_f16_arm80;
+ ptr_ggml_vec_gelu_f32 = ggml_vec_gelu_f32_arm80;
+ ptr_ggml_vec_gelu_quick_f32 = ggml_vec_gelu_quick_f32_arm80;
+ ptr_ggml_vec_silu_f32 = ggml_vec_silu_f32_arm80;
+ ptr_ggml_silu_backward_f32 = ggml_silu_backward_f32_arm80;
+ ptr_ggml_vec_silu_backward_f32 = ggml_vec_silu_backward_f32_arm80;
+ ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_arm80;
+ ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_arm80;
+ ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_arm80;
+ ptr_ggml_vec_max_f32 = ggml_vec_max_f32_arm80;
+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_arm80;
+ ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_arm80;
+ return;
+ }
+#endif
+ }
+} funcs;
+
+void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) {
+ return funcs.ptr_ggml_fp16_to_fp32_row(x, y, n);
+}
+
+void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) {
+ return funcs.ptr_ggml_fp32_to_fp16_row(x, y, n);
+}
+
+void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {
+ return funcs.ptr_ggml_bf16_to_fp32_row(x, y, n);
+}
+
+void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {
+ return funcs.ptr_ggml_fp32_to_bf16_row(x, y, n);
+}
+
+void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_f32(n, s, bs, x, bx, y, by, nrc);
+}
+
+void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_f16(n, s, bs, x, bx, y, by, nrc);
+}
+
+void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc) {
+ return funcs.ptr_ggml_vec_dot_bf16(n, s, bs, x, bx, y, by, nrc);
+}
+
+void ggml_vec_dot_f16_unroll(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y) {
+ return funcs.ptr_ggml_vec_dot_f16_unroll(n, xs, s, xv, y);
+}
+
+void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv) {
+ return funcs.ptr_ggml_vec_mad_f32_unroll(n, xs, vs, y, xv, vv);
+}
+
+void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v) {
+ return funcs.ptr_ggml_vec_set_i8(n, x, v);
+}
+
+void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v) {
+ return funcs.ptr_ggml_vec_set_i16(n, x, v);
+}
+
+void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v) {
+ return funcs.ptr_ggml_vec_set_i32(n, x, v);
+}
+
+void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const int32_t v) {
+ return funcs.ptr_ggml_vec_set_f16(n, x, v);
+}
+
+void ggml_vec_set_bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v) {
+ return funcs.ptr_ggml_vec_set_bf16(n, x, v);
+}
+
+void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) {
+ return funcs.ptr_ggml_vec_add_f32(n, z, x, y);
+}
+
+void ggml_vec_add1_f32(const int n, float * z, const float * x, const float v) {
+ return funcs.ptr_ggml_vec_add1_f32(n, z, x, v);
+}
+
+void ggml_vec_acc_f32 (const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_acc_f32(n, y, x);
+}
+
+void ggml_vec_acc1_f32(const int n, float * y, const float v) {
+ return funcs.ptr_ggml_vec_acc1_f32(n, y, v);
+}
+
+void ggml_vec_sub_f32 (const int n, float * z, const float * x, const float * y) {
+ return funcs.ptr_ggml_vec_sub_f32(n, z, x, y);
+}
+
+void ggml_vec_set_f32 (const int n, float * x, const float v) {
+ return funcs.ptr_ggml_vec_set_f32(n, x, v);
+}
+
+void ggml_vec_cpy_f32 (const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_cpy_f32(n, y, x);
+}
+
+void ggml_vec_neg_f32 (const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_neg_f32(n, y, x);
+}
+
+void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y) {
+ return funcs.ptr_ggml_vec_mul_f32(n, z, x, y);
+}
+
+void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) {
+ return funcs.ptr_ggml_vec_div_f32(n, z, x, y);
+}
+
+void ggml_vec_scale_f32(const int n, float * y, const float v) {
+ return funcs.ptr_ggml_vec_scale_f32(n, y, v);
+}
+
+void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {
+ return funcs.ptr_ggml_vec_scale_f16(n, y, v);
+}
+
+void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v) {
+ return funcs.ptr_ggml_vec_mad_f32(n, y, x, v);
+}
+
+void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v) {
+ return funcs.ptr_ggml_vec_mad_f16(n, y, x, v);
+}
+
+void ggml_vec_norm_f32 (const int n, float * s, const float * x) {
+ return funcs.ptr_ggml_vec_norm_f32(n, s, x);
+}
+
+void ggml_vec_sqr_f32 (const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_sqr_f32(n, y, x);
+}
+
+void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_sqrt_f32(n, y, x);
+}
+
+void ggml_vec_log_f32 (const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_log_f32(n, y, x);
+}
+
+void ggml_vec_abs_f32 (const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_abs_f32(n, y, x);
+}
+
+void ggml_vec_sgn_f32 (const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_sgn_f32(n, y, x);
+}
+
+void ggml_vec_step_f32 (const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_step_f32(n, y, x);
+}
+
+void ggml_vec_tanh_f32 (const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_tanh_f32(n, y, x);
+}
+
+void ggml_vec_elu_f32 (const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_elu_f32(n, y, x);
+}
+
+void ggml_vec_relu_f32 (const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_relu_f32(n, y, x);
+}
+
+void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) {
+ return funcs.ptr_ggml_vec_leaky_relu_f32(n, y, x, ns);
+}
+
+void ggml_vec_hardswish_f32 (const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_hardswish_f32(n, y, x);
+}
+
+void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_hardsigmoid_f32(n, y, x);
+}
+
+void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
+ return funcs.ptr_ggml_vec_gelu_f16(n, y, x);
+}
+
+void ggml_vec_gelu_f32(const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_gelu_f32(n, y, x);
+}
+
+void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_gelu_quick_f32(n, y, x);
+}
+
+void ggml_vec_silu_f32(const int n, float * y, const float * x) {
+ return funcs.ptr_ggml_vec_silu_f32(n, y, x);
+}
+
+float ggml_silu_backward_f32(float x, float dy) {
+ return funcs.ptr_ggml_silu_backward_f32(x, dy);
+}
+
+void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, const float * dy) {
+ return funcs.ptr_ggml_vec_silu_backward_f32(n, dx, x, dy);
+}
+
+void ggml_vec_sum_f32(const int n, float * s, const float * x) {
+ return funcs.ptr_ggml_vec_sum_f32(n, s, x);
+}
+
+void ggml_vec_sum_f32_ggf(const int n, ggml_float * s, const float * x) {
+ return funcs.ptr_ggml_vec_sum_f32_ggf(n, s, x);
+}
+
+void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x) {
+ return funcs.ptr_ggml_vec_sum_f16_ggf(n, s, x);
+}
+
+void ggml_vec_max_f32(const int n, float * s, const float * x) {
+ return funcs.ptr_ggml_vec_max_f32(n, s, x);
+}
+
+void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) {
+ return funcs.ptr_ggml_vec_norm_inv_f32(n, s, x);
+}
+
+void ggml_vec_argmax_f32(const int n, int * s, const float * x) {
+ return funcs.ptr_ggml_vec_argmax_f32(n, s, x);
+}
+
diff --git a/llama.cpp/ggml-vector.h b/llama.cpp/ggml-vector.h
new file mode 100644
index 00000000..cdaff979
--- /dev/null
+++ b/llama.cpp/ggml-vector.h
@@ -0,0 +1,67 @@
+#pragma once
+#include "ggml.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef double ggml_float;
+
+extern float *ggml_table_gelu_f16;
+
+void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n);
+void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n);
+void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n);
+void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n);
+void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
+void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
+void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
+void ggml_vec_dot_f16_unroll(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y);
+void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv);
+void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v);
+void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v);
+void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v);
+void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const int32_t v);
+void ggml_vec_set_bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v);
+void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y);
+void ggml_vec_add1_f32(const int n, float * z, const float * x, const float v);
+void ggml_vec_acc_f32 (const int n, float * y, const float * x);
+void ggml_vec_acc1_f32(const int n, float * y, const float v);
+void ggml_vec_sub_f32 (const int n, float * z, const float * x, const float * y);
+void ggml_vec_set_f32 (const int n, float * x, const float v);
+void ggml_vec_cpy_f32 (const int n, float * y, const float * x);
+void ggml_vec_neg_f32 (const int n, float * y, const float * x);
+void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y);
+void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y);
+void ggml_vec_scale_f32(const int n, float * y, const float v);
+void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v);
+void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v);
+void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v);
+void ggml_vec_norm_f32 (const int n, float * s, const float * x);
+void ggml_vec_sqr_f32 (const int n, float * y, const float * x);
+void ggml_vec_sqrt_f32 (const int n, float * y, const float * x);
+void ggml_vec_log_f32 (const int n, float * y, const float * x);
+void ggml_vec_abs_f32 (const int n, float * y, const float * x);
+void ggml_vec_sgn_f32 (const int n, float * y, const float * x);
+void ggml_vec_step_f32 (const int n, float * y, const float * x);
+void ggml_vec_tanh_f32 (const int n, float * y, const float * x);
+void ggml_vec_elu_f32 (const int n, float * y, const float * x);
+void ggml_vec_relu_f32 (const int n, float * y, const float * x);
+void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns);
+void ggml_vec_hardswish_f32 (const int n, float * y, const float * x);
+void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x);
+void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x);
+void ggml_vec_gelu_f32(const int n, float * y, const float * x);
+void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x);
+void ggml_vec_silu_f32(const int n, float * y, const float * x);
+float ggml_silu_backward_f32(float x, float dy);
+void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, const float * dy);
+void ggml_vec_sum_f32(const int n, float * s, const float * x);
+void ggml_vec_sum_f32_ggf(const int n, ggml_float * s, const float * x);
+void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x);
+void ggml_vec_max_f32(const int n, float * s, const float * x);
+void ggml_vec_norm_inv_f32(const int n, float * s, const float * x);
+void ggml_vec_argmax_f32(const int n, int * s, const float * x);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/llama.cpp/ggml-vector.inc b/llama.cpp/ggml-vector.inc
new file mode 100644
index 00000000..244c63c2
--- /dev/null
+++ b/llama.cpp/ggml-vector.inc
@@ -0,0 +1,1256 @@
+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi
+
+#include "ggml-vector.h"
+#include "ggml-impl.h"
+#include <stdatomic.h>
+
+void ggml_once(atomic_uint *, void (*)(void));
+
+#define UNUSED GGML_UNUSED
+
+#define GGML_SOFT_MAX_UNROLL 4
+#define GGML_VEC_DOT_UNROLL 2
+#define GGML_VEC_MAD_UNROLL 32
+
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)
+
+#define GGML_SIMD
+
+// F32 NEON
+
+#define GGML_F32_STEP 16
+#define GGML_F32_EPR 4
+
+#define GGML_F32x4 float32x4_t
+#define GGML_F32x4_ZERO vdupq_n_f32(0.0f)
+#define GGML_F32x4_SET1(x) vdupq_n_f32(x)
+#define GGML_F32x4_LOAD vld1q_f32
+#define GGML_F32x4_STORE vst1q_f32
+#define GGML_F32x4_FMA(a, b, c) vfmaq_f32(a, b, c)
+#define GGML_F32x4_ADD vaddq_f32
+#define GGML_F32x4_MUL vmulq_f32
+#define GGML_F32x4_REDUCE_ONE(x) vaddvq_f32(x)
+#define GGML_F32x4_REDUCE(res, x) \
+{ \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vaddq_f32(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vaddq_f32(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vaddq_f32(x[i], x[offset+i]); \
+ } \
+ res = GGML_F32x4_REDUCE_ONE(x[0]); \
+}
+
+#define GGML_F32_VEC GGML_F32x4
+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO
+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1
+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD
+#define GGML_F32_VEC_STORE GGML_F32x4_STORE
+#define GGML_F32_VEC_FMA GGML_F32x4_FMA
+#define GGML_F32_VEC_ADD GGML_F32x4_ADD
+#define GGML_F32_VEC_MUL GGML_F32x4_MUL
+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE
+
+// F16 NEON
+
+#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+ #define GGML_F16_STEP 32
+ #define GGML_F16_EPR 8
+
+ #define GGML_F16x8 float16x8_t
+ #define GGML_F16x8_ZERO vdupq_n_f16(0.0f)
+ #define GGML_F16x8_SET1(x) vdupq_n_f16(x)
+ #define GGML_F16x8_LOAD(x) vld1q_f16((const ggml_fp16_internal_t *)(x))
+ #define GGML_F16x8_STORE vst1q_f16
+ #define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c)
+ #define GGML_F16x8_ADD vaddq_f16
+ #define GGML_F16x8_MUL vmulq_f16
+ #define GGML_F16x8_REDUCE(res, x) \
+ do { \
+ int offset = GGML_F16_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vaddq_f16(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vaddq_f16(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vaddq_f16(x[i], x[offset+i]); \
+ } \
+ const float32x4_t t0 = vcvt_f32_f16(vget_low_f16 (x[0])); \
+ const float32x4_t t1 = vcvt_f32_f16(vget_high_f16(x[0])); \
+ res = (ggml_float) vaddvq_f32(vaddq_f32(t0, t1)); \
+ } while (0)
+
+ #define GGML_F16_VEC GGML_F16x8
+ #define GGML_F16_VEC_ZERO GGML_F16x8_ZERO
+ #define GGML_F16_VEC_SET1 GGML_F16x8_SET1
+ #define GGML_F16_VEC_LOAD(p, i) GGML_F16x8_LOAD(p)
+ #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((ggml_fp16_internal_t *)(p), r[i])
+ #define GGML_F16_VEC_FMA GGML_F16x8_FMA
+ #define GGML_F16_VEC_ADD GGML_F16x8_ADD
+ #define GGML_F16_VEC_MUL GGML_F16x8_MUL
+ #define GGML_F16_VEC_REDUCE GGML_F16x8_REDUCE
+#else
+ // if FP16 vector arithmetic is not supported, we use FP32 instead
+ // and take advantage of the vcvt_ functions to convert to/from FP16
+
+ #define GGML_F16_STEP 16
+ #define GGML_F16_EPR 4
+
+ #define GGML_F32Cx4 float32x4_t
+ #define GGML_F32Cx4_ZERO vdupq_n_f32(0.0f)
+ #define GGML_F32Cx4_SET1(x) vdupq_n_f32(x)
+ #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16((const ggml_fp16_internal_t *)(x)))
+ #define GGML_F32Cx4_STORE(x, y) vst1_f16(x, vcvt_f16_f32(y))
+ #define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c)
+ #define GGML_F32Cx4_ADD vaddq_f32
+ #define GGML_F32Cx4_MUL vmulq_f32
+ #define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE
+
+ #define GGML_F16_VEC GGML_F32Cx4
+ #define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO
+ #define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1
+ #define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)
+ #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((ggml_fp16_internal_t *)(p), r[i])
+ #define GGML_F16_VEC_FMA GGML_F32Cx4_FMA
+ #define GGML_F16_VEC_ADD GGML_F32Cx4_ADD
+ #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL
+ #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE
+#endif
+
+#elif defined(__AVX512F__)
+
+#define GGML_SIMD
+
+// F32 AVX512
+
+#define GGML_F32_STEP 64
+#define GGML_F32_EPR 16
+
+#define GGML_F32x16 __m512
+#define GGML_F32x16_ZERO _mm512_setzero_ps()
+#define GGML_F32x16_SET1(x) _mm512_set1_ps(x)
+#define GGML_F32x16_LOAD _mm512_loadu_ps
+#define GGML_F32x16_STORE _mm512_storeu_ps
+// _mm512_fmadd_ps is defined in AVX512F so no guard is required
+#define GGML_F32x16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)
+#define GGML_F32x16_ADD _mm512_add_ps
+#define GGML_F32x16_MUL _mm512_mul_ps
+#define GGML_F32x16_REDUCE(res, x) \
+do { \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \
+ } \
+ res = _mm512_reduce_add_ps(x[0]); \
+} while (0)
+
+// TODO: is this optimal ?
+
+#define GGML_F32_VEC GGML_F32x16
+#define GGML_F32_VEC_ZERO GGML_F32x16_ZERO
+#define GGML_F32_VEC_SET1 GGML_F32x16_SET1
+#define GGML_F32_VEC_LOAD GGML_F32x16_LOAD
+#define GGML_F32_VEC_STORE GGML_F32x16_STORE
+#define GGML_F32_VEC_FMA GGML_F32x16_FMA
+#define GGML_F32_VEC_ADD GGML_F32x16_ADD
+#define GGML_F32_VEC_MUL GGML_F32x16_MUL
+#define GGML_F32_VEC_REDUCE GGML_F32x16_REDUCE
+
+// F16 AVX512
+
+// F16 AVX
+
+#define GGML_F16_STEP 64
+#define GGML_F16_EPR 16
+
+// AVX512 has FP16 extension (AVX512_FP16) but I don't have it on my machine so I use FP32 instead
+
+#define GGML_F32Cx16 __m512
+#define GGML_F32Cx16_ZERO _mm512_setzero_ps()
+#define GGML_F32Cx16_SET1(x) _mm512_set1_ps(x)
+
+// unlike _mm256_cvt intrinsics that require F16C, _mm512_cvt is defined in AVX512F
+// so F16C guard isn't required
+#define GGML_F32Cx16_LOAD(x) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x)))
+#define GGML_F32Cx16_STORE(x, y) _mm256_storeu_si256((__m256i *)(x), _mm512_cvtps_ph(y, 0))
+
+#define GGML_F32Cx16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)
+#define GGML_F32Cx16_ADD _mm512_add_ps
+#define GGML_F32Cx16_MUL _mm512_mul_ps
+#define GGML_F32Cx16_REDUCE(res, x) \
+do { \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm512_add_ps(x[i], x[offset+i]); \
+ } \
+ res = _mm512_reduce_add_ps(x[0]); \
+} while (0)
+
+#define GGML_F16_VEC GGML_F32Cx16
+#define GGML_F16_VEC_ZERO GGML_F32Cx16_ZERO
+#define GGML_F16_VEC_SET1 GGML_F32Cx16_SET1
+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx16_LOAD(p)
+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx16_STORE(p, r[i])
+#define GGML_F16_VEC_FMA GGML_F32Cx16_FMA
+#define GGML_F16_VEC_ADD GGML_F32Cx16_ADD
+#define GGML_F16_VEC_MUL GGML_F32Cx16_MUL
+#define GGML_F16_VEC_REDUCE GGML_F32Cx16_REDUCE
+
+#elif defined(__AVX__)
+
+#define GGML_SIMD
+
+// F32 AVX
+
+#define GGML_F32_STEP 32
+#define GGML_F32_EPR 8
+
+#define GGML_F32x8 __m256
+#define GGML_F32x8_ZERO _mm256_setzero_ps()
+#define GGML_F32x8_SET1(x) _mm256_set1_ps(x)
+#define GGML_F32x8_LOAD _mm256_loadu_ps
+#define GGML_F32x8_STORE _mm256_storeu_ps
+#if defined(__FMA__)
+ #define GGML_F32x8_FMA(a, b, c) _mm256_fmadd_ps(b, c, a)
+#else
+ #define GGML_F32x8_FMA(a, b, c) _mm256_add_ps(_mm256_mul_ps(b, c), a)
+#endif
+#define GGML_F32x8_ADD _mm256_add_ps
+#define GGML_F32x8_MUL _mm256_mul_ps
+#define GGML_F32x8_REDUCE(res, x) \
+do { \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \
+ } \
+ const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), \
+ _mm256_extractf128_ps(x[0], 1)); \
+ const __m128 t1 = _mm_hadd_ps(t0, t0); \
+ res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); \
+} while (0)
+// TODO: is this optimal ?
+
+#define GGML_F32_VEC GGML_F32x8
+#define GGML_F32_VEC_ZERO GGML_F32x8_ZERO
+#define GGML_F32_VEC_SET1 GGML_F32x8_SET1
+#define GGML_F32_VEC_LOAD GGML_F32x8_LOAD
+#define GGML_F32_VEC_STORE GGML_F32x8_STORE
+#define GGML_F32_VEC_FMA GGML_F32x8_FMA
+#define GGML_F32_VEC_ADD GGML_F32x8_ADD
+#define GGML_F32_VEC_MUL GGML_F32x8_MUL
+#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE
+
+// F16 AVX
+
+#define GGML_F16_STEP 32
+#define GGML_F16_EPR 8
+
+// F16 arithmetic is not supported by AVX, so we use F32 instead
+
+#define GGML_F32Cx8 __m256
+#define GGML_F32Cx8_ZERO _mm256_setzero_ps()
+#define GGML_F32Cx8_SET1(x) _mm256_set1_ps(x)
+
+#if defined(__F16C__)
+// the _mm256_cvt intrinsics require F16C
+#define GGML_F32Cx8_LOAD(x) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x)))
+#define GGML_F32Cx8_STORE(x, y) _mm_storeu_si128((__m128i *)(x), _mm256_cvtps_ph(y, 0))
+#else
+static inline __m256 __avx_f32cx8_load(ggml_fp16_t *x) {
+ float tmp[8];
+
+ for (int i = 0; i < 8; i++) {
+ tmp[i] = GGML_FP16_TO_FP32(x[i]);
+ }
+
+ return _mm256_loadu_ps(tmp);
+}
+static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {
+ float arr[8];
+
+ _mm256_storeu_ps(arr, y);
+
+ for (int i = 0; i < 8; i++)
+ x[i] = GGML_FP32_TO_FP16(arr[i]);
+}
+#define GGML_F32Cx8_LOAD(x) __avx_f32cx8_load((void *)(x))
+#define GGML_F32Cx8_STORE(x, y) __avx_f32cx8_store(x, y)
+#endif
+
+#define GGML_F32Cx8_FMA GGML_F32x8_FMA
+#define GGML_F32Cx8_ADD _mm256_add_ps
+#define GGML_F32Cx8_MUL _mm256_mul_ps
+#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE
+
+#define GGML_F16_VEC GGML_F32Cx8
+#define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO
+#define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1
+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx8_LOAD(p)
+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i])
+#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA
+#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD
+#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL
+#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE
+
+#elif defined(__POWER9_VECTOR__)
+
+#define GGML_SIMD
+
+// F32 POWER9
+
+#define GGML_F32_STEP 32
+#define GGML_F32_EPR 4
+
+#define GGML_F32x4 vector float
+#define GGML_F32x4_ZERO 0.0f
+#define GGML_F32x4_SET1 vec_splats
+#define GGML_F32x4_LOAD(p) vec_xl(0, p)
+#define GGML_F32x4_STORE(p, r) vec_xst(r, 0, p)
+#define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a)
+#define GGML_F32x4_ADD vec_add
+#define GGML_F32x4_MUL vec_mul
+#define GGML_F32x4_REDUCE(res, x) \
+{ \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vec_add(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vec_add(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = vec_add(x[i], x[offset+i]); \
+ } \
+ res = vec_extract(x[0], 0) + \
+ vec_extract(x[0], 1) + \
+ vec_extract(x[0], 2) + \
+ vec_extract(x[0], 3); \
+}
+
+#define GGML_F32_VEC GGML_F32x4
+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO
+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1
+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD
+#define GGML_F32_VEC_STORE GGML_F32x4_STORE
+#define GGML_F32_VEC_FMA GGML_F32x4_FMA
+#define GGML_F32_VEC_ADD GGML_F32x4_ADD
+#define GGML_F32_VEC_MUL GGML_F32x4_MUL
+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE
+
+// F16 POWER9
+#define GGML_F16_STEP GGML_F32_STEP
+#define GGML_F16_EPR GGML_F32_EPR
+#define GGML_F16_VEC GGML_F32x4
+#define GGML_F16_VEC_ZERO GGML_F32x4_ZERO
+#define GGML_F16_VEC_SET1 GGML_F32x4_SET1
+#define GGML_F16_VEC_FMA GGML_F32x4_FMA
+#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE
+// Use vec_xl, not vec_ld, in case the load address is not aligned.
+#define GGML_F16_VEC_LOAD(p, i) (i & 0x1) ? \
+ vec_extract_fp32_from_shorth(vec_xl(0, p - GGML_F16_EPR)) : \
+ vec_extract_fp32_from_shortl(vec_xl(0, p))
+#define GGML_ENDIAN_BYTE(i) ((unsigned char *)&(uint16_t){1})[i]
+#define GGML_F16_VEC_STORE(p, r, i) \
+ if (i & 0x1) \
+ vec_xst(vec_pack_to_short_fp32(r[i - GGML_ENDIAN_BYTE(1)], \
+ r[i - GGML_ENDIAN_BYTE(0)]), \
+ 0, p - GGML_F16_EPR)
+
+#elif defined(__wasm_simd128__)
+
+#define GGML_SIMD
+
+// F32 WASM
+
+#define GGML_F32_STEP 16
+#define GGML_F32_EPR 4
+
+#define GGML_F32x4 v128_t
+#define GGML_F32x4_ZERO wasm_f32x4_splat(0.0f)
+#define GGML_F32x4_SET1(x) wasm_f32x4_splat(x)
+#define GGML_F32x4_LOAD wasm_v128_load
+#define GGML_F32x4_STORE wasm_v128_store
+#define GGML_F32x4_FMA(a, b, c) wasm_f32x4_add(wasm_f32x4_mul(b, c), a)
+#define GGML_F32x4_ADD wasm_f32x4_add
+#define GGML_F32x4_MUL wasm_f32x4_mul
+#define GGML_F32x4_REDUCE(res, x) \
+{ \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \
+ } \
+ res = wasm_f32x4_extract_lane(x[0], 0) + \
+ wasm_f32x4_extract_lane(x[0], 1) + \
+ wasm_f32x4_extract_lane(x[0], 2) + \
+ wasm_f32x4_extract_lane(x[0], 3); \
+}
+
+#define GGML_F32_VEC GGML_F32x4
+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO
+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1
+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD
+#define GGML_F32_VEC_STORE GGML_F32x4_STORE
+#define GGML_F32_VEC_FMA GGML_F32x4_FMA
+#define GGML_F32_VEC_ADD GGML_F32x4_ADD
+#define GGML_F32_VEC_MUL GGML_F32x4_MUL
+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE
+
+// F16 WASM
+
+#define GGML_F16_STEP 16
+#define GGML_F16_EPR 4
+
+inline static v128_t __wasm_f16x4_load(const ggml_fp16_t * p) {
+ float tmp[4];
+
+ tmp[0] = GGML_FP16_TO_FP32(p[0]);
+ tmp[1] = GGML_FP16_TO_FP32(p[1]);
+ tmp[2] = GGML_FP16_TO_FP32(p[2]);
+ tmp[3] = GGML_FP16_TO_FP32(p[3]);
+
+ return wasm_v128_load(tmp);
+}
+
+inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {
+ float tmp[4];
+
+ wasm_v128_store(tmp, x);
+
+ p[0] = GGML_FP32_TO_FP16(tmp[0]);
+ p[1] = GGML_FP32_TO_FP16(tmp[1]);
+ p[2] = GGML_FP32_TO_FP16(tmp[2]);
+ p[3] = GGML_FP32_TO_FP16(tmp[3]);
+}
+
+#define GGML_F16x4 v128_t
+#define GGML_F16x4_ZERO wasm_f32x4_splat(0.0f)
+#define GGML_F16x4_SET1(x) wasm_f32x4_splat(x)
+#define GGML_F16x4_LOAD(x) __wasm_f16x4_load(x)
+#define GGML_F16x4_STORE(x, y) __wasm_f16x4_store(x, y)
+#define GGML_F16x4_FMA GGML_F32x4_FMA
+#define GGML_F16x4_ADD wasm_f32x4_add
+#define GGML_F16x4_MUL wasm_f32x4_mul
+#define GGML_F16x4_REDUCE(res, x) \
+{ \
+ int offset = GGML_F16_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = wasm_f32x4_add(x[i], x[offset+i]); \
+ } \
+ res = wasm_f32x4_extract_lane(x[0], 0) + \
+ wasm_f32x4_extract_lane(x[0], 1) + \
+ wasm_f32x4_extract_lane(x[0], 2) + \
+ wasm_f32x4_extract_lane(x[0], 3); \
+}
+
+#define GGML_F16_VEC GGML_F16x4
+#define GGML_F16_VEC_ZERO GGML_F16x4_ZERO
+#define GGML_F16_VEC_SET1 GGML_F16x4_SET1
+#define GGML_F16_VEC_LOAD(p, i) GGML_F16x4_LOAD(p)
+#define GGML_F16_VEC_STORE(p, r, i) GGML_F16x4_STORE(p, r[i])
+#define GGML_F16_VEC_FMA GGML_F16x4_FMA
+#define GGML_F16_VEC_ADD GGML_F16x4_ADD
+#define GGML_F16_VEC_MUL GGML_F16x4_MUL
+#define GGML_F16_VEC_REDUCE GGML_F16x4_REDUCE
+
+#elif defined(__SSE3__)
+
+#define GGML_SIMD
+
+// F32 SSE
+
+#define GGML_F32_STEP 32
+#define GGML_F32_EPR 4
+
+#define GGML_F32x4 __m128
+#define GGML_F32x4_ZERO _mm_setzero_ps()
+#define GGML_F32x4_SET1(x) _mm_set1_ps(x)
+#define GGML_F32x4_LOAD _mm_loadu_ps
+#define GGML_F32x4_STORE _mm_storeu_ps
+#if defined(__FMA__)
+ // TODO: Does this work?
+ #define GGML_F32x4_FMA(a, b, c) _mm_fmadd_ps(b, c, a)
+#else
+ #define GGML_F32x4_FMA(a, b, c) _mm_add_ps(_mm_mul_ps(b, c), a)
+#endif
+#define GGML_F32x4_ADD _mm_add_ps
+#define GGML_F32x4_MUL _mm_mul_ps
+#define GGML_F32x4_REDUCE(res, x) \
+{ \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm_add_ps(x[i], x[offset+i]); \
+ } \
+ const __m128 t0 = _mm_hadd_ps(x[0], x[0]); \
+ res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t0, t0)); \
+}
+// TODO: is this optimal ?
+
+#define GGML_F32_VEC GGML_F32x4
+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO
+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1
+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD
+#define GGML_F32_VEC_STORE GGML_F32x4_STORE
+#define GGML_F32_VEC_FMA GGML_F32x4_FMA
+#define GGML_F32_VEC_ADD GGML_F32x4_ADD
+#define GGML_F32_VEC_MUL GGML_F32x4_MUL
+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE
+
+// F16 SSE
+
+#define GGML_F16_STEP 32
+#define GGML_F16_EPR 4
+
+static inline __m128 __sse_f16x4_load(ggml_fp16_t *x) {
+ float tmp[4];
+
+ tmp[0] = GGML_FP16_TO_FP32(x[0]);
+ tmp[1] = GGML_FP16_TO_FP32(x[1]);
+ tmp[2] = GGML_FP16_TO_FP32(x[2]);
+ tmp[3] = GGML_FP16_TO_FP32(x[3]);
+
+ return _mm_loadu_ps(tmp);
+}
+
+static inline void __sse_f16x4_store(ggml_fp16_t *x, __m128 y) {
+ float arr[4];
+
+ _mm_storeu_ps(arr, y);
+
+ x[0] = GGML_FP32_TO_FP16(arr[0]);
+ x[1] = GGML_FP32_TO_FP16(arr[1]);
+ x[2] = GGML_FP32_TO_FP16(arr[2]);
+ x[3] = GGML_FP32_TO_FP16(arr[3]);
+}
+
+#define GGML_F32Cx4 __m128
+#define GGML_F32Cx4_ZERO _mm_setzero_ps()
+#define GGML_F32Cx4_SET1(x) _mm_set1_ps(x)
+#define GGML_F32Cx4_LOAD(x) __sse_f16x4_load(x)
+#define GGML_F32Cx4_STORE(x, y) __sse_f16x4_store(x, y)
+#define GGML_F32Cx4_FMA GGML_F32x4_FMA
+#define GGML_F32Cx4_ADD _mm_add_ps
+#define GGML_F32Cx4_MUL _mm_mul_ps
+#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE
+
+#define GGML_F16_VEC GGML_F32Cx4
+#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO
+#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1
+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)
+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i])
+#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA
+#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD
+#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL
+#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE
+
+#endif
+
+// GGML_F32_ARR / GGML_F16_ARR
+// number of registers to use per step
+#ifdef GGML_SIMD
+#define GGML_F32_ARR (GGML_F32_STEP/GGML_F32_EPR)
+#define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR)
+#endif
+
+void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) {
+ for (int64_t i = 0; i < n; i++) {
+ y[i] = GGML_FP16_TO_FP32(x[i]);
+ }
+}
+
+void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) {
+ int64_t i = 0;
+#if defined(__F16C__)
+ for (; i + 7 < n; i += 8) {
+ __m256 x_vec = _mm256_loadu_ps(x + i);
+ __m128i y_vec = _mm256_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);
+ _mm_storeu_si128((__m128i *)(y + i), y_vec);
+ }
+ for(; i + 3 < n; i += 4) {
+ __m128 x_vec = _mm_loadu_ps(x + i);
+ __m128i y_vec = _mm_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);
+ _mm_storel_epi64((__m128i *)(y + i), y_vec);
+ }
+#endif
+ for (; i < n; i++) {
+ y[i] = GGML_FP32_TO_FP16(x[i]);
+ }
+}
+
+void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {
+ int64_t i = 0;
+#if defined(__AVX512F__)
+ for (; i + 16 <= n; i += 16) {
+ _mm512_storeu_ps(y + i,
+ _mm512_castsi512_ps(
+ _mm512_slli_epi32(
+ _mm512_cvtepu16_epi32(
+ _mm256_loadu_si256(
+ (const __m256i *)(x + i))),
+ 16)));
+ }
+#elif defined(__AVX2__)
+ for (; i + 8 <= n; i += 8) {
+ _mm256_storeu_ps(y + i,
+ _mm256_castsi256_ps(
+ _mm256_slli_epi32(
+ _mm256_cvtepu16_epi32(
+ _mm_loadu_si128(
+ (const __m128i *)(x + i))),
+ 16)));
+ }
+#endif
+ for (; i < n; i++) {
+ y[i] = GGML_BF16_TO_FP32(x[i]);
+ }
+}
+
+void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {
+ int i = 0;
+#if defined(__AVX512BF16__)
+ for (; i + 32 <= n; i += 32) {
+ _mm512_storeu_ps(
+ (__m512 *)(y + i),
+ (__m512)_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16),
+ _mm512_loadu_ps(x + i)));
+ }
+#endif
+ for (; i < n; i++) {
+ y[i] = GGML_FP32_TO_BF16(x[i]);
+ }
+}
+
+//
+// fundamental operations
+//
+
+void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
+
+void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
+
+void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
+
+void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
+
+void ggml_vec_set_bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
+
+void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] + y[i]; }
+void ggml_vec_add1_f32(const int n, float * z, const float * x, const float v) { for (int i = 0; i < n; ++i) z[i] = x[i] + v; }
+void ggml_vec_acc_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] += x[i]; }
+void ggml_vec_acc1_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] += v; }
+void ggml_vec_sub_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] - y[i]; }
+void ggml_vec_set_f32 (const int n, float * x, const float v) { for (int i = 0; i < n; ++i) x[i] = v; }
+void ggml_vec_cpy_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]; }
+void ggml_vec_neg_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = -x[i]; }
+void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]*y[i]; }
+void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]/y[i]; }
+
+void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+#if defined(GGML_SIMD)
+ float sumf = 0.0f;
+ const int np = (n & ~(GGML_F32_STEP - 1));
+
+ GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };
+
+ GGML_F32_VEC ax[GGML_F32_ARR];
+ GGML_F32_VEC ay[GGML_F32_ARR];
+
+ for (int i = 0; i < np; i += GGML_F32_STEP) {
+ for (int j = 0; j < GGML_F32_ARR; j++) {
+ ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);
+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);
+
+ sum[j] = GGML_F32_VEC_FMA(sum[j], ax[j], ay[j]);
+ }
+ }
+
+ // reduce sum0..sum3 to sum0
+ GGML_F32_VEC_REDUCE(sumf, sum);
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ sumf += x[i]*y[i];
+ }
+#else
+ // scalar
+ ggml_float sumf = 0.0;
+ for (int i = 0; i < n; ++i) {
+ sumf += (ggml_float)(x[i]*y[i]);
+ }
+#endif
+
+ *s = sumf;
+}
+
+void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+ int i = 0;
+ ggml_float sumf = 0;
+
+#if defined(__AVX512BF16__)
+ __m512 c1 = _mm512_setzero_ps();
+ __m512 c2 = _mm512_setzero_ps();
+ for (; i + 64 <= n; i += 64) {
+ c1 = _mm512_dpbf16_ps(c1, (__m512bh)_mm512_loadu_ps((const float *)(x + i)),
+ (__m512bh)_mm512_loadu_ps((const float *)(y + i)));
+ c2 = _mm512_dpbf16_ps(c2, (__m512bh)_mm512_loadu_ps((const float *)(x + i + 32)),
+ (__m512bh)_mm512_loadu_ps((const float *)(y + i + 32)));
+ }
+ sumf += (ggml_float)_mm512_reduce_add_ps(c1);
+ sumf += (ggml_float)_mm512_reduce_add_ps(c2);
+
+#elif defined(__AVX512F__)
+#define LOAD(p) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)(p))), 16))
+ __m512 c1 = _mm512_setzero_ps();
+ __m512 c2 = _mm512_setzero_ps();
+ for (; i + 32 <= n; i += 32) {
+ c1 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i), LOAD(y + i)), c1);
+ c2 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c2);
+ }
+ sumf += (ggml_float)_mm512_reduce_add_ps(c1);
+ sumf += (ggml_float)_mm512_reduce_add_ps(c2);
+
+#undef LOAD
+#elif defined(__AVX2__)
+#define LOAD(p) _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)(p))), 16))
+ __m256 c1 = _mm256_setzero_ps();
+ __m256 c2 = _mm256_setzero_ps();
+ __m256 c3 = _mm256_setzero_ps();
+ __m256 c4 = _mm256_setzero_ps();
+ for (; i + 32 <= n; i += 32) {
+ c1 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i), LOAD(y + i)), c1);
+ c2 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 8), LOAD(y + i + 8)), c2);
+ c3 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c3);
+ c4 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 24), LOAD(y + i + 24)), c4);
+ }
+ __m128 g;
+ c1 = _mm256_add_ps(_mm256_add_ps(c1, c3),
+ _mm256_add_ps(c2, c4));
+ g = _mm_add_ps(_mm256_extractf128_ps(c1, 1),
+ _mm256_castps256_ps128(c1));
+ g = _mm_add_ps(g, _mm_movehl_ps(g, g));
+ g = _mm_add_ss(g, _mm_movehdup_ps(g));
+ sumf += (ggml_float)_mm_cvtss_f32(g);
+
+#undef LOAD
+#endif
+
+ for (; i < n; ++i) {
+ sumf += (ggml_float)(GGML_BF16_TO_FP32(x[i]) *
+ GGML_BF16_TO_FP32(y[i]));
+ }
+ *s = sumf;
+}
+
+void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ ggml_float sumf = 0.0;
+
+#if defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F16_STEP - 1));
+
+ GGML_F16_VEC sum[GGML_F16_ARR] = { GGML_F16_VEC_ZERO };
+
+ GGML_F16_VEC ax[GGML_F16_ARR];
+ GGML_F16_VEC ay[GGML_F16_ARR];
+
+ for (int i = 0; i < np; i += GGML_F16_STEP) {
+ for (int j = 0; j < GGML_F16_ARR; j++) {
+ ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);
+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
+
+ sum[j] = GGML_F16_VEC_FMA(sum[j], ax[j], ay[j]);
+ }
+ }
+
+ // reduce sum0..sum3 to sum0
+ GGML_F16_VEC_REDUCE(sumf, sum);
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ sumf += (ggml_float)(GGML_FP16_TO_FP32(x[i])*GGML_FP16_TO_FP32(y[i]));
+ }
+#else
+ for (int i = 0; i < n; ++i) {
+ sumf += (ggml_float)(GGML_FP16_TO_FP32(x[i])*GGML_FP16_TO_FP32(y[i]));
+ }
+#endif
+
+ *s = sumf;
+}
+
+// compute GGML_VEC_DOT_UNROLL dot products at once
+// xs - x row stride in bytes
+void ggml_vec_dot_f16_unroll(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y) {
+ ggml_float sumf[GGML_VEC_DOT_UNROLL] = { 0.0 };
+
+ ggml_fp16_t * restrict x[GGML_VEC_DOT_UNROLL];
+
+ for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {
+ x[i] = (ggml_fp16_t *) ((char *) xv + i*xs);
+ }
+
+#if defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F16_STEP - 1));
+
+ GGML_F16_VEC sum[GGML_VEC_DOT_UNROLL][GGML_F16_ARR] = { { GGML_F16_VEC_ZERO } };
+
+ GGML_F16_VEC ax[GGML_F16_ARR];
+ GGML_F16_VEC ay[GGML_F16_ARR];
+
+ for (int i = 0; i < np; i += GGML_F16_STEP) {
+ for (int j = 0; j < GGML_F16_ARR; j++) {
+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
+
+ for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {
+ ax[j] = GGML_F16_VEC_LOAD(x[k] + i + j*GGML_F16_EPR, j);
+
+ sum[k][j] = GGML_F16_VEC_FMA(sum[k][j], ax[j], ay[j]);
+ }
+ }
+ }
+
+ // reduce sum0..sum3 to sum0
+ for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {
+ GGML_F16_VEC_REDUCE(sumf[k], sum[k]);
+ }
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {
+ sumf[j] += (ggml_float)(GGML_FP16_TO_FP32(x[j][i])*GGML_FP16_TO_FP32(y[i]));
+ }
+ }
+#else
+ for (int i = 0; i < n; ++i) {
+ for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {
+ sumf[j] += (ggml_float)(GGML_FP16_TO_FP32(x[j][i])*GGML_FP16_TO_FP32(y[i]));
+ }
+ }
+#endif
+
+ for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {
+ s[i] = sumf[i];
+ }
+}
+
+void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v) {
+#if defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F32_STEP - 1));
+
+ GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);
+
+ GGML_F32_VEC ax[GGML_F32_ARR];
+ GGML_F32_VEC ay[GGML_F32_ARR];
+
+ for (int i = 0; i < np; i += GGML_F32_STEP) {
+ for (int j = 0; j < GGML_F32_ARR; j++) {
+ ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);
+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);
+ ay[j] = GGML_F32_VEC_FMA(ay[j], ax[j], vx);
+
+ GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);
+ }
+ }
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ y[i] += x[i]*v;
+ }
+#else
+ // scalar
+ for (int i = 0; i < n; ++i) {
+ y[i] += x[i]*v;
+ }
+#endif
+}
+
+void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v) {
+#if defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F16_STEP - 1));
+
+ GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);
+
+ GGML_F16_VEC ax[GGML_F16_ARR];
+ GGML_F16_VEC ay[GGML_F16_ARR];
+
+ for (int i = 0; i < np; i += GGML_F16_STEP) {
+ for (int j = 0; j < GGML_F16_ARR; j++) {
+ ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);
+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
+ ay[j] = GGML_F16_VEC_FMA(ay[j], ax[j], vx);
+
+ GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);
+ }
+ }
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i]) + GGML_FP16_TO_FP32(x[i])*v);
+ }
+#else
+ // scalar
+ for (int i = 0; i < n; ++i) {
+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i]) + GGML_FP16_TO_FP32(x[i])*v);
+ }
+#endif
+}
+
+// xs and vs are byte strides of x and v
+void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv) {
+
+ const float * restrict x[GGML_VEC_MAD_UNROLL];
+ const float * restrict v[GGML_VEC_MAD_UNROLL];
+
+ for (int i = 0; i < GGML_VEC_MAD_UNROLL; ++i) {
+ x[i] = (const float *) ((const char *) xv + i*xs);
+ v[i] = (const float *) ((const char *) vv + i*vs);
+ }
+
+#if defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F32_STEP - 1));
+
+ GGML_F32_VEC vx[GGML_VEC_MAD_UNROLL];
+
+ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {
+ vx[k] = GGML_F32_VEC_SET1(v[k][0]);
+ }
+
+ GGML_F32_VEC ax[GGML_VEC_MAD_UNROLL][GGML_F32_ARR];
+ GGML_F32_VEC ay[GGML_F32_ARR];
+
+ for (int i = 0; i < np; i += GGML_F32_STEP) {
+ for (int j = 0; j < GGML_F32_ARR; j++) {
+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);
+
+ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {
+ ax[k][j] = GGML_F32_VEC_LOAD(x[k] + i + j*GGML_F32_EPR);
+ ay[j] = GGML_F32_VEC_FMA(ay[j], ax[k][j], vx[k]);
+ }
+
+ GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);
+ }
+ }
+
+ // leftovers
+ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {
+ for (int i = np; i < n; ++i) {
+ y[i] += x[k][i]*v[k][0];
+ }
+ }
+#else
+ // scalar
+ for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {
+ for (int i = 0; i < n; ++i) {
+ y[i] += x[k][i]*v[k][0];
+ }
+ }
+#endif
+}
+
+//inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] *= v; }
+void ggml_vec_scale_f32(const int n, float * y, const float v) {
+#if defined(GGML_USE_ACCELERATE)
+ vDSP_vsmul(y, 1, &v, y, 1, n);
+#elif defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F32_STEP - 1));
+
+ GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);
+
+ GGML_F32_VEC ay[GGML_F32_ARR];
+
+ for (int i = 0; i < np; i += GGML_F32_STEP) {
+ for (int j = 0; j < GGML_F32_ARR; j++) {
+ ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);
+ ay[j] = GGML_F32_VEC_MUL(ay[j], vx);
+
+ GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);
+ }
+ }
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ y[i] *= v;
+ }
+#else
+ // scalar
+ for (int i = 0; i < n; ++i) {
+ y[i] *= v;
+ }
+#endif
+}
+
+void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {
+#if defined(GGML_SIMD)
+ const int np = (n & ~(GGML_F16_STEP - 1));
+
+ GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);
+
+ GGML_F16_VEC ay[GGML_F16_ARR];
+
+ for (int i = 0; i < np; i += GGML_F16_STEP) {
+ for (int j = 0; j < GGML_F16_ARR; j++) {
+ ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
+ ay[j] = GGML_F16_VEC_MUL(ay[j], vx);
+
+ GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);
+ }
+ }
+
+ // leftovers
+ for (int i = np; i < n; ++i) {
+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i])*v);
+ }
+#else
+ // scalar
+ for (int i = 0; i < n; ++i) {
+ y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i])*v);
+ }
+#endif
+}
+
+void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, 0, x, 0, x, 0, 1); *s = sqrtf(*s); }
+void ggml_vec_sqr_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i]; }
+void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrtf(x[i]); }
+void ggml_vec_log_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = logf(x[i]); }
+void ggml_vec_abs_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fabsf(x[i]); }
+void ggml_vec_sgn_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : ((x[i] < 0.f) ? -1.f : 0.f); }
+void ggml_vec_step_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : 0.f; }
+void ggml_vec_tanh_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = tanhf(x[i]); }
+void ggml_vec_elu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : expf(x[i])-1; }
+void ggml_vec_relu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.f; }
+void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); }
+// TODO: optimize performance
+void ggml_vec_hardswish_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }
+void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }
+
+// ACTIVATION FUNCTIONS
+//
+// 0.3 +--------------------------------------------------------------------------------------+
+// | + + + + + + + * % |
+// | x>0?x:0.01 **%**** |
+// | x/(1+exp(-x)) #%##### |
+// | .5*x*(1+erf(x/sqrt(2)))*$%$$$$$ |
+// 0.2 |-+ x*(1/(1+exp(-1.702*x)))*%%%%%%%-|
+// | *%# |
+// | *% |
+// | *% |
+// | *% |
+// 0.1 |-+ *% +-|
+// | *% |
+// | % |
+// | *% |
+// | % |
+// 0 |***************************************************************************% +-|
+// |%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%$$$$$ % |
+// | ########## %%%%%$$$$ % |
+// | ##### %%%%$$ % |
+// | #### %%$$ % |
+// | #### %%%$ % |
+// -0.1 |-+ ### %%% %# +-|
+// | ### % %# |
+// | ### %%% %%# |
+// | ## $%%%%$ # |
+// | ### # |
+// -0.2 |-+ ## # +-|
+// | ### # |
+// | ## # |
+// | #### ### |
+// | + + + + + #####+ + |
+// -0.3 +--------------------------------------------------------------------------------------+
+// -7 -6 -5 -4 -3 -2 -1 0 1
+//
+
+inline static float ggml_silu_f32(float x) {
+ // SiLU is the favored by LLaMA, Mistral, Phi, Rocket, etc.
+ return x/(1.f + expf(-x));
+}
+
+inline static float ggml_gelu_f32(float x) {
+ // GeLU approximation that goes slower and we seem to be stuck with.
+ return .5f * x * (1.f + tanhf(sqrtf(M_2_PI) * (x + .044715f * x * x * x)));
+}
+
+inline static float ggml_gelu_quick_f32(float x) {
+ // "Quick" GeLU is used by LLaVA's CLIP vision model.
+ return x*(1.f/(1.f+expf(-1.702f*x)));
+}
+
+static void ggml_vec_gelu_f16_init(void) {
+ ggml_table_gelu_f16 = malloc(sizeof(float) * 65536);
+ for (int i = 0; i < 65536; ++i) {
+ union {
+ unsigned short i;
+ ggml_fp16_t f;
+ } u = {i};
+ ggml_table_gelu_f16[i] = ggml_gelu_f32(u.f);
+ }
+}
+
+void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
+ static atomic_uint once;
+ ggml_once(&once, ggml_vec_gelu_f16_init);
+ const uint16_t * i16 = (const uint16_t *) x;
+ for (int i = 0; i < n; ++i) {
+ y[i] = ggml_table_gelu_f16[i16[i]];
+ }
+}
+
+void ggml_vec_gelu_f32(const int n, float * y, const float * x) {
+ for (int i = 0; i < n; ++i) {
+ y[i] = ggml_gelu_f32(x[i]);
+ }
+}
+
+void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {
+ for (int i = 0; i < n; ++i) {
+ y[i] = ggml_gelu_quick_f32(x[i]);
+ }
+}
+
+void ggml_vec_silu_f32(const int n, float * y, const float * x) {
+ for (int i = 0; i < n; ++i) {
+ y[i] = ggml_silu_f32(x[i]);
+ }
+}
+
+float ggml_silu_backward_f32(float x, float dy) {
+ const float s = 1.0f/(1.0f + expf(-x));
+ return dy*s*(1.0f + x*(1.0f - s));
+}
+
+void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, const float * dy) {
+ for (int i = 0; i < n; ++i) {
+ dx[i] = ggml_silu_backward_f32(x[i], dy[i]);
+ }
+}
+
+void ggml_vec_sum_f32(const int n, float * s, const float * x) {
+#ifndef GGML_USE_ACCELERATE
+ ggml_float sum = 0.0;
+ for (int i = 0; i < n; ++i) {
+ sum += (ggml_float)x[i];
+ }
+ *s = sum;
+#else
+ vDSP_sve(x, 1, s, n);
+#endif
+}
+
+void ggml_vec_sum_f32_ggf(const int n, ggml_float * s, const float * x) {
+ ggml_float sum = 0.0;
+ for (int i = 0; i < n; ++i) {
+ sum += (ggml_float)x[i];
+ }
+ *s = sum;
+}
+
+void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x) {
+ float sum = 0.0f;
+ for (int i = 0; i < n; ++i) {
+ sum += GGML_FP16_TO_FP32(x[i]);
+ }
+ *s = sum;
+}
+
+void ggml_vec_max_f32(const int n, float * s, const float * x) {
+#ifndef GGML_USE_ACCELERATE
+ float max = -INFINITY;
+ for (int i = 0; i < n; ++i) {
+ max = MAX(max, x[i]);
+ }
+ *s = max;
+#else
+ vDSP_maxv(x, 1, s, n);
+#endif
+}
+
+void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) {
+ ggml_vec_norm_f32(n, s, x);
+ *s = 1.f/(*s);
+}
+
+void ggml_vec_argmax_f32(const int n, int * s, const float * x) {
+ float max = -INFINITY;
+ int idx = 0;
+ for (int i = 0; i < n; ++i) {
+ max = MAX(max, x[i]);
+ if (max == x[i]) { idx = i; }
+ }
+ *s = idx;
+}
diff --git a/llama.cpp/ggml-vector.py b/llama.cpp/ggml-vector.py
new file mode 100644
index 00000000..528bf60d
--- /dev/null
+++ b/llama.cpp/ggml-vector.py
@@ -0,0 +1,67 @@
+import re
+
+with open('llama.cpp/ggml-vector.h') as f:
+ prototypes = f.read()
+prototypes = [line.replace(';', '') for line in prototypes.split('\n')
+ if line.endswith(';') and not line.startswith('//')]
+FUNCS = [(re.search(r'(?<= )\w+', proto).group(0), proto)
+ for proto in prototypes]
+
+# BEGIN SPECIAL FUNCTIONS
+# FUNCS.append(('ggml_validate_row_data', 'bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes)'))
+# END SPECIAL FUNCTIONS
+
+ARCHS = (
+ ('amd_avx512bf16', '__x86_64__', ('X86_HAVE(FMA)', 'X86_HAVE(F16C)', 'X86_HAVE(AVX2)', 'X86_HAVE(AVX512F)', 'X86_HAVE(AVX512VL)', 'X86_HAVE(AVX512_BF16)')),
+ ('amd_avx512', '__x86_64__', ('X86_HAVE(FMA)', 'X86_HAVE(F16C)', 'X86_HAVE(AVX2)', 'X86_HAVE(AVX512F)')),
+ ('amd_avx2', '__x86_64__', ('X86_HAVE(FMA)', 'X86_HAVE(F16C)', 'X86_HAVE(AVX2)')),
+ ('amd_f16c', '__x86_64__', ('X86_HAVE(F16C)',)),
+ ('amd_fma', '__x86_64__', ('X86_HAVE(FMA)',)),
+ ('amd_avx', '__x86_64__', ()),
+ ('arm82', '__aarch64__', ('(getauxval(AT_HWCAP) & HWCAP_FPHP)', '(getauxval(AT_HWCAP) & HWCAP_ASIMDHP)')),
+ ('arm80', '__aarch64__', ()),
+)
+
+for arch, mac, needs in ARCHS:
+ path = 'llama.cpp/ggml-vector-%s.c' % (arch.replace('_', '-'))
+ with open(path, 'w') as f:
+ f.write('#ifdef %s\n' % (mac))
+ for func, proto in FUNCS:
+ f.write('#define %s %s_%s\n' % (func, func, arch))
+ f.write('#define GGML_VECTOR\n')
+ f.write('#include "ggml-vector.inc"\n')
+ f.write('#endif // %s\n' % (mac))
+
+with open('llama.cpp/ggml-vector.cpp', 'w') as f:
+ f.write('#include <cosmo.h>\n')
+ f.write('#include <sys/auxv.h>\n')
+ f.write('#include <libc/sysv/consts/hwcap.h>\n')
+ f.write('#include "ggml-vector.h"\n')
+ f.write('\n')
+ for func, proto in FUNCS:
+ for arch, mac, needs in ARCHS:
+ f.write('extern "C" %s;\n' % (proto.replace(func, '%s_%s' % (func, arch))))
+ f.write('\n')
+ f.write('static const struct VectorFuncs {\n')
+ for func, proto in FUNCS:
+ f.write(' typeof(%s) *ptr_%s;\n' % (func, func))
+ f.write('\n')
+ f.write(' VectorFuncs() {\n')
+ for arch, mac, needs in ARCHS:
+ f.write('#ifdef %s\n' % (mac))
+ f.write(' if (%s) {\n' % (' && '.join(needs) or '1'))
+ for func, proto in FUNCS:
+ f.write(' ptr_%s = %s_%s;\n' % (func, func, arch))
+ f.write(' return;\n')
+ f.write(' }\n')
+ f.write('#endif\n')
+ f.write(' }\n')
+ f.write('} funcs;\n')
+ f.write('\n')
+ for func, proto in FUNCS:
+ proto = proto.replace(';', '')
+ args = [s.split(' ')[-1] for s in re.search(r'(?<=\().*(?=\))', proto).group(0).split(',')]
+ f.write(proto + ' {\n')
+ f.write(' return funcs.ptr_%s(%s);\n' % (func, ', '.join(args)))
+ f.write('}\n')
+ f.write('\n')
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 9970c813..f4012a25 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -35,6 +35,7 @@ SOFTWARE.");
#include "ggml-quants.h"
#include "ggml-metal.h"
#include "ggml-cuda.h"
+#include "ggml-vector.h"
#include "llamafile/llamafile.h"
#include "llamafile/log.h"
@@ -117,11 +118,6 @@ void ggml_print_backtrace(void) {
/*#define GGML_PERF*/
#define GGML_DEBUG 0
-#define GGML_KAHAN 0
-
-#define GGML_SOFT_MAX_UNROLL 4
-#define GGML_VEC_DOT_UNROLL 2
-#define GGML_VEC_MAD_UNROLL 32
//
// logging
@@ -246,7 +242,7 @@ typedef double ggml_float;
// global data
//
-static float *ggml_table_gelu_f16;
+float *ggml_table_gelu_f16;
// precomputed f32 table for f16 (256 KB) (ggml-impl.h)
float ggml_table_f32_f16[1 << 16];
@@ -262,147 +258,24 @@ GGML_CALL const char * ggml_status_to_string(enum ggml_status status) {
return "GGML status: unknown";
}
-// note: do not use these inside ggml.c
-// these are meant to be used via the ggml.h API
float ggml_fp16_to_fp32(ggml_fp16_t x) {
+#define ggml_fp16_to_fp32 do_not_use__ggml_fp16_to_fp32__in_ggml
return GGML_FP16_TO_FP32(x);
}
ggml_fp16_t ggml_fp32_to_fp16(float x) {
+#define ggml_fp32_to_fp16 do_not_use__ggml_fp32_to_fp16__in_ggml
return GGML_FP32_TO_FP16(x);
}
-void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) {
- for (int64_t i = 0; i < n; i++) {
- y[i] = GGML_FP16_TO_FP32(x[i]);
- }
-}
-
-#ifdef __x86_64__
-#pragma GCC push_options
-#pragma GCC target("avx512f")
-static void ggml_bf16_to_fp32_row_avx512f(const ggml_bf16_t * x, float * y, int64_t n) {
- int64_t i = 0;
- for (; i + 16 <= n; i += 16) {
- _mm512_storeu_ps(y + i,
- _mm512_castsi512_ps(
- _mm512_slli_epi32(
- _mm512_cvtepu16_epi32(
- _mm256_loadu_si256(
- (const __m256i *)(x + i))),
- 16)));
- }
- for (; i < n; i++) {
- y[i] = GGML_BF16_TO_FP32(x[i]);
- }
-}
-#pragma GCC pop_options
-#endif
-
-#ifdef __x86_64__
-#pragma GCC push_options
-#pragma GCC target("avx2")
-static void ggml_bf16_to_fp32_row_avx2(const ggml_bf16_t * x, float * y, int64_t n) {
- int64_t i = 0;
- for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(y + i,
- _mm256_castsi256_ps(
- _mm256_slli_epi32(
- _mm256_cvtepu16_epi32(
- _mm_loadu_si128(
- (const __m128i *)(x + i))),
- 16)));
- }
- for (; i < n; i++) {
- y[i] = GGML_BF16_TO_FP32(x[i]);
- }
-}
-#pragma GCC pop_options
-#endif
-
-void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {
-#ifdef __x86_64__
- if (X86_HAVE(AVX512F))
- return ggml_bf16_to_fp32_row_avx512f(x, y, n);
- if (X86_HAVE(AVX2))
- return ggml_bf16_to_fp32_row_avx2(x, y, n);
-#endif
- for (int i = 0; i < n; i++) {
- y[i] = GGML_BF16_TO_FP32(x[i]);
- }
-}
-
-#ifdef __x86_64__
-#pragma GCC push_options
-#pragma GCC target("f16c")
-static void ggml_fp32_to_fp16_row_f16c(const float * x, ggml_fp16_t * y, int64_t n) {
- int64_t i = 0;
- for (; i + 7 < n; i += 8) {
- __m256 x_vec = _mm256_loadu_ps(x + i);
- __m128i y_vec = _mm256_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);
- _mm_storeu_si128((__m128i *)(y + i), y_vec);
- }
- for(; i + 3 < n; i += 4) {
- __m128 x_vec = _mm_loadu_ps(x + i);
- __m128i y_vec = _mm_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);
- _mm_storel_epi64((__m128i *)(y + i), y_vec);
- }
- for (; i < n; i++) {
- y[i] = GGML_FP32_TO_FP16(x[i]);
- }
-}
-#pragma GCC pop_options
-#endif
-
-#ifdef __aarch64__
-static void ggml_fp32_to_fp16_row_neon(const float *x, ggml_fp16_t *y, int64_t n) {
- int64_t i = 0;
- for (; i + 3 < n; i += 4) {
- vst1_f16((ggml_fp16_t *)(y + i), vcvt_f16_f32(vld1q_f32(x + i)));
- }
- for (; i < n; i++) {
- y[i] = GGML_FP32_TO_FP16(x[i]);
- }
-}
-#endif
-
-void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) {
-#ifdef __x86_64__
- if (X86_HAVE(F16C)) {
- return ggml_fp32_to_fp16_row_f16c(x, y, n);
- }
-#elif defined(__aarch64__)
- return ggml_fp32_to_fp16_row_neon(x, y, n);
-#endif
- for (int64_t i = 0; i < n; i++) {
- y[i] = GGML_FP32_TO_FP16(x[i]);
- }
+float ggml_bf16_to_fp32(ggml_bf16_t x) {
+#define ggml_bf16_to_fp32 do_not_use__ggml_bf16_to_fp32__in_ggml
+ return GGML_BF16_TO_FP32(x); // it just left shifts
}
-#ifdef __x86_64__
-#pragma GCC push_options
-#pragma GCC target("avx512bf16")
-static void ggml_fp32_to_bf16_row_avx512bf16(const float * x, ggml_bf16_t * y, int64_t n) {
- int64_t i = 0;
- for (; i + 32 <= n; i += 32)
- _mm512_storeu_ps(
- (__m512 *)(y + i),
- (__m512)_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16),
- _mm512_loadu_ps(x + i)));
- for (; i < n; i++)
- y[i] = GGML_FP32_TO_BF16(x[i]);
-}
-#pragma GCC pop_options
-#endif
-
-void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {
-#ifdef __x86_64__
- if (X86_HAVE(AVX512_BF16))
- return ggml_fp32_to_bf16_row_avx512bf16(x, y, n);
-#endif
- for (int64_t i = 0; i < n; i++) {
- y[i] = GGML_FP32_TO_BF16(x[i]);
- }
+ggml_bf16_t ggml_fp32_to_bf16(float x) {
+#define ggml_fp32_to_bf16 do_not_use__ggml_fp32_to_bf16__in_ggml
+ return GGML_FP32_TO_BF16(x);
}
GGML_CALL bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b) {
@@ -471,10 +344,6 @@ FILE * ggml_fopen(const char * fname, const char * mode) {
static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float);
-static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
-static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
-static void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
-
static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {
[GGML_TYPE_I8] = {
.type_name = "i8",
@@ -852,25 +721,9 @@ ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type) {
// number of elements to fit in a single register
//
-#define GGML_F32x8_REDUCE_AVX(res, x) \
-do { \
- int offset = (16/4) >> 1; \
- for (int i = 0; i < offset; ++i) { \
- x[i] = _mm256_add_ps(x[i], x[offset+i]); \
- } \
- offset >>= 1; \
- for (int i = 0; i < offset; ++i) { \
- x[i] = _mm256_add_ps(x[i], x[offset+i]); \
- } \
- offset >>= 1; \
- for (int i = 0; i < offset; ++i) { \
- x[i] = _mm256_add_ps(x[i], x[offset+i]); \
- } \
- const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), \
- _mm256_extractf128_ps(x[0], 1)); \
- const __m128 t1 = _mm_hadd_ps(t0, t0); \
- res = _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); \
-} while (0)
+#define GGML_SOFT_MAX_UNROLL 4
+#define GGML_VEC_DOT_UNROLL 2
+#define GGML_VEC_MAD_UNROLL 32
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)
@@ -888,7 +741,6 @@ do { \
#define GGML_F32x4_STORE vst1q_f32
#define GGML_F32x4_FMA(a, b, c) vfmaq_f32(a, b, c)
#define GGML_F32x4_ADD vaddq_f32
-#define GGML_F32x4_SUB vsubq_f32
#define GGML_F32x4_MUL vmulq_f32
#define GGML_F32x4_REDUCE_ONE(x) vaddvq_f32(x)
#define GGML_F32x4_REDUCE(res, x) \
@@ -915,7 +767,6 @@ do { \
#define GGML_F32_VEC_STORE GGML_F32x4_STORE
#define GGML_F32_VEC_FMA GGML_F32x4_FMA
#define GGML_F32_VEC_ADD GGML_F32x4_ADD
-#define GGML_F32_VEC_SUB GGML_F32x4_SUB
#define GGML_F32_VEC_MUL GGML_F32x4_MUL
#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE
@@ -932,7 +783,6 @@ do { \
#define GGML_F16x8_STORE vst1q_f16
#define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c)
#define GGML_F16x8_ADD vaddq_f16
- #define GGML_F16x8_SUB vsubq_f16
#define GGML_F16x8_MUL vmulq_f16
#define GGML_F16x8_REDUCE(res, x) \
do { \
@@ -960,7 +810,6 @@ do { \
#define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((ggml_fp16_internal_t *)(p), r[i])
#define GGML_F16_VEC_FMA GGML_F16x8_FMA
#define GGML_F16_VEC_ADD GGML_F16x8_ADD
- #define GGML_F16_VEC_SUB GGML_F16x8_SUB
#define GGML_F16_VEC_MUL GGML_F16x8_MUL
#define GGML_F16_VEC_REDUCE GGML_F16x8_REDUCE
#else
@@ -977,7 +826,6 @@ do { \
#define GGML_F32Cx4_STORE(x, y) vst1_f16(x, vcvt_f16_f32(y))
#define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c)
#define GGML_F32Cx4_ADD vaddq_f32
- #define GGML_F32Cx4_SUB vsubq_f32
#define GGML_F32Cx4_MUL vmulq_f32
#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE
@@ -988,7 +836,6 @@ do { \
#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((ggml_fp16_internal_t *)(p), r[i])
#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA
#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD
- #define GGML_F16_VEC_SUB GGML_F32Cx4_SUB
#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL
#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE
#endif
@@ -1010,7 +857,6 @@ do { \
// _mm512_fmadd_ps is defined in AVX512F so no guard is required
#define GGML_F32x16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)
#define GGML_F32x16_ADD _mm512_add_ps
-#define GGML_F32x16_SUB _mm512_sub_ps
#define GGML_F32x16_MUL _mm512_mul_ps
#define GGML_F32x16_REDUCE(res, x) \
do { \
@@ -1038,7 +884,6 @@ do { \
#define GGML_F32_VEC_STORE GGML_F32x16_STORE
#define GGML_F32_VEC_FMA GGML_F32x16_FMA
#define GGML_F32_VEC_ADD GGML_F32x16_ADD
-#define GGML_F32_VEC_SUB GGML_F32x16_SUB
#define GGML_F32_VEC_MUL GGML_F32x16_MUL
#define GGML_F32_VEC_REDUCE GGML_F32x16_REDUCE
@@ -1062,7 +907,6 @@ do { \
#define GGML_F32Cx16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)
#define GGML_F32Cx16_ADD _mm512_add_ps
-#define GGML_F32Cx16_SUB _mm512_sub_ps
#define GGML_F32Cx16_MUL _mm512_mul_ps
#define GGML_F32Cx16_REDUCE(res, x) \
do { \
@@ -1088,7 +932,6 @@ do { \
#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx16_STORE(p, r[i])
#define GGML_F16_VEC_FMA GGML_F32Cx16_FMA
#define GGML_F16_VEC_ADD GGML_F32Cx16_ADD
-#define GGML_F16_VEC_SUB GGML_F32Cx16_SUB
#define GGML_F16_VEC_MUL GGML_F32Cx16_MUL
#define GGML_F16_VEC_REDUCE GGML_F32Cx16_REDUCE
@@ -1112,9 +955,26 @@ do { \
#define GGML_F32x8_FMA(a, b, c) _mm256_add_ps(_mm256_mul_ps(b, c), a)
#endif
#define GGML_F32x8_ADD _mm256_add_ps
-#define GGML_F32x8_SUB _mm256_sub_ps
#define GGML_F32x8_MUL _mm256_mul_ps
-#define GGML_F32x8_REDUCE(res, x) GGML_F32x8_REDUCE_AVX(res, x)
+#define GGML_F32x8_REDUCE(res, x) \
+do { \
+ int offset = GGML_F32_ARR >> 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \
+ } \
+ offset >>= 1; \
+ for (int i = 0; i < offset; ++i) { \
+ x[i] = _mm256_add_ps(x[i], x[offset+i]); \
+ } \
+ const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), \
+ _mm256_extractf128_ps(x[0], 1)); \
+ const __m128 t1 = _mm_hadd_ps(t0, t0); \
+ res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); \
+} while (0)
// TODO: is this optimal ?
#define GGML_F32_VEC GGML_F32x8
@@ -1124,7 +984,6 @@ do { \
#define GGML_F32_VEC_STORE GGML_F32x8_STORE
#define GGML_F32_VEC_FMA GGML_F32x8_FMA
#define GGML_F32_VEC_ADD GGML_F32x8_ADD
-#define GGML_F32_VEC_SUB GGML_F32x8_SUB
#define GGML_F32_VEC_MUL GGML_F32x8_MUL
#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE
@@ -1144,7 +1003,7 @@ do { \
#define GGML_F32Cx8_LOAD(x) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x)))
#define GGML_F32Cx8_STORE(x, y) _mm_storeu_si128((__m128i *)(x), _mm256_cvtps_ph(y, 0))
#else
-static inline __m256 __avx_f32cx8_load(const ggml_fp16_t *x) {
+static inline __m256 __avx_f32cx8_load(ggml_fp16_t *x) {
float tmp[8];
for (int i = 0; i < 8; i++) {
@@ -1167,7 +1026,6 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {
#define GGML_F32Cx8_FMA GGML_F32x8_FMA
#define GGML_F32Cx8_ADD _mm256_add_ps
-#define GGML_F32Cx8_SUB _mm256_sub_ps
#define GGML_F32Cx8_MUL _mm256_mul_ps
#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE
@@ -1178,7 +1036,6 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {
#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i])
#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA
#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD
-#define GGML_F16_VEC_SUB GGML_F32Cx8_SUB
#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL
#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE
@@ -1198,7 +1055,6 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {
#define GGML_F32x4_STORE(p, r) vec_xst(r, 0, p)
#define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a)
#define GGML_F32x4_ADD vec_add
-#define GGML_F32x4_SUB vec_sub
#define GGML_F32x4_MUL vec_mul
#define GGML_F32x4_REDUCE(res, x) \
{ \
@@ -1227,7 +1083,6 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {
#define GGML_F32_VEC_STORE GGML_F32x4_STORE
#define GGML_F32_VEC_FMA GGML_F32x4_FMA
#define GGML_F32_VEC_ADD GGML_F32x4_ADD
-#define GGML_F32_VEC_SUB GGML_F32x4_SUB
#define GGML_F32_VEC_MUL GGML_F32x4_MUL
#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE
@@ -1266,7 +1121,6 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {
#define GGML_F32x4_STORE wasm_v128_store
#define GGML_F32x4_FMA(a, b, c) wasm_f32x4_add(wasm_f32x4_mul(b, c), a)
#define GGML_F32x4_ADD wasm_f32x4_add
-#define GGML_F32x4_SUB wasm_f32x4_sub
#define GGML_F32x4_MUL wasm_f32x4_mul
#define GGML_F32x4_REDUCE(res, x) \
{ \
@@ -1295,7 +1149,6 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {
#define GGML_F32_VEC_STORE GGML_F32x4_STORE
#define GGML_F32_VEC_FMA GGML_F32x4_FMA
#define GGML_F32_VEC_ADD GGML_F32x4_ADD
-#define GGML_F32_VEC_SUB GGML_F32x4_SUB
#define GGML_F32_VEC_MUL GGML_F32x4_MUL
#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE
@@ -1333,7 +1186,6 @@ inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {
#define GGML_F16x4_STORE(x, y) __wasm_f16x4_store(x, y)
#define GGML_F16x4_FMA GGML_F32x4_FMA
#define GGML_F16x4_ADD wasm_f32x4_add
-#define GGML_F16x4_SUB wasm_f32x4_sub
#define GGML_F16x4_MUL wasm_f32x4_mul
#define GGML_F16x4_REDUCE(res, x) \
{ \
@@ -1362,7 +1214,6 @@ inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {
#define GGML_F16_VEC_STORE(p, r, i) GGML_F16x4_STORE(p, r[i])
#define GGML_F16_VEC_FMA GGML_F16x4_FMA
#define GGML_F16_VEC_ADD GGML_F16x4_ADD
-#define GGML_F16_VEC_SUB GGML_F16x4_SUB
#define GGML_F16_VEC_MUL GGML_F16x4_MUL
#define GGML_F16_VEC_REDUCE GGML_F16x4_REDUCE
@@ -1387,7 +1238,6 @@ inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {
#define GGML_F32x4_FMA(a, b, c) _mm_add_ps(_mm_mul_ps(b, c), a)
#endif
#define GGML_F32x4_ADD _mm_add_ps
-#define GGML_F32x4_SUB _mm_sub_ps
#define GGML_F32x4_MUL _mm_mul_ps
#define GGML_F32x4_REDUCE(res, x) \
{ \
@@ -1415,7 +1265,6 @@ inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {
#define GGML_F32_VEC_STORE GGML_F32x4_STORE
#define GGML_F32_VEC_FMA GGML_F32x4_FMA
#define GGML_F32_VEC_ADD GGML_F32x4_ADD
-#define GGML_F32_VEC_SUB GGML_F32x4_SUB
#define GGML_F32_VEC_MUL GGML_F32x4_MUL
#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE
@@ -1453,7 +1302,6 @@ static inline void __sse_f16x4_store(ggml_fp16_t *x, __m128 y) {
#define GGML_F32Cx4_STORE(x, y) __sse_f16x4_store(x, y)
#define GGML_F32Cx4_FMA GGML_F32x4_FMA
#define GGML_F32Cx4_ADD _mm_add_ps
-#define GGML_F32Cx4_SUB _mm_sub_ps
#define GGML_F32Cx4_MUL _mm_mul_ps
#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE
@@ -1464,7 +1312,6 @@ static inline void __sse_f16x4_store(ggml_fp16_t *x, __m128 y) {
#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i])
#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA
#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD
-#define GGML_F16_VEC_SUB GGML_F32Cx4_SUB
#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL
#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE
@@ -1477,423 +1324,6 @@ static inline void __sse_f16x4_store(ggml_fp16_t *x, __m128 y) {
#define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR)
#endif
-//
-// fundamental operations
-//
-
-inline static void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
-
-inline static void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
-
-inline static void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
-
-inline static void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
-
-inline static void ggml_vec_set_bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
-
-inline static void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] + y[i]; }
-inline static void ggml_vec_add1_f32(const int n, float * z, const float * x, const float v) { for (int i = 0; i < n; ++i) z[i] = x[i] + v; }
-inline static void ggml_vec_acc_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] += x[i]; }
-inline static void ggml_vec_acc1_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] += v; }
-inline static void ggml_vec_sub_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] - y[i]; }
-inline static void ggml_vec_set_f32 (const int n, float * x, const float v) { for (int i = 0; i < n; ++i) x[i] = v; }
-inline static void ggml_vec_cpy_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]; }
-inline static void ggml_vec_neg_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = -x[i]; }
-inline static void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]*y[i]; }
-inline static void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]/y[i]; }
-
-static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc) {
- assert(nrc == 1);
- UNUSED(nrc);
- UNUSED(bx);
- UNUSED(by);
- UNUSED(bs);
-
-#ifdef GGML_SIMD
- float sumf = 0.0f;
- const int np = (n & ~(GGML_F32_STEP - 1));
-
- GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };
-
- GGML_F32_VEC ax[GGML_F32_ARR];
- GGML_F32_VEC ay[GGML_F32_ARR];
-
- for (int i = 0; i < np; i += GGML_F32_STEP) {
- for (int j = 0; j < GGML_F32_ARR; j++) {
- ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);
- ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);
-
- sum[j] = GGML_F32_VEC_FMA(sum[j], ax[j], ay[j]);
- }
- }
-
- // reduce sum0..sum3 to sum0
- GGML_F32_VEC_REDUCE(sumf, sum);
-
- // leftovers
- for (int i = np; i < n; ++i) {
- sumf += x[i]*y[i];
- }
-#else
- // scalar
- ggml_float sumf = 0.0;
- for (int i = 0; i < n; ++i) {
- sumf += (ggml_float)(x[i]*y[i]);
- }
-#endif
-
- *s = sumf;
-}
-
-static void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc) {
- assert(nrc == 1);
- UNUSED(nrc);
- UNUSED(bx);
- UNUSED(by);
- UNUSED(bs);
- ggml_float sumf = 0;
- for (int i = 0; i < n; ++i) {
- sumf += GGML_BF16_TO_FP32(x[i]) * GGML_BF16_TO_FP32(y[i]);
- }
- *s = sumf;
-}
-
-#ifdef __x86_64__
-#pragma GCC push_options
-#pragma GCC target("avx")
-#pragma GCC target("f16c")
-#pragma GCC target("fma")
-static void ggml_vec_dot_f16_avx(const int n, float * restrict s, ggml_fp16_t * restrict x, ggml_fp16_t * restrict y) {
- ggml_float sumf = 0.0;
- int np = (n & ~(32 - 1));
- __m256 ax[(32/8)], ay[(32/8)], sum[(32/8)] = { _mm256_setzero_ps() };
- for (int i = 0; i < np; i += 32) {
- for (int j = 0; j < (32/8); j++) {
- ax[j] = _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(x + i + j*8)));
- ay[j] = _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(y + i + j*8)));
- sum[j] = _mm256_fmadd_ps(ax[j], ay[j], sum[j]);
- }
- }
- // reduce sum0..sum3 to sum0
- GGML_F32x8_REDUCE_AVX(sumf, sum);
- for (int i = np; i < n; ++i) {
- sumf += (ggml_float)(ggml_lookup_fp16_to_fp32(x[i]) *
- ggml_lookup_fp16_to_fp32(y[i]));
- }
- *s = sumf;
-}
-#pragma GCC pop_options
-#endif /* __x86_64__ */
-
-static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc) {
- assert(nrc == 1);
- UNUSED(nrc);
- UNUSED(bx);
- UNUSED(by);
- UNUSED(bs);
-
-#ifdef __x86_64__
- if (X86_HAVE(AVX) && X86_HAVE(F16C) && X86_HAVE(FMA)) {
- return ggml_vec_dot_f16_avx(n, s, x, y);
- }
-#endif
-
- ggml_float sumf = 0.0;
-
-#if defined(GGML_SIMD)
- const int np = (n & ~(GGML_F16_STEP - 1));
-
- GGML_F16_VEC sum[GGML_F16_ARR] = { GGML_F16_VEC_ZERO };
- GGML_F16_VEC err[GGML_F16_ARR] = { GGML_F16_VEC_ZERO };
-
- GGML_F16_VEC ax[GGML_F16_ARR];
- GGML_F16_VEC ay[GGML_F16_ARR];
-
- for (int i = 0; i < np; i += GGML_F16_STEP) {
- for (int j = 0; j < GGML_F16_ARR; j++) {
- ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);
- ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
-
- if (GGML_KAHAN) {
- GGML_F16_VEC ky = GGML_F16_VEC_SUB(GGML_F16_VEC_MUL(ax[j], ay[j]), err[j]);
- GGML_F16_VEC kt = GGML_F16_VEC_ADD(sum[j], ky);
- err[j] = GGML_F16_VEC_SUB(GGML_F16_VEC_SUB(kt, sum[j]), ky);
- sum[j] = kt;
- } else {
- sum[j] = GGML_F16_VEC_FMA(sum[j], ax[j], ay[j]);
- }
- }
- }
-
- // reduce sum0..sum3 to sum0
- GGML_F16_VEC_REDUCE(sumf, sum);
-
- // leftovers
- for (int i = np; i < n; ++i) {
- sumf += (ggml_float)(GGML_FP16_TO_FP32(x[i])*GGML_FP16_TO_FP32(y[i]));
- }
-#else
- for (int i = 0; i < n; ++i) {
- sumf += (ggml_float)(GGML_FP16_TO_FP32(x[i])*GGML_FP16_TO_FP32(y[i]));
- }
-#endif
-
- *s = sumf;
-}
-
-// compute GGML_VEC_DOT_UNROLL dot products at once
-// xs - x row stride in bytes
-inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * restrict s, void * restrict xv, ggml_fp16_t * restrict y) {
- ggml_float sumf[GGML_VEC_DOT_UNROLL] = { 0.0 };
-
- ggml_fp16_t * restrict x[GGML_VEC_DOT_UNROLL];
-
- for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {
- x[i] = (ggml_fp16_t *) ((char *) xv + i*xs);
- }
-
-#if defined(GGML_SIMD)
- const int np = (n & ~(GGML_F16_STEP - 1));
-
- GGML_F16_VEC sum[GGML_VEC_DOT_UNROLL][GGML_F16_ARR] = { { GGML_F16_VEC_ZERO } };
- GGML_F16_VEC err[GGML_VEC_DOT_UNROLL][GGML_F16_ARR] = { { GGML_F16_VEC_ZERO } };
-
- GGML_F16_VEC ax[GGML_F16_ARR];
- GGML_F16_VEC ay[GGML_F16_ARR];
-
- for (int i = 0; i < np; i += GGML_F16_STEP) {
- for (int j = 0; j < GGML_F16_ARR; j++) {
- ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
-
- for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {
- ax[j] = GGML_F16_VEC_LOAD(x[k] + i + j*GGML_F16_EPR, j);
-
- if (GGML_KAHAN) {
- GGML_F16_VEC ky = GGML_F16_VEC_SUB(GGML_F16_VEC_MUL(ax[j], ay[j]), err[k][j]);
- GGML_F16_VEC kt = GGML_F16_VEC_ADD(sum[k][j], ky);
- err[k][j] = GGML_F16_VEC_SUB(GGML_F16_VEC_SUB(kt, sum[k][j]), ky);
- sum[k][j] = kt;
- } else {
- sum[k][j] = GGML_F16_VEC_FMA(sum[k][j], ax[j], ay[j]);
- }
- }
- }
- }
-
- // reduce sum0..sum3 to sum0
- for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {
- GGML_F16_VEC_REDUCE(sumf[k], sum[k]);
- }
-
- // leftovers
- for (int i = np; i < n; ++i) {
- for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {
- sumf[j] += (ggml_float)(GGML_FP16_TO_FP32(x[j][i])*GGML_FP16_TO_FP32(y[i]));
- }
- }
-#else
- for (int i = 0; i < n; ++i) {
- for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {
- sumf[j] += (ggml_float)(GGML_FP16_TO_FP32(x[j][i])*GGML_FP16_TO_FP32(y[i]));
- }
- }
-#endif
-
- for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {
- s[i] = sumf[i];
- }
-}
-
-inline static void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v) {
-#if defined(GGML_SIMD)
- const int np = (n & ~(GGML_F32_STEP - 1));
-
- GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);
-
- GGML_F32_VEC ax[GGML_F32_ARR];
- GGML_F32_VEC ay[GGML_F32_ARR];
-
- for (int i = 0; i < np; i += GGML_F32_STEP) {
- for (int j = 0; j < GGML_F32_ARR; j++) {
- ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);
- ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);
- ay[j] = GGML_F32_VEC_FMA(ay[j], ax[j], vx);
-
- GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);
- }
- }
-
- // leftovers
- for (int i = np; i < n; ++i) {
- y[i] += x[i]*v;
- }
-#else
- // scalar
- for (int i = 0; i < n; ++i) {
- y[i] += x[i]*v;
- }
-#endif
-}
-
-inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v) {
-#if defined(GGML_SIMD)
- const int np = (n & ~(GGML_F16_STEP - 1));
-
- GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);
-
- GGML_F16_VEC ax[GGML_F16_ARR];
- GGML_F16_VEC ay[GGML_F16_ARR];
-
- for (int i = 0; i < np; i += GGML_F16_STEP) {
- for (int j = 0; j < GGML_F16_ARR; j++) {
- ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);
- ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
- ay[j] = GGML_F16_VEC_FMA(ay[j], ax[j], vx);
-
- GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);
- }
- }
-
- // leftovers
- for (int i = np; i < n; ++i) {
- y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i]) + GGML_FP16_TO_FP32(x[i])*v);
- }
-#else
- // scalar
- for (int i = 0; i < n; ++i) {
- y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i]) + GGML_FP16_TO_FP32(x[i])*v);
- }
-#endif
-}
-
-// xs and vs are byte strides of x and v
-inline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * restrict y, const float * restrict xv, const float * restrict vv) {
-
- const float * restrict x[GGML_VEC_MAD_UNROLL];
- const float * restrict v[GGML_VEC_MAD_UNROLL];
-
- for (int i = 0; i < GGML_VEC_MAD_UNROLL; ++i) {
- x[i] = (const float *) ((const char *) xv + i*xs);
- v[i] = (const float *) ((const char *) vv + i*vs);
- }
-
-#if defined(GGML_SIMD)
- const int np = (n & ~(GGML_F32_STEP - 1));
-
- GGML_F32_VEC vx[GGML_VEC_MAD_UNROLL];
-
- for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {
- vx[k] = GGML_F32_VEC_SET1(v[k][0]);
- }
-
- GGML_F32_VEC ax[GGML_VEC_MAD_UNROLL][GGML_F32_ARR];
- GGML_F32_VEC ay[GGML_F32_ARR];
-
- for (int i = 0; i < np; i += GGML_F32_STEP) {
- for (int j = 0; j < GGML_F32_ARR; j++) {
- ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);
-
- for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {
- ax[k][j] = GGML_F32_VEC_LOAD(x[k] + i + j*GGML_F32_EPR);
- ay[j] = GGML_F32_VEC_FMA(ay[j], ax[k][j], vx[k]);
- }
-
- GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);
- }
- }
-
- // leftovers
- for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {
- for (int i = np; i < n; ++i) {
- y[i] += x[k][i]*v[k][0];
- }
- }
-#else
- // scalar
- for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {
- for (int i = 0; i < n; ++i) {
- y[i] += x[k][i]*v[k][0];
- }
- }
-#endif
-}
-
-//inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] *= v; }
-inline static void ggml_vec_scale_f32(const int n, float * y, const float v) {
-#if defined(GGML_USE_ACCELERATE)
- vDSP_vsmul(y, 1, &v, y, 1, n);
-#elif defined(GGML_SIMD)
- const int np = (n & ~(GGML_F32_STEP - 1));
-
- GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);
-
- GGML_F32_VEC ay[GGML_F32_ARR];
-
- for (int i = 0; i < np; i += GGML_F32_STEP) {
- for (int j = 0; j < GGML_F32_ARR; j++) {
- ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);
- ay[j] = GGML_F32_VEC_MUL(ay[j], vx);
-
- GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);
- }
- }
-
- // leftovers
- for (int i = np; i < n; ++i) {
- y[i] *= v;
- }
-#else
- // scalar
- for (int i = 0; i < n; ++i) {
- y[i] *= v;
- }
-#endif
-}
-
-inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {
-#if defined(GGML_SIMD)
- const int np = (n & ~(GGML_F16_STEP - 1));
-
- GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);
-
- GGML_F16_VEC ay[GGML_F16_ARR];
-
- for (int i = 0; i < np; i += GGML_F16_STEP) {
- for (int j = 0; j < GGML_F16_ARR; j++) {
- ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);
- ay[j] = GGML_F16_VEC_MUL(ay[j], vx);
-
- GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);
- }
- }
-
- // leftovers
- for (int i = np; i < n; ++i) {
- y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i])*v);
- }
-#else
- // scalar
- for (int i = 0; i < n; ++i) {
- y[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(y[i])*v);
- }
-#endif
-}
-
-inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, 0, x, 0, x, 0, 1); *s = sqrtf(*s); }
-inline static void ggml_vec_sqr_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i]; }
-inline static void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrtf(x[i]); }
-inline static void ggml_vec_log_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = logf(x[i]); }
-inline static void ggml_vec_abs_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fabsf(x[i]); }
-inline static void ggml_vec_sgn_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : ((x[i] < 0.f) ? -1.f : 0.f); }
-inline static void ggml_vec_step_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : 0.f; }
-inline static void ggml_vec_tanh_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = tanhf(x[i]); }
-inline static void ggml_vec_elu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : expf(x[i])-1; }
-inline static void ggml_vec_relu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.f; }
-inline static void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); }
-// TODO: optimize performance
-inline static void ggml_vec_hardswish_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }
-inline static void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }
-
////////////////////////////////////////////////////////////////////////////////
// SYNCHRONIZATION PRIMITIVES
@@ -1911,162 +1341,6 @@ void ggml_once(atomic_uint * once, void init(void)) {
}
}
-// ACTIVATION FUNCTIONS
-//
-// 0.3 +--------------------------------------------------------------------------------------+
-// | + + + + + + + * % |
-// | x>0?x:0.01 **%**** |
-// | x/(1+exp(-x)) #%##### |
-// | .5*x*(1+erf(x/sqrt(2)))*$%$$$$$ |
-// 0.2 |-+ x*(1/(1+exp(-1.702*x)))*%%%%%%%-|
-// | *%# |
-// | *% |
-// | *% |
-// | *% |
-// 0.1 |-+ *% +-|
-// | *% |
-// | % |
-// | *% |
-// | % |
-// 0 |***************************************************************************% +-|
-// |%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%$$$$$ % |
-// | ########## %%%%%$$$$ % |
-// | ##### %%%%$$ % |
-// | #### %%$$ % |
-// | #### %%%$ % |
-// -0.1 |-+ ### %%% %# +-|
-// | ### % %# |
-// | ### %%% %%# |
-// | ## $%%%%$ # |
-// | ### # |
-// -0.2 |-+ ## # +-|
-// | ### # |
-// | ## # |
-// | #### ### |
-// | + + + + + #####+ + |
-// -0.3 +--------------------------------------------------------------------------------------+
-// -7 -6 -5 -4 -3 -2 -1 0 1
-//
-
-inline static float ggml_silu_f32(float x) {
- // SiLU is the favored by LLaMA, Mistral, Phi, Rocket, etc.
- return x/(1.f + expf(-x));
-}
-
-inline static float ggml_gelu_f32(float x) {
- // GeLU approximation that goes slower and we seem to be stuck with.
- return .5f * x * (1.f + tanhf(sqrtf(M_2_PI) * (x + .044715f * x * x * x)));
-}
-
-inline static float ggml_gelu_quick_f32(float x) {
- // "Quick" GeLU is used by LLaVA's CLIP vision model.
- return x*(1.f/(1.f+expf(-1.702f*x)));
-}
-
-static void ggml_vec_gelu_f16_init(void) {
- ggml_table_gelu_f16 = malloc(sizeof(float) * 65536);
- for (int i = 0; i < 65536; ++i) {
- union {
- unsigned short i;
- ggml_fp16_t f;
- } u = {i};
- ggml_table_gelu_f16[i] = ggml_gelu_f32(u.f);
- }
-}
-
-inline static void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
- static atomic_uint once;
- ggml_once(&once, ggml_vec_gelu_f16_init);
- const uint16_t * i16 = (const uint16_t *) x;
- for (int i = 0; i < n; ++i) {
- y[i] = ggml_table_gelu_f16[i16[i]];
- }
-}
-
-inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {
- for (int i = 0; i < n; ++i) {
- y[i] = ggml_gelu_f32(x[i]);
- }
-}
-
-inline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {
- for (int i = 0; i < n; ++i) {
- y[i] = ggml_gelu_quick_f32(x[i]);
- }
-}
-
-inline static void ggml_vec_silu_f32(const int n, float * y, const float * x) {
- for (int i = 0; i < n; ++i) {
- y[i] = ggml_silu_f32(x[i]);
- }
-}
-
-inline static float ggml_silu_backward_f32(float x, float dy) {
- const float s = 1.0f/(1.0f + expf(-x));
- return dy*s*(1.0f + x*(1.0f - s));
-}
-
-inline static void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, const float * dy) {
- for (int i = 0; i < n; ++i) {
- dx[i] = ggml_silu_backward_f32(x[i], dy[i]);
- }
-}
-
-inline static void ggml_vec_sum_f32(const int n, float * s, const float * x) {
-#ifndef GGML_USE_ACCELERATE
- ggml_float sum = 0.0;
- for (int i = 0; i < n; ++i) {
- sum += (ggml_float)x[i];
- }
- *s = sum;
-#else
- vDSP_sve(x, 1, s, n);
-#endif
-}
-
-inline static void ggml_vec_sum_f32_ggf(const int n, ggml_float * s, const float * x) {
- ggml_float sum = 0.0;
- for (int i = 0; i < n; ++i) {
- sum += (ggml_float)x[i];
- }
- *s = sum;
-}
-
-inline static void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x) {
- float sum = 0.0f;
- for (int i = 0; i < n; ++i) {
- sum += GGML_FP16_TO_FP32(x[i]);
- }
- *s = sum;
-}
-
-inline static void ggml_vec_max_f32(const int n, float * s, const float * x) {
-#ifndef GGML_USE_ACCELERATE
- float max = -INFINITY;
- for (int i = 0; i < n; ++i) {
- max = MAX(max, x[i]);
- }
- *s = max;
-#else
- vDSP_maxv(x, 1, s, n);
-#endif
-}
-
-inline static void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) {
- ggml_vec_norm_f32(n, s, x);
- *s = 1.f/(*s);
-}
-
-inline static void ggml_vec_argmax_f32(const int n, int * s, const float * x) {
- float max = -INFINITY;
- int idx = 0;
- for (int i = 0; i < n; ++i) {
- max = MAX(max, x[i]);
- if (max == x[i]) { idx = i; }
- }
- *s = idx;
-}
-
//
// data types
//
@@ -11184,6 +10458,7 @@ UseGgmlGemm2:;
}
#if 0
+ if (ne11 > 1)
fprintf(stderr, "MULMAT %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) * %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) -> %s[%8ld,%8ld,%8ld](%8zu,%8zu,%8zu) vdt=%s src1_cont=%d\n",
ggml_type_name(src0->type), ne00, ne01, ne02, nb00, nb01, nb02,
ggml_type_name(src1->type), ne10, ne11, ne12, nb10, nb11, nb12,
@@ -16908,6 +16183,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
GGML_ASSERT(false);
} break;
}
+
+#ifdef LLAMAFILE_DEBUG
+ llamafile_trapping_restore();
+#endif
}
////////////////////////////////////////////////////////////////////////////////
@@ -18855,6 +18134,72 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
else {
ggml_graph_compute_thread_sync_task(&task_phase, state, false);
}
+
+#if 1
+ // expensive integrity check
+ // not useful to test mt-safety
+ if (!state->ith) {
+ int64_t nans = 0;
+ int64_t infs = 0;
+ if (node->type == GGML_TYPE_F32)
+ for (int i3 = 0; i3 < node->ne[3]; ++i3)
+ for (int i2 = 0; i2 < node->ne[2]; ++i2)
+ for (int i1 = 0; i1 < node->ne[1]; ++i1)
+ for (int i0 = 0; i0 < node->ne[0]; ++i0) {
+ float x = *(const float *)((const char *)node->data +
+ i3 * node->nb[3] +
+ i2 * node->nb[2] +
+ i1 * node->nb[1] +
+ i0 * node->nb[0]);
+ if (isnan(x))
+ ++nans;
+ if (isinf(x))
+ ++infs;
+ }
+ else if (node->type == GGML_TYPE_F16)
+ for (int i3 = 0; i3 < node->ne[3]; ++i3)
+ for (int i2 = 0; i2 < node->ne[2]; ++i2)
+ for (int i1 = 0; i1 < node->ne[1]; ++i1)
+ for (int i0 = 0; i0 < node->ne[0]; ++i0) {
+ float x = GGML_FP16_TO_FP32(*(const ggml_fp16_t *)((const char *)node->data +
+ i3 * node->nb[3] +
+ i2 * node->nb[2] +
+ i1 * node->nb[1] +
+ i0 * node->nb[0]));
+ if (isnan(x))
+ ++nans;
+ if (isinf(x))
+ ++infs;
+ }
+ else if (node->type == GGML_TYPE_BF16)
+ for (int i3 = 0; i3 < node->ne[3]; ++i3)
+ for (int i2 = 0; i2 < node->ne[2]; ++i2)
+ for (int i1 = 0; i1 < node->ne[1]; ++i1)
+ for (int i0 = 0; i0 < node->ne[0]; ++i0) {
+ float x = GGML_BF16_TO_FP32(*(const ggml_bf16_t *)((const char *)node->data +
+ i3 * node->nb[3] +
+ i2 * node->nb[2] +
+ i1 * node->nb[1] +
+ i0 * node->nb[0]));
+ if (isnan(x))
+ ++nans;
+ if (isinf(x))
+ ++infs;
+ }
+ else
+ continue;
+ if (nans || infs) {
+ flockfile(stderr);
+ fprintf(stderr, "ERROR: node #%d produced %" PRId64 " NaNs and %" PRId64 " infinities in sequence %s -> %s\n",
+ node_n, nans, infs, node_n ? ggml_op_name(cgraph->nodes[node_n - 1]->op) : "n/a",
+ ggml_op_name(node->op));
+ for (int i = 0; i < GGML_MAX_SRC && node->src[i]; ++i)
+ fprintf(stderr, "\t- src[%d] is %s\n", i, ggml_op_name(node->src[i]->op));
+ exit(1);
+ }
+ }
+#endif
+
}
return 0;
diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.h
index b45c29ba..478865c5 100644
--- a/llama.cpp/ggml.h
+++ b/llama.cpp/ggml.h
@@ -328,14 +328,20 @@ extern "C" {
// get ggml_status name string
GGML_API GGML_CALL const char * ggml_status_to_string(enum ggml_status status);
+ // ieee 754-2008 half-precision float16
+ // todo: make this not an integral type
typedef uint16_t ggml_fp16_t;
-
- // convert FP16 <-> FP32
- GGML_API float ggml_fp16_to_fp32(ggml_fp16_t x);
- GGML_API ggml_fp16_t ggml_fp32_to_fp16(float x);
-
- GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n);
- GGML_API void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n);
+ GGML_API float ggml_fp16_to_fp32(ggml_fp16_t);
+ GGML_API ggml_fp16_t ggml_fp32_to_fp16(float);
+ GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t);
+ GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t);
+
+ // google brain half-precision bfloat16
+ typedef struct { uint16_t bits; } ggml_bf16_t;
+ GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);
+ GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16
+ GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);
+ GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);
struct ggml_object;
struct ggml_context;
@@ -2394,90 +2400,6 @@ extern "C" {
GGML_API int ggml_cpu_has_vsx (void);
GGML_API int ggml_cpu_has_matmul_int8(void);
- /**
- * Google Brain 16-bit floating point number.
- *
- * ┌sign
- * │
- * │ ┌exponent
- * │ │
- * │ │ ┌mantissa
- * │ │ │
- * │┌──┴───┐┌─┴───┐
- * 0b0000000000000000 bfloat16
- *
- * Since bf16 has the same number of exponent bits as a 32bit float,
- * encoding and decoding numbers becomes relatively straightforward.
- *
- * ┌sign
- * │
- * │ ┌exponent
- * │ │
- * │ │ ┌mantissa
- * │ │ │
- * │┌──┴───┐┌─┴───────────────────┐
- * 0b00000000000000000000000000000000 IEEE binary32
- *
- * For comparison, the standard fp16 format has fewer exponent bits.
- *
- * ┌sign
- * │
- * │ ┌exponent
- * │ │
- * │ │ ┌mantissa
- * │ │ │
- * │┌─┴─┐┌─┴──────┐
- * 0b0000000000000000 IEEE binary16
- *
- * So be warned that converting between them yields a ten bit float.
- *
- * @see IEEE 754-2008
- */
- typedef struct {
- uint16_t x;
- } ggml_bf16_t;
-
- /**
- * Converts bfloat16 to float32.
- */
- static inline float ggml_bf16_to_fp32(ggml_bf16_t h) {
- union {
- float f;
- uint32_t i;
- } u;
- u.i = (uint32_t)h.x << 16;
- return u.f;
- }
-
- /**
- * Converts float32 to bfloat16.
- *
- * This function is binary identical to AMD Zen4 VCVTNEPS2BF16.
- * Subnormals shall be flushed to zero, and NANs will be quiet.
- * This code should vectorize nicely if using modern compilers.
- */
- static inline ggml_bf16_t ggml_fp32_to_bf16(float s) {
- ggml_bf16_t h;
- union {
- float f;
- uint32_t i;
- } u;
- u.f = s;
- if ((u.i & 0x7fffffff) > 0x7f800000) { /* nan */
- h.x = (u.i >> 16) | 64; /* force to quiet */
- return h;
- }
- if (!(u.i & 0x7f800000)) { /* subnormal */
- h.x = (u.i & 0x80000000) >> 16; /* flush to zero */
- return h;
- }
- h.x = (u.i + (0x7fff + ((u.i >> 16) & 1))) >> 16;
- return h;
- }
-
- GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n);
- GGML_API void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n);
-
//
// Internal types and functions exposed for tests and benchmarks
//
diff --git a/llama.cpp/llava/clip.cpp b/llama.cpp/llava/clip.cpp
index ec8d3654..abf8872e 100644
--- a/llama.cpp/llava/clip.cpp
+++ b/llama.cpp/llava/clip.cpp
@@ -908,7 +908,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
LOG_TEE("%s: n_tensors: %d\n", __func__, n_tensors);
LOG_TEE("%s: n_kv: %d\n", __func__, n_kv);
LOG_TEE("%s: ftype: %s\n", __func__, ftype_str.c_str());
- LOG_TEE("\n");
+ // LOG_TEE("\n"); // [jart] do not want
}
const int n_tensors = gguf_get_n_tensors(ctx);
@@ -1157,7 +1157,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
for (int i = 0; i < 32 && (hparams.image_grid_pinpoints[i] != 0); ++i) {
LOG_TEE("%d ", hparams.image_grid_pinpoints[i]);
}
- LOG_TEE("\n");
+ // LOG_TEE("\n"); // [jart] do not want
LOG_TEE("v_mm_patch_merge_type: %s\n", hparams.mm_patch_merge_type);
}
diff --git a/llama.cpp/llava/llava.cpp b/llama.cpp/llava/llava.cpp
index bdd50471..280a7ec7 100644
--- a/llama.cpp/llava/llava.cpp
+++ b/llama.cpp/llava/llava.cpp
@@ -297,7 +297,8 @@ static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const cli
const int64_t t_img_enc_end_us = ggml_time_us();
float t_img_enc_ms = (t_img_enc_end_us - t_img_enc_start_us) / 1000.0;
- LOG_TEE("\n%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / *n_img_pos);
+ // [jart] get rid of sneak newline
+ LOG_TEE("%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / *n_img_pos);
return true;
}
diff --git a/llama.cpp/log.h b/llama.cpp/log.h
index 3c2d077c..db1ab355 100644
--- a/llama.cpp/log.h
+++ b/llama.cpp/log.h
@@ -1,4 +1,6 @@
#pragma once
+#define LOG_NO_TIMESTAMPS
+#define LOG_NO_FILE_LINE_FUNCTION
#include <chrono>
#include <cstring>
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index e0b5ceec..989295d8 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -89,13 +89,20 @@ o/$(MODE)/llamafile: \
#
o/$(MODE)/llamafile/sgemm.o: private CXXFLAGS += -Os
-o/$(MODE)/llamafile/tinyblas_cpu_amd_avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge -Xx86_64-mf16c
-o/$(MODE)/llamafile/tinyblas_cpu_amd_fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mf16c -Xx86_64-mfma
-o/$(MODE)/llamafile/tinyblas_cpu_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mfma
-o/$(MODE)/llamafile/tinyblas_cpu_amd_avxvnni.o: private TARGET_ARCH += -Xx86_64-mtune=alderlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavxvnni
-o/$(MODE)/llamafile/tinyblas_cpu_amd_avx512f.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f
-o/$(MODE)/llamafile/tinyblas_cpu_amd_zen4.o: private TARGET_ARCH += -Xx86_64-mtune=znver4 -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512vl -Xx86_64-mavx512vnni -Xx86_64-mavx512bf16
-o/$(MODE)/llamafile/tinyblas_cpu_arm82.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+dotprod+fp16
+o/$(MODE)/llamafile/tinyblas_cpu_sgemm_amd_avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge -Xx86_64-mf16c
+o/$(MODE)/llamafile/tinyblas_cpu_mixmul_amd_avx.o: private TARGET_ARCH += -Xx86_64-mtune=sandybridge -Xx86_64-mf16c
+o/$(MODE)/llamafile/tinyblas_cpu_sgemm_amd_fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mf16c -Xx86_64-mfma
+o/$(MODE)/llamafile/tinyblas_cpu_mixmul_amd_fma.o: private TARGET_ARCH += -Xx86_64-mtune=bdver2 -Xx86_64-mf16c -Xx86_64-mfma
+o/$(MODE)/llamafile/tinyblas_cpu_sgemm_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mfma
+o/$(MODE)/llamafile/tinyblas_cpu_mixmul_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mtune=skylake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mfma
+o/$(MODE)/llamafile/tinyblas_cpu_sgemm_amd_avxvnni.o: private TARGET_ARCH += -Xx86_64-mtune=alderlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavxvnni
+o/$(MODE)/llamafile/tinyblas_cpu_mixmul_amd_avxvnni.o: private TARGET_ARCH += -Xx86_64-mtune=alderlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavxvnni
+o/$(MODE)/llamafile/tinyblas_cpu_sgemm_amd_avx512f.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f
+o/$(MODE)/llamafile/tinyblas_cpu_mixmul_amd_avx512f.o: private TARGET_ARCH += -Xx86_64-mtune=cannonlake -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f
+o/$(MODE)/llamafile/tinyblas_cpu_sgemm_amd_zen4.o: private TARGET_ARCH += -Xx86_64-mtune=znver4 -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512vl -Xx86_64-mavx512vnni -Xx86_64-mavx512bf16
+o/$(MODE)/llamafile/tinyblas_cpu_mixmul_amd_zen4.o: private TARGET_ARCH += -Xx86_64-mtune=znver4 -Xx86_64-mf16c -Xx86_64-mfma -Xx86_64-mavx2 -Xx86_64-mavx512f -Xx86_64-mavx512vl -Xx86_64-mavx512vnni -Xx86_64-mavx512bf16
+o/$(MODE)/llamafile/tinyblas_cpu_sgemm_arm82.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+dotprod+fp16
+o/$(MODE)/llamafile/tinyblas_cpu_mixmul_arm82.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+dotprod+fp16
################################################################################
# testing
diff --git a/llamafile/debug.cpp b/llamafile/debug.cpp
index e8c9732a..15512354 100644
--- a/llamafile/debug.cpp
+++ b/llamafile/debug.cpp
@@ -16,9 +16,11 @@
// limitations under the License.
#include "llamafile.h"
+#include "log.h"
#include <cosmo.h>
#include <fenv.h>
+#include <libc/calls/struct/aarch64.internal.h>
#include <libc/calls/struct/ucontext.internal.h>
#include <signal.h>
#include <stdatomic.h>
@@ -37,6 +39,7 @@ static int get_config(void) {
}
bool FLAG_trap;
+static thread_local int g_enabled;
thread_local int llamafile_debug_op_index;
const struct ggml_cgraph *llamafile_debug_graph;
@@ -105,45 +108,89 @@ static inline void spinlock(atomic_uint *lock) {
}
}
+static inline void spunlock(atomic_uint *lock) {
+ atomic_store_explicit(lock, 0, memory_order_release);
+}
+
static void on_sigfpe(int sig, siginfo_t *si, void *arg) {
- ucontext_t *ctx = (ucontext_t *)arg;
+
static atomic_uint lock;
- spinlock(&lock); // first thread to crash wins
- g_terminal_buddy.restore();
- const char *s = "\nerror: ";
+ spinlock(&lock);
+
+ const char *issue;
+ ucontext_t *ctx = (ucontext_t *)arg;
if (si->si_code == FPE_INTDIV)
- tinyprint(2, s, "integer divide by zero\n", NULL);
+ issue = "integer divide by zero";
else if (si->si_code == FPE_INTOVF)
- tinyprint(2, s, "integer overflow\n", NULL);
+ issue = "integer overflow";
else if (si->si_code == FPE_FLTDIV)
- tinyprint(2, s, "floating point divide by zero\n", NULL);
+ issue = "floating point divide by zero";
else if (si->si_code == FPE_FLTOVF)
- tinyprint(2, s, "floating point overflow\n", NULL);
+ issue = "floating point overflow";
else if (si->si_code == FPE_FLTUND)
- tinyprint(2, s, "floating point underflow\n", NULL);
+ issue = "floating point underflow";
else if (si->si_code == FPE_FLTRES)
- tinyprint(2, s, "floating point inexact\n", NULL);
+ issue = "floating point inexact";
else if (si->si_code == FPE_FLTINV)
- tinyprint(2, s, "invalid floating point operation\n", NULL);
+ issue = "invalid floating point operation";
else if (si->si_code == FPE_FLTSUB)
- tinyprint(2, s, "subscript out of range\n", NULL);
+ issue = "subscript out of range";
else
- tinyprint(2, s, "caught sigfpe\n", NULL);
- struct StackFrame sf = {.next = (struct StackFrame *)ctx->uc_mcontext.BP,
- .addr = ctx->uc_mcontext.PC};
- ShowBacktrace(2, &sf);
-#ifdef LLAMAFILE_DEBUG
+ issue = "sigfpe";
+
+ // show detailed information
+ static bool once;
+ char location[21];
+ const char *path = "/tmp/cgraph.txt";
+ int idx = llamafile_debug_op_index;
const struct ggml_cgraph *g;
if ((g = llamafile_debug_graph)) {
- int idx = llamafile_debug_op_index;
- const char *path = "/tmp/cgraph.txt";
- fprintf(stderr, "while executing %s at index #%d of %s\n", //
- ggml_op_name(g->nodes[idx]->op), idx, path);
- FILE *f = fopen(path, "w");
- print_graph(f, g);
- fclose(f);
+ FormatInt32(location, idx);
+ tinyprint(2, "error: ", issue, " at ", ggml_op_name(g->nodes[idx]->op), " operation in ",
+ path, " at index #", location, "\n", NULL);
+ } else {
+ FormatHex64(location, ctx->uc_mcontext.PC, 2);
+ tinyprint(2, "error: ", issue, " at pc ", location, "\n", NULL);
+ }
+ if (!once) {
+ struct StackFrame sf = {.next = (struct StackFrame *)ctx->uc_mcontext.BP,
+ .addr = ctx->uc_mcontext.PC};
+ ShowBacktrace(2, &sf);
+ const struct ggml_cgraph *g;
+ if ((g = llamafile_debug_graph)) {
+ FILE *f = fopen(path, "w");
+ print_graph(f, g);
+ fclose(f);
+ }
+ once = true;
+ }
+
+ // recover from trap so that execution may resume
+ // without this the same signal will just keep getting raised
+ int traps = get_config();
+#ifdef __x86_64__
+ if (ctx->uc_mcontext.fpregs) {
+ ctx->uc_mcontext.fpregs->mxcsr |= traps << 7; // disable traps
+ ctx->uc_mcontext.fpregs->mxcsr &= ~traps; // clear cages
+ spunlock(&lock);
+ return;
+ }
+#elif defined(__aarch64__)
+ struct _aarch64_ctx *ac;
+ for (ac = (struct _aarch64_ctx *)ctx->uc_mcontext.__reserved; ac->magic;
+ ac = (struct _aarch64_ctx *)((char *)ac + ac->size)) {
+ if (ac->magic == FPSIMD_MAGIC) {
+ struct fpsimd_context *sm = (struct fpsimd_context *)ac;
+ sm->fpcr &= ~(traps << 8); // disable traps
+ sm->fpsr &= ~traps; // clear cages
+ spunlock(&lock);
+ return;
+ }
}
#endif
+
+ // time to die
+ g_terminal_buddy.restore();
_exit(128 + SIGFPE);
}
@@ -160,7 +207,6 @@ static void setup_sigfpe(void) {
int llamafile_trapping_enabled(int delta) {
static atomic_uint once;
- static thread_local int g_enabled;
bool was_enabled = g_enabled > 0;
bool is_enabled = (g_enabled += delta) > 0;
int config = get_config();
@@ -174,3 +220,10 @@ int llamafile_trapping_enabled(int delta) {
}
return g_enabled;
}
+
+void llamafile_trapping_restore(void) {
+ if (g_enabled > 0) {
+ feclearexcept(FE_ALL_EXCEPT);
+ feenableexcept(get_config());
+ }
+}
diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
index 0503676c..ddf056ce 100644
--- a/llamafile/llamafile.h
+++ b/llamafile/llamafile.h
@@ -69,6 +69,7 @@ struct ggml_cgraph;
int feenableexcept(int);
int fedisableexcept(int);
int llamafile_trapping_enabled(int);
+void llamafile_trapping_restore(void);
void ShowBacktrace(int, const struct StackFrame *);
extern const struct ggml_cgraph *llamafile_debug_graph;
extern thread_local int llamafile_debug_op_index;
diff --git a/llamafile/tinyblas_cpu.h b/llamafile/tinyblas_cpu.h
index 850ac507..e4b4ec8c 100644
--- a/llamafile/tinyblas_cpu.h
+++ b/llamafile/tinyblas_cpu.h
@@ -43,6 +43,7 @@
#include "llama.cpp/ggml-impl.h"
#include "llama.cpp/ggml-quants.h"
#include "llamafile.h"
+#include "log.h"
#include "sgemm.h"
#include <cosmo.h>
@@ -65,8 +66,20 @@
#define VECTOR_REGISTERS 16
#endif
+#if 0
+#define NOT_SUPPORTED tinyBLAS_not_supported(__FILE__, __LINE__)
+#else
+#define NOT_SUPPORTED false
+#endif
+#define WANT_QUANTIZATION false
+
namespace {
+bool tinyBLAS_not_supported(const char *file, int line) {
+ tinylogf("%s:%d: tinyBLAS not supported\n", file, line);
+ return false;
+}
+
inline float unhalf(ggml_fp16_t d) {
return GGML_FP16_TO_FP32(d);
}
diff --git a/llamafile/tinyblas_cpu_mixmul.inc b/llamafile/tinyblas_cpu_mixmul.inc
index bd35e4e1..87d4ba57 100644
--- a/llamafile/tinyblas_cpu_mixmul.inc
+++ b/llamafile/tinyblas_cpu_mixmul.inc
@@ -148,6 +148,8 @@ class MixMul {
switch (result->type) {
case GGML_TYPE_F32:
return mixmuler<float>();
+ case GGML_TYPE_BF16:
+ return mixmuler<ggml_bf16_t>();
case GGML_TYPE_F16:
return mixmuler<ggml_fp16_t>();
default:
diff --git a/llamafile/tinyblas_cpu_sgemm.inc b/llamafile/tinyblas_cpu_sgemm.inc
index 27448e7e..84520fd1 100644
--- a/llamafile/tinyblas_cpu_sgemm.inc
+++ b/llamafile/tinyblas_cpu_sgemm.inc
@@ -51,91 +51,95 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
case GGML_TYPE_F32: {
if (Btype != GGML_TYPE_F32)
- return false;
+ return NOT_SUPPORTED;
#if defined(__AVX512F__)
if (k % 16)
- return false;
+ return NOT_SUPPORTED;
tinyBLAS<0, 16, __m512, __m512, float, float, TC> tb{
k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__AVX__) || defined(__AVX2__)
if (k % 8)
- return false;
+ return NOT_SUPPORTED;
tinyBLAS<0, 8, __m256, __m256, float, float, TC> tb{
k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__ARM_NEON)
if (k % 4)
- return false;
+ return NOT_SUPPORTED;
tinyBLAS<0, 4, float32x4_t, float32x4_t, float, float, TC> tb{
k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#else
- return false;
+ return NOT_SUPPORTED;
#endif
}
case GGML_TYPE_BF16: {
#if defined(__AVX512BF16__)
if (k % 32)
- return false;
+ return NOT_SUPPORTED;
if (Btype == GGML_TYPE_F32 && n < 2) {
tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, float, TC> tb{
k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
}
+ if (Btype == GGML_TYPE_F32)
+ return WANT_QUANTIZATION;
if (Btype != GGML_TYPE_BF16)
- return false;
+ return NOT_SUPPORTED;
tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, TC> tb{
k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__AVX512F__)
if (k % 16)
- return false;
+ return NOT_SUPPORTED;
tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, TC> tb{
k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__AVX2__)
if (k % 8)
- return false;
+ return NOT_SUPPORTED;
if (Btype != GGML_TYPE_F32)
- return false;
+ return NOT_SUPPORTED;
tinyBLAS<0, 8, __m256, __m256, ggml_bf16_t, float, TC> tb{
k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__ARM_NEON) && !defined(_MSC_VER)
if (k % 4)
- return false;
+ return NOT_SUPPORTED;
if (Btype != GGML_TYPE_F32)
- return false;
+ return NOT_SUPPORTED;
tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_bf16_t, float, TC> tb{
k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#else
- return false;
+ return NOT_SUPPORTED;
#endif
}
case GGML_TYPE_F16: {
#if defined(__AVX512F__)
if (k % 16)
- return false;
+ return NOT_SUPPORTED;
if (Btype == GGML_TYPE_F32 && n < 2) {
tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, float, TC> tb{
k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
}
+ if (Btype == GGML_TYPE_F32)
+ return WANT_QUANTIZATION;
if (Btype != GGML_TYPE_F16)
- return false;
+ return NOT_SUPPORTED;
tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, TC> tb{
k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
@@ -143,40 +147,44 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)
if (X86_HAVE(F16C)) {
if (k % 8)
- return false;
+ return NOT_SUPPORTED;
if (Btype == GGML_TYPE_F32 && n < 2) {
tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, TC> tb{
k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
}
+ if (Btype == GGML_TYPE_F32)
+ return WANT_QUANTIZATION;
if (Btype != GGML_TYPE_F16)
- return false;
+ return NOT_SUPPORTED;
tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, TC> tb{
k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
} else {
- return false;
+ return NOT_SUPPORTED;
}
#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
if (n < 2)
// TODO(jart): Why is ggml_vec_dot_f16_unroll() so fast at matvec?
- return false;
+ return NOT_SUPPORTED;
if (precision == GGML_PREC_F32) {
if (k % 4)
- return false;
+ return NOT_SUPPORTED;
if (Btype != GGML_TYPE_F32)
- return false;
+ return NOT_SUPPORTED;
tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, TC> tb{
k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
} else {
if (k % 8)
- return false;
+ return NOT_SUPPORTED;
+ if (Btype == GGML_TYPE_F32)
+ return WANT_QUANTIZATION;
if (Btype != GGML_TYPE_F16)
- return false;
+ return NOT_SUPPORTED;
tinyBLAS<0, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, TC> tb{
k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
@@ -185,23 +193,25 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
#elif defined(__ARM_NEON) && !defined(_MSC_VER)
if (n < 2)
// TODO(jart): Why is ggml_vec_dot_f16_unroll() so fast at matvec?
- return false;
+ return NOT_SUPPORTED;
if (k % 4)
- return false;
+ return NOT_SUPPORTED;
if (Btype != GGML_TYPE_F32)
- return false;
+ return NOT_SUPPORTED;
tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, TC> tb{
k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#else
- return false;
+ return NOT_SUPPORTED;
#endif
}
case GGML_TYPE_Q8_0: {
+ if (Btype == GGML_TYPE_F32)
+ return WANT_QUANTIZATION;
if (Btype != GGML_TYPE_Q8_0)
- return false;
+ return NOT_SUPPORTED;
#if defined(__AVX2__) || defined(__AVX512F__)
tinyBLAS_Q0_AVX2<0, block_q8_0, block_q8_0, TC> tb{
k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth};
@@ -213,13 +223,15 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
tb.matmul(m, n, task);
return true;
#else
- return false;
+ return NOT_SUPPORTED;
#endif
}
case GGML_TYPE_Q4_0: {
+ if (Btype == GGML_TYPE_F32)
+ return WANT_QUANTIZATION;
if (Btype != GGML_TYPE_Q8_0)
- return false;
+ return NOT_SUPPORTED;
#if defined(__AVX2__) || defined(__AVX512F__)
tinyBLAS_Q0_AVX2<0, block_q4_0, block_q8_0, TC> tb{
k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth};
@@ -231,12 +243,12 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
tb.matmul(m, n, task);
return true;
#else
- return false;
+ return NOT_SUPPORTED;
#endif
}
default:
- return false;
+ return NOT_SUPPORTED;
}
(void)m;
@@ -305,10 +317,13 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
case GGML_TYPE_F32:
return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (float *)C, ldc, ith, nth, task, Atype,
Btype, precision);
+ case GGML_TYPE_BF16:
+ return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (ggml_bf16_t *)C, ldc, ith, nth, task,
+ Atype, Btype, precision);
case GGML_TYPE_F16:
return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (ggml_fp16_t *)C, ldc, ith, nth, task,
Atype, Btype, precision);
default:
- return false;
+ return NOT_SUPPORTED;
}
}