LLM inference in C/C++
C++ ggml
ggml-cpu: add F16 input to the FWHT (#27779)
bri-prism4fea119de30f
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.cindex 87a329f269..8bb0ff7bc3 100644--- a/ggml/src/ggml-cpu/ggml-cpu.c+++ b/ggml/src/ggml-cpu/ggml-cpu.c@@ -1329,7 +1329,9 @@ UseGgmlGemm1:;const size_t nbw3 = nbw2*ne12;assert(params->wsize >= ne13*nbw3);- GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT(src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16);+ // the F16 path below writes plain floats into wdata, so it needs an F32 vec_dot_type+ GGML_ASSERT(src1->type == GGML_TYPE_F32 || vec_dot_type == GGML_TYPE_F32);#if 0for (int64_t i13 = 0; i13 < ne13; ++i13) {@@ -1348,9 +1350,15 @@ UseGgmlGemm1:;size_t bs = ggml_blck_size(vec_dot_type);int64_t ne10_block_start = (ith * ne10/bs) / nth;int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth;- from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10),- (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0),- (ne10_block_end - ne10_block_start) * bs);+ const char * src1_block = (const char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10;+ char * dst_block = wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0;+ const int64_t n_block = (ne10_block_end - ne10_block_start) * bs;++ if (src1->type == GGML_TYPE_F32) {+ from_float((const float *) src1_block, dst_block, n_block);+ } else {+ ggml_cpu_fp16_to_fp32((const ggml_fp16_t *) src1_block, (float *) dst_block, n_block);+ }}}}diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppindex 8cece71f18..1df0f2bb92 100644--- a/ggml/src/ggml-cpu/ggml-cpu.cpp+++ b/ggml/src/ggml-cpu/ggml-cpu.cpp@@ -451,6 +451,10 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const stop->type != GGML_TYPE_IQ1_S &&op->type != GGML_TYPE_IQ1_M; // missing type_traits.from_floatcase GGML_OP_MUL_MAT:+ if (ggml_get_op_params_i32(op, 1) == GGML_HINT_SRC0_IS_HADAMARD &&+ src0->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) {+ return src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16;+ }return src1->type == GGML_TYPE_F32 || src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type;case GGML_OP_SOFT_MAX_BACK: {if (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32) {diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cppindex 23001254c1..ba00a0a73e 100644--- a/ggml/src/ggml-cpu/ops.cpp+++ b/ggml/src/ggml-cpu/ops.cpp@@ -12015,11 +12015,20 @@ void ggml_compute_forward_opt_step_sgd(const ggml_compute_params * params, ggml_}}-static void ggml_compute_forward_fwht_f32(const ggml_compute_params * params, ggml_tensor * dst) {+static inline float ggml_fwht_load(const float value) {+ return value;+}++static inline float ggml_fwht_load(const ggml_fp16_t value) {+ return ggml_fp16_to_fp32(value);+}++template<typename src_t>+static void ggml_compute_forward_fwht_impl(const ggml_compute_params * params, ggml_tensor * dst) {const ggml_tensor * src0 = dst->src[0];const ggml_tensor * src1 = dst->src[1];- GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT(src1->type == (std::is_same_v<src_t, float> ? GGML_TYPE_F32 : GGML_TYPE_F16));GGML_ASSERT(dst->type == GGML_TYPE_F32);GGML_TENSOR_BINARY_OP_LOCALS@@ -12046,11 +12055,11 @@ static void ggml_compute_forward_fwht_f32(const ggml_compute_params * params, ggconst int64_t i12 = (r - i13 * ne11 * ne12) / ne11;const int64_t i11 = r - i13 * ne11 * ne12 - i12 * ne11;- const float * src_row = (const float *) ((const char *) src1->data + i11 * nb11 + i12 * nb12 + i13 * nb13);+ const src_t * src_row = (const src_t *) ((const char *) src1->data + i11 * nb11 + i12 * nb12 + i13 * nb13);float * dst_row = (float *) ((char *) dst->data + i11 * nb1 + i12 * nb2 + i13 * nb3);for (int64_t j = 0; j < n; j++) {- dst_row[j] = src_row[j] * scale;+ dst_row[j] = ggml_fwht_load(src_row[j]) * scale;}// Scalar passes@@ -12097,12 +12106,17 @@ void ggml_compute_forward_fwht(const ggml_compute_params * params, ggml_tensor *switch (src1->type) {case GGML_TYPE_F32:{- ggml_compute_forward_fwht_f32(params, dst);+ ggml_compute_forward_fwht_impl<float>(params, dst);+ }+ break;+ case GGML_TYPE_F16:+ {+ ggml_compute_forward_fwht_impl<ggml_fp16_t>(params, dst);}break;default:{- GGML_ABORT("fatal error - fwht is F32 only");+ GGML_ABORT("fatal error - fwht supports F32 and F16 input");}}}