LLM inference in C/C++
C++ ggml
ggml: uniformize im2col dst_type for all conv ops (#23660)
JusteLeo47a39665e708
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cppindex 85eaebb573..7c54cb6f46 100644--- a/ggml/src/ggml-cpu/ops.cpp+++ b/ggml/src/ggml-cpu/ops.cpp@@ -6362,7 +6362,6 @@ static void ggml_compute_forward_im2col_f16(const ggml_tensor * src0 = dst->src[0];const ggml_tensor * src1 = dst->src[1];- GGML_ASSERT(src0->type == GGML_TYPE_F16);GGML_ASSERT(src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F16);@@ -6393,7 +6392,6 @@ static void ggml_compute_forward_im2col_f16(int ofs0 = is_2D ? nb13 : nb12;int ofs1 = is_2D ? nb12 : nb11;- GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));GGML_ASSERT(nb10 == ggml_type_size(src1->type));// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]@@ -6563,7 +6561,6 @@ static void ggml_compute_forward_im2col_3d_f16(const ggml_tensor * src0 = dst->src[0];const ggml_tensor * src1 = dst->src[1];- GGML_ASSERT(src0->type == GGML_TYPE_F16);GGML_ASSERT(src1->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F16);diff --git a/ggml/src/ggml.c b/ggml/src/ggml.cindex 5d88049916..f9cc7465e8 100644--- a/ggml/src/ggml.c+++ b/ggml/src/ggml.c@@ -4507,7 +4507,7 @@ struct ggml_tensor * ggml_conv_1d(int s0,int p0,int d0) {- struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, 0, p0, 0, d0, 0, false, GGML_TYPE_F16); // [N, OL, IC * K]+ struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, 0, p0, 0, d0, 0, false, a->type == GGML_TYPE_BF16 ? GGML_TYPE_F32 : GGML_TYPE_F16); // [N, OL, IC * K]struct ggml_tensor * result =ggml_mul_mat(ctx,@@ -4541,7 +4541,7 @@ struct ggml_tensor * ggml_conv_1d_dw(int d0) {struct ggml_tensor * new_b = ggml_reshape_4d(ctx, b, b->ne[0], 1, b->ne[1], b->ne[2]);- struct ggml_tensor * im2col = ggml_im2col(ctx, a, new_b, s0, 0, p0, 0, d0, 0, false, GGML_TYPE_F16);+ struct ggml_tensor * im2col = ggml_im2col(ctx, a, new_b, s0, 0, p0, 0, d0, 0, false, a->type == GGML_TYPE_BF16 ? GGML_TYPE_F32 : GGML_TYPE_F16);struct ggml_tensor * result = ggml_mul_mat(ctx, im2col, a);@@ -4647,7 +4647,7 @@ struct ggml_tensor * ggml_conv_2d(int p1,int d0,int d1) {- struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, s1, p0, p1, d0, d1, true, a->type); // [N, OH, OW, IC * KH * KW]+ struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, s1, p0, p1, d0, d1, true, a->type == GGML_TYPE_BF16 ? GGML_TYPE_F32 : GGML_TYPE_F16); // [N, OH, OW, IC * KH * KW]struct ggml_tensor * result =ggml_mul_mat(ctx,@@ -4729,7 +4729,7 @@ struct ggml_tensor * ggml_conv_3d(int d1, // dilation heightint d2 // dilation depth) {- struct ggml_tensor * im2col = ggml_im2col_3d(ctx, a, b, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, a->type); // [N*OD, OH, OW, IC * KD * KH * KW]+ struct ggml_tensor * im2col = ggml_im2col_3d(ctx, a, b, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, a->type == GGML_TYPE_BF16 ? GGML_TYPE_F32 : GGML_TYPE_F16); // [N*OD, OH, OW, IC * KD * KH * KW]int64_t OC = a->ne[3] / IC;int64_t N = b->ne[3] / IC;@@ -4779,7 +4779,7 @@ struct ggml_tensor * ggml_conv_2d_dw(struct ggml_tensor * new_a = ggml_reshape_4d(ctx, a, a->ne[0], a->ne[1], 1, a->ne[2] * a->ne[3]);struct ggml_tensor * im2col = ggml_im2col(ctx, new_a,ggml_reshape_4d(ctx, b, b->ne[0], b->ne[1], 1, b->ne[2] * b->ne[3]),- s0, s1, p0, p1, d0, d1, true, GGML_TYPE_F16); // [N * IC, OH, OW, KH * KW]+ s0, s1, p0, p1, d0, d1, true, a->type == GGML_TYPE_BF16 ? GGML_TYPE_F32 : GGML_TYPE_F16); // [N * IC, OH, OW, KH * KW]struct ggml_tensor * new_b = ggml_reshape_4d(ctx, im2col, im2col->ne[0], im2col->ne[2] * im2col->ne[1], b->ne[2], b->ne[3]); // [N * IC, OH, OW, KH * KW] => [N, IC, OH * OW, KH * KW]new_a = ggml_reshape_4d(ctx, new_a, (new_a->ne[0] * new_a->ne[1]), new_a->ne[2], new_a->ne[3], 1); // [OC,1, KH, KW] => [1, OC, 1, KH * KW]diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 86d17b3f60..a83cb302b6 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -8013,6 +8013,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {// im2col 2Dtest_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));+ test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F16));test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32));test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16));for (int s0 : {1, 3}) {