LLM inference in C/C++
C++ ggml
ggml : fix conv_transpose_2d for multiple batches (#26132)
Tekin Ertekina43c3986b442
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.cindex 87ac0a702e..b9c0fa3ddc 100644--- a/ggml/src/ggml-cpu/ggml-cpu.c+++ b/ggml/src/ggml-cpu/ggml-cpu.c@@ -2936,12 +2936,13 @@ struct ggml_cplan ggml_graph_plan(const int64_t ne10 = node->src[1]->ne[0]; // Wconst int64_t ne11 = node->src[1]->ne[1]; // Hconst int64_t ne12 = node->src[1]->ne[2]; // Channels In+ const int64_t ne13 = node->src[1]->ne[3]; // BatchGGML_ASSERT(node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_F32);GGML_ASSERT(node->src[1]->type == GGML_TYPE_F32);cur += ggml_type_size(node->src[0]->type) * ne00 * ne01 * ne02 * ne03;- cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12;+ cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12 * ne13;} break;case GGML_OP_TOP_K:diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cppindex b869f4bddd..b47ce5463c 100644--- a/ggml/src/ggml-cpu/ops.cpp+++ b/ggml/src/ggml-cpu/ops.cpp@@ -7267,18 +7267,21 @@ static void ggml_compute_forward_conv_transpose_2d_impl(}}- // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh)+ // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh), for all batches{kernel_t * const wdata = (kernel_t *) params->wdata + nk;- for (int i12 = 0; i12 < ne12; i12++) {- for (int i11 = 0; i11 < ne11; i11++) {- const float * const src = (float *)((char *) src1->data + i12*nb12 + i11*nb11);- kernel_t * dst_data = wdata + i11*ne10*ne12;- for (int i10 = 0; i10 < ne10; i10++) {- if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {- dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]);- } else {- dst_data[i10*ne12 + i12] = src[i10];+ for (int i13 = 0; i13 < ne13; i13++) {+ kernel_t * const wdata_b = wdata + i13*ne10*ne11*ne12;+ for (int i12 = 0; i12 < ne12; i12++) {+ for (int i11 = 0; i11 < ne11; i11++) {+ const float * const src = (float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);+ kernel_t * dst_data = wdata_b + i11*ne10*ne12;+ for (int i10 = 0; i10 < ne10; i10++) {+ if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {+ dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]);+ } else {+ dst_data[i10*ne12 + i12] = src[i10];+ }}}}@@ -7305,24 +7308,27 @@ static void ggml_compute_forward_conv_transpose_2d_impl(kernel_t * const wdata_src = wdata + nk;for (int i2 = ip0; i2 < ip1; i2++) { // Cout- float * dst_data = (float *)((char *) dst->data + i2*nb2);kernel_t * wdata_kernel = wdata + i2*ne01*ne00*ne03;- for (int i11 = 0; i11 < ne11; i11++) {- for (int i10 = 0; i10 < ne10; i10++) {- const int i1n = i11*ne10*ne12 + i10*ne12;- for (int i01 = 0; i01 < ne01; i01++) {- for (int i00 = 0; i00 < ne00; i00++) {- float v = 0;- if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {- ggml_vec_dot_f16(ne03, &v, 0,- wdata_src + i1n, 0,- wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);- } else {- ggml_vec_dot_f32(ne03, &v, 0,- wdata_src + i1n, 0,- wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);+ for (int i3 = 0; i3 < ne3; i3++) { // batch+ float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2);+ kernel_t * wdata_src_b = wdata_src + i3*ne10*ne11*ne12;+ for (int i11 = 0; i11 < ne11; i11++) {+ for (int i10 = 0; i10 < ne10; i10++) {+ const int i1n = i11*ne10*ne12 + i10*ne12;+ for (int i01 = 0; i01 < ne01; i01++) {+ for (int i00 = 0; i00 < ne00; i00++) {+ float v = 0;+ if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {+ ggml_vec_dot_f16(ne03, &v, 0,+ wdata_src_b + i1n, 0,+ wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);+ } else {+ ggml_vec_dot_f32(ne03, &v, 0,+ wdata_src_b + i1n, 0,+ wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);+ }+ dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v;}- dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v;}}}diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.hindex 9becf04797..49102afe9c 100644--- a/ggml/src/ggml-metal/ggml-metal-impl.h+++ b/ggml/src/ggml-metal/ggml-metal-impl.h@@ -660,6 +660,7 @@ typedef struct {uint64_t nb0;uint64_t nb1;uint64_t nb2;+ uint64_t nb3;} ggml_metal_kargs_conv_transpose_2d;typedef struct {diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cppindex 75de0f6dd0..f6f2fdc86c 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp@@ -4645,6 +4645,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) {const int32_t OW = op->ne[0];const int32_t OH = op->ne[1];const int32_t OC = op->ne[2];+ const int32_t N = op->src[1]->ne[3];ggml_metal_kargs_conv_transpose_2d args = {/*.IC =*/ IC,@@ -4657,6 +4658,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) {/*.nb0 =*/ nb0,/*.nb1 =*/ nb1,/*.nb2 =*/ nb2,+ /*.nb3 =*/ nb3,};auto pipeline = ggml_metal_library_get_pipeline_conv_transpose_2d(lib, op);@@ -4671,7 +4673,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) {const size_t smem = GGML_PAD(KW * KH * sizeof(float), 16);ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);- ggml_metal_encoder_dispatch_threadgroups(enc, OW, OH, OC, KW, KH, 1);+ ggml_metal_encoder_dispatch_threadgroups(enc, OW, OH, OC * N, KW, KH, 1);return 1;}diff --git a/ggml/src/ggml-metal/kernels/conv.metal b/ggml/src/ggml-metal/kernels/conv.metalindex 5685b5cd49..a5d5aa9d92 100644--- a/ggml/src/ggml-metal/kernels/conv.metal+++ b/ggml/src/ggml-metal/kernels/conv.metal@@ -366,7 +366,8 @@ kernel void kernel_conv_transpose_2d(const int64_t out_x = tgpig[0];const int64_t out_y = tgpig[1];- const int64_t out_c = tgpig[2];+ const int64_t batch = tgpig[2] / args.OC;+ const int64_t out_c = tgpig[2] % args.OC;const int64_t kw = tpitg[0];const int64_t kh = tpitg[1];@@ -390,7 +391,7 @@ kernel void kernel_conv_transpose_2d(if (in_x >= args.IW) continue;- const int64_t input_idx = (args.IW * args.IH) * in_c + (args.IW) * in_y + in_x;+ const int64_t input_idx = (args.IW * args.IH) * (args.IC * batch + in_c) + (args.IW) * in_y + in_x;const int64_t kernel_idx = (args.KH * args.KW * args.OC) * in_c + (args.KH * args.KW) * out_c + (args.KW) * kh + kw;v += (float)src0[kernel_idx] * src1[input_idx];@@ -408,7 +409,7 @@ kernel void kernel_conv_transpose_2d(total += shared_sum[i];}- device float * dst_ptr = (device float *) (dst + out_x*args.nb0 + out_y * args.nb1 + out_c*args.nb2);+ device float * dst_ptr = (device float *) (dst + batch*args.nb3 + out_c*args.nb2 + out_y * args.nb1 + out_x*args.nb0);dst_ptr[0] = total;}}diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex df0b9aa666..6be83ac161 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -8785,6 +8785,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_conv_transpose_2d({3, 2, 3, 1}, {2, 2, 1, 3}, 1, kernel_type));test_cases.emplace_back(new test_conv_transpose_2d({10, 10, 9, 1}, {3, 3, 1, 9}, 2, kernel_type));test_cases.emplace_back(new test_conv_transpose_2d({129, 63, 35, 1}, {3, 3, 48, 35}, 1, kernel_type));+ test_cases.emplace_back(new test_conv_transpose_2d({10, 10, 9, 2}, {3, 3, 1, 9}, 2, kernel_type)); // for multiple batches}test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 500, 1, 1}));