Skip to content

LLM inference in C/C++

C++ ggml

ggml : fix conv_transpose_2d for multiple batches (#26132)

Tekin Ertekina43c3986b442
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
index 87ac0a702e..b9c0fa3ddc 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -2936,12 +2936,13 @@ struct ggml_cplan ggml_graph_plan(
const int64_t ne10 = node->src[1]->ne[0]; // W
const int64_t ne11 = node->src[1]->ne[1]; // H
const int64_t ne12 = node->src[1]->ne[2]; // Channels In
+ const int64_t ne13 = node->src[1]->ne[3]; // Batch
GGML_ASSERT(node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_F32);
GGML_ASSERT(node->src[1]->type == GGML_TYPE_F32);
cur += ggml_type_size(node->src[0]->type) * ne00 * ne01 * ne02 * ne03;
- cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12;
+ cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12 * ne13;
} break;
case GGML_OP_TOP_K:
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index b869f4bddd..b47ce5463c 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -7267,18 +7267,21 @@ static void ggml_compute_forward_conv_transpose_2d_impl(
}
}
- // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh)
+ // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh), for all batches
{
kernel_t * const wdata = (kernel_t *) params->wdata + nk;
- for (int i12 = 0; i12 < ne12; i12++) {
- for (int i11 = 0; i11 < ne11; i11++) {
- const float * const src = (float *)((char *) src1->data + i12*nb12 + i11*nb11);
- kernel_t * dst_data = wdata + i11*ne10*ne12;
- for (int i10 = 0; i10 < ne10; i10++) {
- if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {
- dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]);
- } else {
- dst_data[i10*ne12 + i12] = src[i10];
+ for (int i13 = 0; i13 < ne13; i13++) {
+ kernel_t * const wdata_b = wdata + i13*ne10*ne11*ne12;
+ for (int i12 = 0; i12 < ne12; i12++) {
+ for (int i11 = 0; i11 < ne11; i11++) {
+ const float * const src = (float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);
+ kernel_t * dst_data = wdata_b + i11*ne10*ne12;
+ for (int i10 = 0; i10 < ne10; i10++) {
+ if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {
+ dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]);
+ } else {
+ dst_data[i10*ne12 + i12] = src[i10];
+ }
}
}
}
@@ -7305,24 +7308,27 @@ static void ggml_compute_forward_conv_transpose_2d_impl(
kernel_t * const wdata_src = wdata + nk;
for (int i2 = ip0; i2 < ip1; i2++) { // Cout
- float * dst_data = (float *)((char *) dst->data + i2*nb2);
kernel_t * wdata_kernel = wdata + i2*ne01*ne00*ne03;
- for (int i11 = 0; i11 < ne11; i11++) {
- for (int i10 = 0; i10 < ne10; i10++) {
- const int i1n = i11*ne10*ne12 + i10*ne12;
- for (int i01 = 0; i01 < ne01; i01++) {
- for (int i00 = 0; i00 < ne00; i00++) {
- float v = 0;
- if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {
- ggml_vec_dot_f16(ne03, &v, 0,
- wdata_src + i1n, 0,
- wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
- } else {
- ggml_vec_dot_f32(ne03, &v, 0,
- wdata_src + i1n, 0,
- wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
+ for (int i3 = 0; i3 < ne3; i3++) { // batch
+ float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2);
+ kernel_t * wdata_src_b = wdata_src + i3*ne10*ne11*ne12;
+ for (int i11 = 0; i11 < ne11; i11++) {
+ for (int i10 = 0; i10 < ne10; i10++) {
+ const int i1n = i11*ne10*ne12 + i10*ne12;
+ for (int i01 = 0; i01 < ne01; i01++) {
+ for (int i00 = 0; i00 < ne00; i00++) {
+ float v = 0;
+ if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {
+ ggml_vec_dot_f16(ne03, &v, 0,
+ wdata_src_b + i1n, 0,
+ wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
+ } else {
+ ggml_vec_dot_f32(ne03, &v, 0,
+ wdata_src_b + i1n, 0,
+ wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
+ }
+ dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v;
}
- dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v;
}
}
}
diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h
index 9becf04797..49102afe9c 100644
--- a/ggml/src/ggml-metal/ggml-metal-impl.h
+++ b/ggml/src/ggml-metal/ggml-metal-impl.h
@@ -660,6 +660,7 @@ typedef struct {
uint64_t nb0;
uint64_t nb1;
uint64_t nb2;
+ uint64_t nb3;
} ggml_metal_kargs_conv_transpose_2d;
typedef struct {
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp
index 75de0f6dd0..f6f2fdc86c 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp
@@ -4645,6 +4645,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) {
const int32_t OW = op->ne[0];
const int32_t OH = op->ne[1];
const int32_t OC = op->ne[2];
+ const int32_t N = op->src[1]->ne[3];
ggml_metal_kargs_conv_transpose_2d args = {
/*.IC =*/ IC,
@@ -4657,6 +4658,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) {
/*.nb0 =*/ nb0,
/*.nb1 =*/ nb1,
/*.nb2 =*/ nb2,
+ /*.nb3 =*/ nb3,
};
auto pipeline = ggml_metal_library_get_pipeline_conv_transpose_2d(lib, op);
@@ -4671,7 +4673,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) {
const size_t smem = GGML_PAD(KW * KH * sizeof(float), 16);
ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);
- ggml_metal_encoder_dispatch_threadgroups(enc, OW, OH, OC, KW, KH, 1);
+ ggml_metal_encoder_dispatch_threadgroups(enc, OW, OH, OC * N, KW, KH, 1);
return 1;
}
diff --git a/ggml/src/ggml-metal/kernels/conv.metal b/ggml/src/ggml-metal/kernels/conv.metal
index 5685b5cd49..a5d5aa9d92 100644
--- a/ggml/src/ggml-metal/kernels/conv.metal
+++ b/ggml/src/ggml-metal/kernels/conv.metal
@@ -366,7 +366,8 @@ kernel void kernel_conv_transpose_2d(
const int64_t out_x = tgpig[0];
const int64_t out_y = tgpig[1];
- const int64_t out_c = tgpig[2];
+ const int64_t batch = tgpig[2] / args.OC;
+ const int64_t out_c = tgpig[2] % args.OC;
const int64_t kw = tpitg[0];
const int64_t kh = tpitg[1];
@@ -390,7 +391,7 @@ kernel void kernel_conv_transpose_2d(
if (in_x >= args.IW) continue;
- const int64_t input_idx = (args.IW * args.IH) * in_c + (args.IW) * in_y + in_x;
+ const int64_t input_idx = (args.IW * args.IH) * (args.IC * batch + in_c) + (args.IW) * in_y + in_x;
const int64_t kernel_idx = (args.KH * args.KW * args.OC) * in_c + (args.KH * args.KW) * out_c + (args.KW) * kh + kw;
v += (float)src0[kernel_idx] * src1[input_idx];
@@ -408,7 +409,7 @@ kernel void kernel_conv_transpose_2d(
total += shared_sum[i];
}
- device float * dst_ptr = (device float *) (dst + out_x*args.nb0 + out_y * args.nb1 + out_c*args.nb2);
+ device float * dst_ptr = (device float *) (dst + batch*args.nb3 + out_c*args.nb2 + out_y * args.nb1 + out_x*args.nb0);
dst_ptr[0] = total;
}
}
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index df0b9aa666..6be83ac161 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -8785,6 +8785,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_conv_transpose_2d({3, 2, 3, 1}, {2, 2, 1, 3}, 1, kernel_type));
test_cases.emplace_back(new test_conv_transpose_2d({10, 10, 9, 1}, {3, 3, 1, 9}, 2, kernel_type));
test_cases.emplace_back(new test_conv_transpose_2d({129, 63, 35, 1}, {3, 3, 48, 35}, 1, kernel_type));
+ test_cases.emplace_back(new test_conv_transpose_2d({10, 10, 9, 2}, {3, 3, 1, 9}, 2, kernel_type)); // for multiple batches
}
test_cases.emplace_back(new test_count_equal(GGML_TYPE_F32, {4, 500, 1, 1}));