LLM inference in C/C++
C++ ggml
Ggml/cuda col2im 1d (#24417)
Pascal3a3edc9ac65c
diff --git a/ggml/src/ggml-cuda/col2im-1d.cu b/ggml/src/ggml-cuda/col2im-1d.cunew file mode 100644index 0000000000..fecd4c6a95--- /dev/null+++ b/ggml/src/ggml-cuda/col2im-1d.cu@@ -0,0 +1,81 @@+#include "col2im-1d.cuh"+#include "convert.cuh"++// col2im_1d: scatter-add GEMM columns to 1D signal (gather approach)+// columns: [K*OC, T_in] -> output: [T_out, OC]+// Supports F32, F16, BF16 data with F32 accumulator.++template <typename T>+static __global__ void col2im_1d_kernel(+ const T * __restrict__ col,+ T * __restrict__ dst,+ const int T_in, const uint3 T_out_fd,+ const int OC, const int K, const int K_OC,+ const int s0, const int p0, const int total) {++ const int idx = threadIdx.x + blockIdx.x * blockDim.x;+ if (idx >= total) return;++ // dst layout: [T_out, OC], ne[0]=T_out fastest+ const uint2 qr = fast_div_modulo((uint32_t)idx, T_out_fd); // qr.x = idx / T_out, qr.y = idx % T_out+ const int oc = (int)qr.x;+ const int t_out = (int)qr.y;+ const int t_abs = t_out + p0; // absolute position in uncropped signal++ // Gather: find all (t_in, k) where t_in*s + k == t_abs, 0 <= k < K+ int t_in_min = (t_abs - K + s0) / s0; // ceil((t_abs - K + 1) / s)+ if (t_in_min < 0) t_in_min = 0;+ int t_in_max = t_abs / s0;+ if (t_in_max >= T_in) t_in_max = T_in - 1;++ float sum = 0.0f;+ for (int t_in = t_in_min; t_in <= t_in_max; t_in++) {+ const int k = t_abs - t_in * s0;+ // col layout: [K*OC, T_in], column index = oc * K + k+ sum += ggml_cuda_cast<float>(col[(oc * K + k) + t_in * K_OC]);+ }++ dst[idx] = ggml_cuda_cast<T>(sum);+}++void ggml_cuda_op_col2im_1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ cudaStream_t stream = ctx.stream();++ GGML_ASSERT(ggml_is_contiguous(src0));++ const int32_t s0 = ((const int32_t *)(dst->op_params))[0];+ const int32_t OC = ((const int32_t *)(dst->op_params))[1];+ const int32_t p0 = ((const int32_t *)(dst->op_params))[2];++ const int K_OC = (int) src0->ne[0];+ const int T_in = (int) src0->ne[1];+ const int K = K_OC / OC;+ const int T_out = (int) dst->ne[0];++ const uint3 T_out_fd = init_fastdiv_values((uint32_t)T_out);++ const int total = T_out * OC;+ const int block_size = 256;+ const int num_blocks = (total + block_size - 1) / block_size;++ switch (src0->type) {+ case GGML_TYPE_F32: {+ col2im_1d_kernel<<<num_blocks, block_size, 0, stream>>>(+ (const float *)src0->data, (float *)dst->data,+ T_in, T_out_fd, OC, K, K_OC, s0, p0, total);+ } break;+ case GGML_TYPE_F16: {+ col2im_1d_kernel<<<num_blocks, block_size, 0, stream>>>(+ (const half *)src0->data, (half *)dst->data,+ T_in, T_out_fd, OC, K, K_OC, s0, p0, total);+ } break;+ case GGML_TYPE_BF16: {+ col2im_1d_kernel<<<num_blocks, block_size, 0, stream>>>(+ (const nv_bfloat16 *)src0->data, (nv_bfloat16 *)dst->data,+ T_in, T_out_fd, OC, K, K_OC, s0, p0, total);+ } break;+ default:+ GGML_ABORT("col2im_1d: unsupported type");+ }+}diff --git a/ggml/src/ggml-cuda/col2im-1d.cuh b/ggml/src/ggml-cuda/col2im-1d.cuhnew file mode 100644index 0000000000..efc3313c4d--- /dev/null+++ b/ggml/src/ggml-cuda/col2im-1d.cuh@@ -0,0 +1,3 @@+#include "common.cuh"++void ggml_cuda_op_col2im_1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 34cdbc81c2..3d4b5f6056 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -11,6 +11,7 @@#include "ggml-cuda/argsort.cuh"#include "ggml-cuda/binbcast.cuh"#include "ggml-cuda/clamp.cuh"+#include "ggml-cuda/col2im-1d.cuh"#include "ggml-cuda/concat.cuh"#include "ggml-cuda/conv-transpose-1d.cuh"#include "ggml-cuda/conv2d.cuh"@@ -3051,6 +3052,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggcase GGML_OP_CONV_TRANSPOSE_1D:ggml_cuda_op_conv_transpose_1d(ctx,dst);break;+ case GGML_OP_COL2IM_1D:+ ggml_cuda_op_col2im_1d(ctx, dst);+ break;case GGML_OP_POOL_2D:ggml_cuda_op_pool2d(ctx, dst);break;@@ -5316,6 +5320,14 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g}return false;} break;+ case GGML_OP_COL2IM_1D:+ {+ ggml_type src0_type = op->src[0]->type;+ return (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_F16 || src0_type == GGML_TYPE_BF16) &&+ op->type == src0_type &&+ ggml_is_contiguous(op->src[0]) &&+ ggml_is_contiguous(op);+ } break;case GGML_OP_SILU_BACK:return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;break;