LLM inference in C/C++
C++ ggml
CUDA: fix get_rows_back for tables with more than 65535 rows (grid-y clamp + stride) (#25103)
Matt Jallo931eb37f8cac
diff --git a/ggml/src/ggml-cuda/getrows.cu b/ggml/src/ggml-cuda/getrows.cuindex eb157b8baf..0e15707093 100644--- a/ggml/src/ggml-cuda/getrows.cu+++ b/ggml/src/ggml-cuda/getrows.cu@@ -78,26 +78,29 @@ static __global__ void k_get_rows_float(template<typename grad_t, typename dst_t>static __global__ void k_get_rows_back_float(- const grad_t * __restrict__ grad, const int32_t * __restrict__ rows, dst_t * __restrict__ dst, const int64_t ncols, const int64_t nrows_grad) {+ const grad_t * __restrict__ grad, const int32_t * __restrict__ rows, dst_t * __restrict__ dst,+ const int64_t ncols, const int64_t nrows_grad, const int64_t nrows_dst) {const int col = blockIdx.x*blockDim.x + threadIdx.x;if (col >= ncols) {return;}- const int dst_row = blockIdx.y*blockDim.y + threadIdx.y;+ ggml_cuda_pdl_sync();- float sum = 0.0f;+ // grid.y is clamped to the CUDA grid limit, so stride over the destination rows+ for (int64_t dst_row = blockIdx.y; dst_row < nrows_dst; dst_row += gridDim.y) {+ float sum = 0.0f;- ggml_cuda_pdl_sync();- for (int64_t i = 0; i < nrows_grad; ++i) {- if (rows[i] != dst_row) {- continue;+ for (int64_t i = 0; i < nrows_grad; ++i) {+ if (rows[i] != dst_row) {+ continue;+ }+ sum += grad[i*ncols + col];}- sum += grad[i*ncols + col];- }- dst[dst_row*ncols + col] = sum;+ dst[dst_row*ncols + col] = sum;+ }}template<int qk, int qr, dequantize_kernel_t dq, typename dst_t>@@ -302,7 +305,7 @@ void ggml_cuda_op_get_rows_back(ggml_backend_cuda_context & ctx, ggml_tensor * dconst dim3 block_dims(CUDA_GET_ROWS_BACK_BLOCK_SIZE, 1, 1);const int block_num_x = (ne00 + CUDA_GET_ROWS_BACK_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BACK_BLOCK_SIZE;- const dim3 block_nums(block_num_x, ne1, 1);+ const dim3 block_nums(block_num_x, MIN(ne1, (int64_t)UINT16_MAX), 1);- k_get_rows_back_float<<<block_nums, block_dims, 0, stream>>>(src0_d, src1_d, dst_d, ne00, ne10);+ k_get_rows_back_float<<<block_nums, block_dims, 0, stream>>>(src0_d, src1_d, dst_d, ne00, ne10, ne1);}diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 15b50209c8..34811bd504 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -7759,6 +7759,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {}test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 8, 2, 1, false));+ test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 70000, 4, 1, false)); // row count > CUDA grid-y limit (65535)for (ggml_type type : all_types) {for (bool v : {false, true}) {test_cases.emplace_back(new test_get_rows_back(type, 256, 5, 4, 1, v));