LLM inference in C/C++
C++ ggml
CUDA: fix thread/block count in quantized cpy kernel launches (#26731)
Rafail Giavrimis69bf64379145
diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cuindex eb5eb0eb4e..fd7ffc0bc5 100644--- a/ggml/src/ggml-cuda/cpy.cu+++ b/ggml/src/ggml-cuda/cpy.cu@@ -253,9 +253,9 @@ static void ggml_cpy_f32_q8_0_cuda(const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {GGML_ASSERT(ne % QK8_0 == 0);- const int64_t num_blocks = ne / QK8_0;+ const int64_t num_blocks = (ne/QK8_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;GGML_ASSERT(num_blocks <= INT_MAX);- cpy_f32_q<cpy_blck_f32_q8_0, QK8_0><<<num_blocks, 1, 0, stream>>>+ cpy_f32_q<cpy_blck_f32_q8_0, QK8_0><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}@@ -264,9 +264,9 @@ static void ggml_cpy_q8_0_f32_cuda(const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {- const int64_t num_blocks = ne;+ const int64_t num_blocks = (ne/QK8_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;GGML_ASSERT(num_blocks <= INT_MAX);- cpy_q_f32<cpy_blck_q8_0_f32, QK8_0><<<num_blocks, 1, 0, stream>>>+ cpy_q_f32<cpy_blck_q8_0_f32, QK8_0><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}@@ -276,9 +276,9 @@ static void ggml_cpy_f32_q4_0_cuda(const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {GGML_ASSERT(ne % QK4_0 == 0);- const int64_t num_blocks = ne / QK4_0;+ const int64_t num_blocks = (ne/QK4_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;GGML_ASSERT(num_blocks <= INT_MAX);- cpy_f32_q<cpy_blck_f32_q4_0, QK4_0><<<num_blocks, 1, 0, stream>>>+ cpy_f32_q<cpy_blck_f32_q4_0, QK4_0><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}@@ -289,9 +289,9 @@ static void ggml_cpy_q4_0_f32_cuda(const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,cudaStream_t stream) {- const int64_t num_blocks = ne;+ const int64_t num_blocks = (ne/QK4_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;GGML_ASSERT(num_blocks <= INT_MAX);- cpy_q_f32<cpy_blck_q_f32<dequantize_q4_0, QK4_0>, QK4_0><<<num_blocks, 1, 0, stream>>>(+ cpy_q_f32<cpy_blck_q_f32<dequantize_q4_0, QK4_0>, QK4_0><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,ne10, ne11, ne12, nb10, nb11, nb12, nb13);}@@ -302,9 +302,9 @@ static void ggml_cpy_f32_q4_1_cuda(const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {GGML_ASSERT(ne % QK4_1 == 0);- const int64_t num_blocks = ne / QK4_1;+ const int64_t num_blocks = (ne/QK4_1 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;GGML_ASSERT(num_blocks <= INT_MAX);- cpy_f32_q<cpy_blck_f32_q4_1, QK4_1><<<num_blocks, 1, 0, stream>>>+ cpy_f32_q<cpy_blck_f32_q4_1, QK4_1><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}@@ -315,9 +315,9 @@ static void ggml_cpy_q4_1_f32_cuda(const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,cudaStream_t stream) {- const int64_t num_blocks = ne;+ const int64_t num_blocks = (ne/QK4_1 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;GGML_ASSERT(num_blocks <= INT_MAX);- cpy_q_f32<cpy_blck_q_f32<dequantize_q4_1, QK4_1>, QK4_1><<<num_blocks, 1, 0, stream>>>(+ cpy_q_f32<cpy_blck_q_f32<dequantize_q4_1, QK4_1>, QK4_1><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,ne10, ne11, ne12, nb10, nb11, nb12, nb13);}@@ -328,9 +328,9 @@ static void ggml_cpy_f32_q5_0_cuda(const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {GGML_ASSERT(ne % QK5_0 == 0);- const int64_t num_blocks = ne / QK5_0;+ const int64_t num_blocks = (ne/QK5_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;GGML_ASSERT(num_blocks <= INT_MAX);- cpy_f32_q<cpy_blck_f32_q5_0, QK5_0><<<num_blocks, 1, 0, stream>>>+ cpy_f32_q<cpy_blck_f32_q5_0, QK5_0><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}@@ -341,9 +341,9 @@ static void ggml_cpy_q5_0_f32_cuda(const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,cudaStream_t stream) {- const int64_t num_blocks = ne;+ const int64_t num_blocks = (ne/QK5_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;GGML_ASSERT(num_blocks <= INT_MAX);- cpy_q_f32<cpy_blck_q_f32<dequantize_q5_0, QK5_0>, QK5_0><<<num_blocks, 1, 0, stream>>>(+ cpy_q_f32<cpy_blck_q_f32<dequantize_q5_0, QK5_0>, QK5_0><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,ne10, ne11, ne12, nb10, nb11, nb12, nb13);}@@ -354,9 +354,9 @@ static void ggml_cpy_f32_q5_1_cuda(const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {GGML_ASSERT(ne % QK5_1 == 0);- const int64_t num_blocks = ne / QK5_1;+ const int64_t num_blocks = (ne/QK5_1 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;GGML_ASSERT(num_blocks <= INT_MAX);- cpy_f32_q<cpy_blck_f32_q5_1, QK5_1><<<num_blocks, 1, 0, stream>>>+ cpy_f32_q<cpy_blck_f32_q5_1, QK5_1><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}@@ -367,9 +367,9 @@ static void ggml_cpy_q5_1_f32_cuda(const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,cudaStream_t stream) {- const int64_t num_blocks = ne;+ const int64_t num_blocks = (ne/QK5_1 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;GGML_ASSERT(num_blocks <= INT_MAX);- cpy_q_f32<cpy_blck_q_f32<dequantize_q5_1, QK5_1>, QK5_1><<<num_blocks, 1, 0, stream>>>(+ cpy_q_f32<cpy_blck_q_f32<dequantize_q5_1, QK5_1>, QK5_1><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,ne10, ne11, ne12, nb10, nb11, nb12, nb13);}@@ -380,9 +380,9 @@ static void ggml_cpy_f32_iq4_nl_cuda(const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {GGML_ASSERT(ne % QK4_NL == 0);- const int64_t num_blocks = ne / QK4_NL;+ const int64_t num_blocks = (ne/QK4_NL + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;GGML_ASSERT(num_blocks <= INT_MAX);- cpy_f32_q<cpy_blck_f32_iq4_nl, QK4_NL><<<num_blocks, 1, 0, stream>>>+ cpy_f32_q<cpy_blck_f32_iq4_nl, QK4_NL><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 6d474bc114..6688debd4a 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -8576,6 +8576,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3})); // cpy not-contiguous}}+ // quant block count not a multiple of the kernel block size+ test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_Q4_0, {96, 1, 1, 1}));+ test_cases.emplace_back(new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_F32, {96, 1, 1, 1}));test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_I32, {256, 2, 3, 4}));test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_I32, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3}));test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_F32, {256, 2, 3, 4}));