Skip to content

LLM inference in C/C++

C++ ggml

ggml-cuda: use per-thread stream for buffer-init padding memset (#28782)

Harkirat Gill51ce9c11a6f2
diff --git a/ci/run.sh b/ci/run.sh
index 43dccc5c1b..e96b6ba49a 100755
--- a/ci/run.sh
+++ b/ci/run.sh
@@ -649,12 +649,6 @@ function gg_run_test_backend_ops {
fi
local args_extra="-j ${n_jobs}"
- # TODO: fix multi-threaded for ROCm
- # https://github.com/ggml-org/llama.cpp/actions/runs/34576278519/job/103297889044?pr=28740#step:3:4865
- if [ ! -z ${GG_BUILD_ROCM} ]; then
- args_extra=""
- fi
-
# TODO: MoltenVK bug?
# https://github.com/ggml-org/llama.cpp/actions/runs/34611260059/job/103302413736?pr=28740#step:3:5897
if [ ! -z "${GG_BUILD_VULKAN}" ] && [ "$(uname -s)" = "Darwin" ]; then
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 67b5e16d8f..0f88dfe880 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -762,7 +762,8 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer
if (padded_size > original_size) {
ggml_cuda_set_device(ctx->device);
- CUDA_CHECK(cudaMemset((char *)tensor->data + original_size, 0, padded_size - original_size));
+ CUDA_CHECK(cudaMemsetAsync((char *)tensor->data + original_size, 0, padded_size - original_size, cudaStreamPerThread));
+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
}
return GGML_STATUS_SUCCESS;