LLM inference in C/C++
C++ ggml
ggml-cuda: use per-thread stream for buffer-init padding memset (#28782)
Harkirat Gill51ce9c11a6f2
diff --git a/ci/run.sh b/ci/run.shindex 43dccc5c1b..e96b6ba49a 100755--- a/ci/run.sh+++ b/ci/run.sh@@ -649,12 +649,6 @@ function gg_run_test_backend_ops {filocal args_extra="-j ${n_jobs}"- # TODO: fix multi-threaded for ROCm- # https://github.com/ggml-org/llama.cpp/actions/runs/34576278519/job/103297889044?pr=28740#step:3:4865- if [ ! -z ${GG_BUILD_ROCM} ]; then- args_extra=""- fi-# TODO: MoltenVK bug?# https://github.com/ggml-org/llama.cpp/actions/runs/34611260059/job/103302413736?pr=28740#step:3:5897if [ ! -z "${GG_BUILD_VULKAN}" ] && [ "$(uname -s)" = "Darwin" ]; thendiff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 67b5e16d8f..0f88dfe880 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -762,7 +762,8 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_bufferif (padded_size > original_size) {ggml_cuda_set_device(ctx->device);- CUDA_CHECK(cudaMemset((char *)tensor->data + original_size, 0, padded_size - original_size));+ CUDA_CHECK(cudaMemsetAsync((char *)tensor->data + original_size, 0, padded_size - original_size, cudaStreamPerThread));+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));}}return GGML_STATUS_SUCCESS;