LLM inference in C/C++
C++ ggml
Limit max CUDA pinned host buffer size to 4 GB (#30449)
Gaurav Gargb6334e37e6ef
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 7c45d0d1fb..6fa22289f3 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -1348,6 +1348,12 @@ static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggmreturn buffer;}+static size_t ggml_backend_cuda_host_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {+ GGML_UNUSED(buft);+ // very large pinned allocations can fail (e.g. on Windows) and fall back to pageable memory, smaller ones succeed+ return 4ull*1024*1024*1024;+}+ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = {/* .iface = */ {@@ -1355,7 +1361,7 @@ ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {/* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer,/* .alloc_buffer_n = */ NULL,/* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,- /* .get_max_size = */ NULL, // defaults to SIZE_MAX+ /* .get_max_size = */ ggml_backend_cuda_host_buffer_type_get_max_size,/* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,/* .get_alloc_size_n = */ NULL,/* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host,diff --git a/src/llama-model.cpp b/src/llama-model.cppindex 5d48d702c5..3da32333b1 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -1950,7 +1950,8 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {if (buf == nullptr) {throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft)));}- if (use_mlock && ggml_backend_buffer_is_host(buf)) {+ // a multi_buffer has no base to lock, its parts are usually pinned memory (host buffer types with a max size)+ if (use_mlock && ggml_backend_buffer_is_host(buf) && ggml_backend_buffer_get_base(buf) != nullptr) {pimpl->mlock_bufs.emplace_back(new llama_mlock);auto & mlock_buf = pimpl->mlock_bufs.back();mlock_buf->init (ggml_backend_buffer_get_base(buf));