LLM inference in C/C++
C++ ggml
Fix !30793
Gaurav Garg36cbf415f1c7
diff --git a/src/llama-model.cpp b/src/llama-model.cppindex 60199d6d99..4cb04097ef 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -1968,12 +1968,20 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {if (buf == nullptr) {throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft)));}- // a multi_buffer has no base to lock, its parts are usually pinned memory (host buffer types with a max size)- if (use_mlock && ggml_backend_buffer_is_host(buf) && ggml_backend_buffer_get_base(buf) != nullptr) {- pimpl->mlock_bufs.emplace_back(new llama_mlock);- auto & mlock_buf = pimpl->mlock_bufs.back();- mlock_buf->init (ggml_backend_buffer_get_base(buf));- mlock_buf->grow_to(ggml_backend_buffer_get_size(buf));+ // lock the buffers that hold the tensors: a multi_buffer has no base, and its parts are pageable memory when pinning fails+ if (use_mlock && !ml.no_alloc && ggml_backend_buffer_is_host(buf)) {+ std::vector<ggml_backend_buffer_t> lock_bufs;+ for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {+ if (t->buffer != nullptr && std::find(lock_bufs.begin(), lock_bufs.end(), t->buffer) == lock_bufs.end()) {+ lock_bufs.push_back(t->buffer);+ }+ }+ for (ggml_backend_buffer_t lock_buf : lock_bufs) {+ pimpl->mlock_bufs.emplace_back(new llama_mlock);+ auto & mlock_buf = pimpl->mlock_bufs.back();+ mlock_buf->init (ggml_backend_buffer_get_base(lock_buf));+ mlock_buf->grow_to(ggml_backend_buffer_get_size(lock_buf));+ }}bufs.emplace_back(buf);}