LLM inference in C/C++
C++ ggml
quant : Optimise memory usage by evicting weights after processing each layer (#22877)
Ed Addario0596704284bf
diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cppindex 1ca698704b..9b22cb05f2 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -1395,6 +1395,11 @@ void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void *}}+void llama_model_loader::unmap_weight(const llama_tensor_weight & w) const {+ if (!use_mmap) { return; }+ mappings.at(w.idx)->unmap_fragment(w.offs, w.offs + ggml_nbytes(w.tensor));+}+void llama_model_loader::load_data_for(struct ggml_tensor * cur) const {const auto & w = require_weight(ggml_get_name(cur));diff --git a/src/llama-model-loader.h b/src/llama-model-loader.hindex d6b31c2311..e9fe3592d4 100644--- a/src/llama-model-loader.h+++ b/src/llama-model-loader.h@@ -194,6 +194,9 @@ struct llama_model_loader {void get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const;+ // release a weight's mmap pages+ void unmap_weight(const llama_tensor_weight & w) const;+// for backwards compatibility, does not support ggml-backendvoid load_data_for(struct ggml_tensor * cur) const;diff --git a/src/llama-quant.cpp b/src/llama-quant.cppindex 7f99e96bc3..20252815d5 100644--- a/src/llama-quant.cpp+++ b/src/llama-quant.cpp@@ -1270,7 +1270,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::total_size_org += tensor_size;total_size_new += new_size;- // update the gguf meta data as we go+ // update the gguf metadata as we gogguf_set_tensor_type(ctx_outs[cur_split].get(), metadata[i].name.c_str(), new_type);GGML_ASSERT(gguf_get_tensor_size(ctx_outs[cur_split].get(), gguf_find_tensor(ctx_outs[cur_split].get(), metadata[i].name.c_str())) == new_size);gguf_set_tensor_data(ctx_outs[cur_split].get(), metadata[i].name.c_str(), new_data);@@ -1278,6 +1278,10 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::// write tensor data + paddingfout.write((const char *) new_data, new_size);zeros(fout, GGML_PAD(new_size, align) - new_size);++ // unmap the tensor to free memory+ if (ml.use_mmap) { ml.unmap_weight(weight); }+} // no --dry-run} // main loopdiff --git a/tools/perplexity/perplexity.cpp b/tools/perplexity/perplexity.cppindex 92f88306c7..ba41287d8e 100644--- a/tools/perplexity/perplexity.cpp+++ b/tools/perplexity/perplexity.cpp@@ -2023,7 +2023,6 @@ int llama_perplexity(int argc, char ** argv) {}const int32_t n_ctx = params.n_ctx;-if (n_ctx <= 0) {LOG_ERR("%s: perplexity tool requires '--ctx-size' > 0\n", __func__);return 1;