Skip to content

LLM inference in C/C++

C++ ggml

quant : Optimise memory usage by evicting weights after processing each layer (#22877)

Ed Addario0596704284bf
diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp
index 1ca698704b..9b22cb05f2 100644
--- a/src/llama-model-loader.cpp
+++ b/src/llama-model-loader.cpp
@@ -1395,6 +1395,11 @@ void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void *
}
}
+void llama_model_loader::unmap_weight(const llama_tensor_weight & w) const {
+ if (!use_mmap) { return; }
+ mappings.at(w.idx)->unmap_fragment(w.offs, w.offs + ggml_nbytes(w.tensor));
+}
+
void llama_model_loader::load_data_for(struct ggml_tensor * cur) const {
const auto & w = require_weight(ggml_get_name(cur));
diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h
index d6b31c2311..e9fe3592d4 100644
--- a/src/llama-model-loader.h
+++ b/src/llama-model-loader.h
@@ -194,6 +194,9 @@ struct llama_model_loader {
void get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const;
+ // release a weight's mmap pages
+ void unmap_weight(const llama_tensor_weight & w) const;
+
// for backwards compatibility, does not support ggml-backend
void load_data_for(struct ggml_tensor * cur) const;
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
index 7f99e96bc3..20252815d5 100644
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -1270,7 +1270,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
total_size_org += tensor_size;
total_size_new += new_size;
- // update the gguf meta data as we go
+ // update the gguf metadata as we go
gguf_set_tensor_type(ctx_outs[cur_split].get(), metadata[i].name.c_str(), new_type);
GGML_ASSERT(gguf_get_tensor_size(ctx_outs[cur_split].get(), gguf_find_tensor(ctx_outs[cur_split].get(), metadata[i].name.c_str())) == new_size);
gguf_set_tensor_data(ctx_outs[cur_split].get(), metadata[i].name.c_str(), new_data);
@@ -1278,6 +1278,10 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
// write tensor data + padding
fout.write((const char *) new_data, new_size);
zeros(fout, GGML_PAD(new_size, align) - new_size);
+
+ // unmap the tensor to free memory
+ if (ml.use_mmap) { ml.unmap_weight(weight); }
+
} // no --dry-run
} // main loop
diff --git a/tools/perplexity/perplexity.cpp b/tools/perplexity/perplexity.cpp
index 92f88306c7..ba41287d8e 100644
--- a/tools/perplexity/perplexity.cpp
+++ b/tools/perplexity/perplexity.cpp
@@ -2023,7 +2023,6 @@ int llama_perplexity(int argc, char ** argv) {
}
const int32_t n_ctx = params.n_ctx;
-
if (n_ctx <= 0) {
LOG_ERR("%s: perplexity tool requires '--ctx-size' > 0\n", __func__);
return 1;