Skip to content

LLM inference in C/C++

C++ ggml

llama : do not map foreign ranges into a backend buffer (#30405)

Georgi Gerganovb9f3599fb1c4
diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
index 045489df7a..80a719670b 100644
--- a/src/llama-mmap.cpp
+++ b/src/llama-mmap.cpp
@@ -835,7 +835,7 @@ const bool llama_mlock::SUPPORTED = false;
#endif
void llama_prefetch(llama_memory_ranges mr) {
-#if defined(__linux__) || (defined(_WIN32) && _WIN32_WINNT >= 0x602)
+#if defined(__linux__) || defined(__APPLE__) || (defined(_WIN32) && _WIN32_WINNT >= 0x602)
if (mr.empty()) {
return;
}
@@ -874,6 +874,7 @@ void llama_prefetch(llama_memory_ranges mr) {
};
#else
auto prefetch = [&]() {
+ // TODO: in a follow-up, use posix_madvise(.., POSIX_MADV_WILLNEED) to match the rest of this file
if (madvise((void *) begin, end - begin, MADV_WILLNEED) != 0) {
LLAMA_LOG_WARN("llama_prefetch: madvise(MADV_WILLNEED) failed: %s\n", strerror(errno));
return false;
diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp
index 6cc0b20288..bc93c6f9e6 100644
--- a/src/llama-model-loader.cpp
+++ b/src/llama-model-loader.cpp
@@ -13,6 +13,7 @@
#include <cstring>
#include <future>
#include <regex>
+#include <unordered_set>
static const size_t kiB = 1024;
static const size_t MiB = 1024*kiB;
@@ -1446,21 +1447,80 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps
}
}
-void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const {
- GGML_ASSERT(!mappings.empty());
- const auto & mapping = mappings.at(idx);
+using mapping_ranges = std::vector<llama_model_loader::mapping_range>;
+
+// merge a list of [first, last) ranges into a sorted, non-overlapping list
+static mapping_ranges merge_ranges(mapping_ranges ranges) {
+ std::sort(ranges.begin(), ranges.end(), [](const auto & a, const auto & b) { return a.first < b.first; });
+
+ mapping_ranges res;
+ for (const auto & range : ranges) {
+ if (!res.empty() && range.first <= res.back().last) {
+ res.back().last = std::max(res.back().last, range.last);
+ } else {
+ res.push_back(range);
+ }
+ }
+
+ return res;
+}
+
+// subtract the sorted, non-overlapping ranges 'sub' from 'a'
+static mapping_ranges sub_ranges(const mapping_ranges & a, const mapping_ranges & sub) {
+ mapping_ranges res;
+
+ for (const auto & x : a) {
+ size_t cur = x.first;
+ for (const auto & y : sub) {
+ if (y.last <= cur || y.first >= x.last) {
+ continue;
+ }
+ if (y.first > cur) {
+ res.push_back({ cur, y.first });
+ }
+ cur = std::max(cur, y.last);
+ }
+ if (cur < x.last) {
+ res.push_back({ cur, x.last });
+ }
+ }
+
+ return res;
+}
+
+std::vector<llama_model_loader::mapping_range> llama_model_loader::get_mapping_ranges(uint32_t idx, ggml_context * ctx) const {
+ GGML_ASSERT(idx < mappings.size());
+
+ // data of this context, and data of the other contexts that share the same file
+ // a weight that is mapped in several contexts (e.g. the token embedding) is not foreign data
+ std::unordered_set<const llama_tensor_weight *> own;
+ mapping_ranges own_ranges;
+ mapping_ranges other;
- *first = mapping->size();
- *last = 0;
- *addr = mapping->addr();
for (ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor; tensor = ggml_get_next_tensor(ctx, tensor)) {
const auto * weight = get_weight(ggml_get_name(tensor));
if (!weight || weight->idx != idx) {
continue;
}
- *first = std::min(*first, weight->offs);
- *last = std::max(*last, weight->offs + ggml_nbytes(tensor));
+ own.insert(weight);
+ own_ranges.push_back({ weight->offs, weight->offs + ggml_nbytes(tensor) });
+ }
+
+ for (const auto & it : ctx_map) {
+ if (!it.second || it.second.get() == ctx) {
+ continue;
+ }
+
+ for (ggml_tensor * tensor = ggml_get_first_tensor(it.second.get()); tensor; tensor = ggml_get_next_tensor(it.second.get(), tensor)) {
+ const auto * weight = get_weight(ggml_get_name(tensor));
+ if (!weight || weight->idx != idx || own.count(weight)) {
+ continue;
+ }
+ other.push_back({ weight->offs, weight->offs + ggml_nbytes(tensor) });
+ }
}
+
+ return sub_ranges(merge_ranges(std::move(own_ranges)), merge_ranges(std::move(other)));
}
void llama_model_loader::unmap_weight(const llama_tensor_weight & w) const {
@@ -1492,7 +1552,7 @@ const void * llama_model_loader::load_data_range(const llama_tensor_weight & w,
bool llama_model_loader::load_all_data(
struct ggml_context * ctx,
- llama_buf_map & bufs,
+ std::vector<ggml_backend_buffer_t> & bufs,
llama_mlocks * lmlocks,
llama_progress_callback progress_callback,
void * progress_callback_user_data) {
@@ -1528,7 +1588,7 @@ bool llama_model_loader::load_all_data(
}
// When not using mmaped io use async uploads from pinned memory to GPU memory.
// First determine if the backend supports the necessary features for async uploads.
- auto * buf = bufs.count(0) ? bufs.at(0) : nullptr;
+ auto * buf = bufs.empty() ? nullptr : bufs[0];
if (!buf) {
LLAMA_LOG_DEBUG("%s: no buffer found for async uploads\n", func);
return nullptr;
@@ -1599,7 +1659,7 @@ bool llama_model_loader::load_all_data(
if (upload_backend) {
LLAMA_LOG_DEBUG("%s: using async uploads for device %s, buffer type %s, backend %s\n", __func__,
ggml_backend_dev_name(ggml_backend_get_device(upload_backend)),
- ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs.at(0))),
+ ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs[0])),
ggml_backend_name(upload_backend));
}
@@ -1640,11 +1700,17 @@ bool llama_model_loader::load_all_data(
if (from_mapping) {
const auto & mapping = mappings.at(weight->idx);
+ uint8_t * data = (uint8_t *) mapping->addr() + weight->offs;
+
+ // the tensors of a file can be spread across several buffers - find the one containing this tensor's data
ggml_backend_buffer_t buf_mmap = nullptr;
- if (bufs.count(weight->idx)) {
- buf_mmap = bufs.at(weight->idx);
+ for (const auto & buf : bufs) {
+ const uint8_t * base = (const uint8_t *) ggml_backend_buffer_get_base(buf);
+ if (base && base <= data && data + n_size <= base + ggml_backend_buffer_get_size(buf)) {
+ buf_mmap = buf;
+ break;
+ }
}
- uint8_t * data = (uint8_t *) mapping->addr() + weight->offs;
if (check_tensors) {
validation_result.emplace_back(std::async(std::launch::async, [cur, data, n_size] {
diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h
index 9e51d0ce75..ed59079a4e 100644
--- a/src/llama-model-loader.h
+++ b/src/llama-model-loader.h
@@ -16,8 +16,6 @@
#include <stdexcept>
#include <unordered_map>
-using llama_buf_map = std::unordered_map<uint32_t, ggml_backend_buffer_t>;
-
// lists of buffer types used for each layer
using buft_list_t = std::vector<std::pair<ggml_backend_dev_t, ggml_backend_buffer_type_t>>;
@@ -242,7 +240,16 @@ struct llama_model_loader {
void init_mappings(bool prefetch = true, llama_mlocks * mlock_mmaps = nullptr);
- void get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const;
+ // byte range within a mapped file
+ struct mapping_range {
+ size_t first; // inclusive
+ size_t last; // exclusive
+ };
+
+ // sorted, non-overlapping ranges of file 'idx' that back the tensors of 'ctx'
+ // ranges owned by other contexts are excluded, so that a buffer wrapping them won't pin
+ // data that this context never reads (lazy tensors, -cmoe tensors, ...)
+ std::vector<mapping_range> get_mapping_ranges(uint32_t idx, ggml_context * ctx) const;
// release a weight's mmap pages
void unmap_weight(const llama_tensor_weight & w) const;
@@ -254,7 +261,7 @@ struct llama_model_loader {
// Returns false if cancelled by progress_callback
bool load_all_data(
struct ggml_context * ctx,
- llama_buf_map & bufs,
+ std::vector<ggml_backend_buffer_t> & bufs,
llama_mlocks * lmlocks,
llama_progress_callback progress_callback,
void * progress_callback_user_data);
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 7072d029c9..aeab7b75b3 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1875,12 +1875,13 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
pimpl->mappings.reserve(ml.mappings.size());
// create the backend buffers
- std::vector<std::pair<ggml_context *, llama_buf_map>> ctx_buf_maps;
- ctx_buf_maps.reserve(ml.ctx_map.size());
+ // note: the buffers are owned by pimpl->ctxs_bufs, this is a raw-pointer view used to load the data
+ std::vector<std::pair<ggml_context *, std::vector<ggml_backend_buffer_t>>> ctx_bufs;
+ ctx_bufs.reserve(ml.ctx_map.size());
- // Ensure we have enough capacity for the maximum backend buffer we will potentially create
- const size_t n_max_backend_buffer = ml.ctx_map.size() * ml.files.size();
- pimpl->ctxs_bufs.reserve(n_max_backend_buffer);
+ // reserve hint: each context creates one buffer per file, or more when its tensors are spread
+ // across disjoint ranges of a file (see get_mapping_ranges)
+ pimpl->ctxs_bufs.reserve(ml.ctx_map.size() * ml.files.size());
for (auto & [ctx_key, ctx_ptr] : ml.ctx_map) {
ggml_backend_buffer_type_t buft = ctx_key.buft;
@@ -1891,9 +1892,6 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
continue;
}
- llama_buf_map buf_map;
- buf_map.reserve(n_max_backend_buffer);
-
// check if it is possible to use buffer_from_host_ptr with this buffer type
ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft);
if (!dev) {
@@ -1915,24 +1913,24 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
if ((ml.use_mmap || is_lazy_mapped) && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft) {
GGML_ASSERT(!ml.no_alloc);
+ const size_t max_size = ggml_get_max_tensor_size(ctx);
+
for (uint32_t idx = 0; idx < ml.files.size(); idx++) {
- // only the mmap region containing the tensors in the model is mapped to the backend buffer
- // this is important for metal with apple silicon: if the entire model could be mapped to a metal buffer,
- // then we could just use metal for all layers
- // this allows using partial offloading when the model size exceeds the metal buffer size, but not the RAM size
- void * addr = nullptr;
- size_t first, last; // NOLINT
- ml.get_mapping_range(&first, &last, &addr, idx, ctx);
- if (first >= last) {
- continue;
- }
- const size_t max_size = ggml_get_max_tensor_size(ctx);
- ggml_backend_buffer_t buf = ggml_backend_dev_buffer_from_host_ptr(dev, (char *) addr + first, last - first, max_size);
- if (buf == nullptr) {
- throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft)));
+ // map the mmap regions that contain the tensors of this context
+ // this is important for metal with apple silicon: a backend buffer pins everything it
+ // wraps, so the regions owned by the other contexts (lazy tensors read on demand
+ // from disk, MoE tensors kept on the CPU, etc) are left out and stay out of RAM
+ // this also allows using partial offloading when the model size exceeds the metal buffer
+ // size, but not the RAM size
+ void * addr = ml.mappings.at(idx)->addr();
+
+ for (const auto & range : ml.get_mapping_ranges(idx, ctx)) {
+ ggml_backend_buffer_t buf = ggml_backend_dev_buffer_from_host_ptr(dev, (char *) addr + range.first, range.last - range.first, max_size);
+ if (buf == nullptr) {
+ throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft)));
+ }
+ bufs.emplace_back(buf);
}
- bufs.emplace_back(buf);
- buf_map.emplace(idx, buf);
}
} else {
ggml_backend_buffer_t buf;
@@ -1954,20 +1952,22 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
mlock_buf->grow_to(ggml_backend_buffer_get_size(buf));
}
bufs.emplace_back(buf);
- for (uint32_t idx = 0; idx < ml.files.size(); idx++) {
- buf_map.emplace(idx, buf);
- }
}
+ std::vector<ggml_backend_buffer_t> buf_ptrs;
+ buf_ptrs.reserve(bufs.size());
+
for (auto & buf : bufs) {
// indicate that this buffer contains weights
// this is used by ggml_backend_sched to improve op scheduling: ops that use a weight are preferably scheduled to the backend that contains the weight
ggml_backend_buffer_set_usage(buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
+
+ buf_ptrs.emplace_back(buf.get());
}
pimpl->ctxs_bufs.emplace_back(std::move(ctx_ptr), std::move(bufs));
- ctx_buf_maps.emplace_back(ctx, buf_map);
+ ctx_bufs.emplace_back(ctx, std::move(buf_ptrs));
}
if (llama_supports_gpu_offload()) {
@@ -2000,15 +2000,15 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
// without mmap, load non-host buffers first: their tensors go through a staging buffer, which is cheapest while the fewest weights are resident
if (!ml.use_mmap) {
- std::stable_partition(ctx_buf_maps.begin(), ctx_buf_maps.end(), [](const auto & ctx_buf_map) {
- const auto & buf_map = ctx_buf_map.second;
- return !buf_map.empty() && !ggml_backend_buffer_is_host(buf_map.begin()->second);
+ std::stable_partition(ctx_bufs.begin(), ctx_bufs.end(), [](const auto & x) {
+ const auto & bufs = x.second;
+ return !bufs.empty() && !ggml_backend_buffer_is_host(bufs[0]);
});
}
// load tensor data
- for (auto & [ctx, buf_map] : ctx_buf_maps) {
- if (!ml.load_all_data(ctx, buf_map, use_mlock ? &pimpl->mlock_mmaps : NULL, params.progress_callback, params.progress_callback_user_data)) {
+ for (auto & [ctx, bufs] : ctx_bufs) {
+ if (!ml.load_all_data(ctx, bufs, use_mlock ? &pimpl->mlock_mmaps : NULL, params.progress_callback, params.progress_callback_user_data)) {
return false;
}
}