LLM inference in C/C++
C++ ggml
llama : do not map foreign ranges into a backend buffer (#30405)
Georgi Gerganovb9f3599fb1c4
diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cppindex 045489df7a..80a719670b 100644--- a/src/llama-mmap.cpp+++ b/src/llama-mmap.cpp@@ -835,7 +835,7 @@ const bool llama_mlock::SUPPORTED = false;#endifvoid llama_prefetch(llama_memory_ranges mr) {-#if defined(__linux__) || (defined(_WIN32) && _WIN32_WINNT >= 0x602)+#if defined(__linux__) || defined(__APPLE__) || (defined(_WIN32) && _WIN32_WINNT >= 0x602)if (mr.empty()) {return;}@@ -874,6 +874,7 @@ void llama_prefetch(llama_memory_ranges mr) {};#elseauto prefetch = [&]() {+ // TODO: in a follow-up, use posix_madvise(.., POSIX_MADV_WILLNEED) to match the rest of this fileif (madvise((void *) begin, end - begin, MADV_WILLNEED) != 0) {LLAMA_LOG_WARN("llama_prefetch: madvise(MADV_WILLNEED) failed: %s\n", strerror(errno));return false;diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cppindex 6cc0b20288..bc93c6f9e6 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -13,6 +13,7 @@#include <cstring>#include <future>#include <regex>+#include <unordered_set>static const size_t kiB = 1024;static const size_t MiB = 1024*kiB;@@ -1446,21 +1447,80 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps}}-void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const {- GGML_ASSERT(!mappings.empty());- const auto & mapping = mappings.at(idx);+using mapping_ranges = std::vector<llama_model_loader::mapping_range>;++// merge a list of [first, last) ranges into a sorted, non-overlapping list+static mapping_ranges merge_ranges(mapping_ranges ranges) {+ std::sort(ranges.begin(), ranges.end(), [](const auto & a, const auto & b) { return a.first < b.first; });++ mapping_ranges res;+ for (const auto & range : ranges) {+ if (!res.empty() && range.first <= res.back().last) {+ res.back().last = std::max(res.back().last, range.last);+ } else {+ res.push_back(range);+ }+ }++ return res;+}++// subtract the sorted, non-overlapping ranges 'sub' from 'a'+static mapping_ranges sub_ranges(const mapping_ranges & a, const mapping_ranges & sub) {+ mapping_ranges res;++ for (const auto & x : a) {+ size_t cur = x.first;+ for (const auto & y : sub) {+ if (y.last <= cur || y.first >= x.last) {+ continue;+ }+ if (y.first > cur) {+ res.push_back({ cur, y.first });+ }+ cur = std::max(cur, y.last);+ }+ if (cur < x.last) {+ res.push_back({ cur, x.last });+ }+ }++ return res;+}++std::vector<llama_model_loader::mapping_range> llama_model_loader::get_mapping_ranges(uint32_t idx, ggml_context * ctx) const {+ GGML_ASSERT(idx < mappings.size());++ // data of this context, and data of the other contexts that share the same file+ // a weight that is mapped in several contexts (e.g. the token embedding) is not foreign data+ std::unordered_set<const llama_tensor_weight *> own;+ mapping_ranges own_ranges;+ mapping_ranges other;- *first = mapping->size();- *last = 0;- *addr = mapping->addr();for (ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor; tensor = ggml_get_next_tensor(ctx, tensor)) {const auto * weight = get_weight(ggml_get_name(tensor));if (!weight || weight->idx != idx) {continue;}- *first = std::min(*first, weight->offs);- *last = std::max(*last, weight->offs + ggml_nbytes(tensor));+ own.insert(weight);+ own_ranges.push_back({ weight->offs, weight->offs + ggml_nbytes(tensor) });+ }++ for (const auto & it : ctx_map) {+ if (!it.second || it.second.get() == ctx) {+ continue;+ }++ for (ggml_tensor * tensor = ggml_get_first_tensor(it.second.get()); tensor; tensor = ggml_get_next_tensor(it.second.get(), tensor)) {+ const auto * weight = get_weight(ggml_get_name(tensor));+ if (!weight || weight->idx != idx || own.count(weight)) {+ continue;+ }+ other.push_back({ weight->offs, weight->offs + ggml_nbytes(tensor) });+ }}++ return sub_ranges(merge_ranges(std::move(own_ranges)), merge_ranges(std::move(other)));}void llama_model_loader::unmap_weight(const llama_tensor_weight & w) const {@@ -1492,7 +1552,7 @@ const void * llama_model_loader::load_data_range(const llama_tensor_weight & w,bool llama_model_loader::load_all_data(struct ggml_context * ctx,- llama_buf_map & bufs,+ std::vector<ggml_backend_buffer_t> & bufs,llama_mlocks * lmlocks,llama_progress_callback progress_callback,void * progress_callback_user_data) {@@ -1528,7 +1588,7 @@ bool llama_model_loader::load_all_data(}// When not using mmaped io use async uploads from pinned memory to GPU memory.// First determine if the backend supports the necessary features for async uploads.- auto * buf = bufs.count(0) ? bufs.at(0) : nullptr;+ auto * buf = bufs.empty() ? nullptr : bufs[0];if (!buf) {LLAMA_LOG_DEBUG("%s: no buffer found for async uploads\n", func);return nullptr;@@ -1599,7 +1659,7 @@ bool llama_model_loader::load_all_data(if (upload_backend) {LLAMA_LOG_DEBUG("%s: using async uploads for device %s, buffer type %s, backend %s\n", __func__,ggml_backend_dev_name(ggml_backend_get_device(upload_backend)),- ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs.at(0))),+ ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs[0])),ggml_backend_name(upload_backend));}@@ -1640,11 +1700,17 @@ bool llama_model_loader::load_all_data(if (from_mapping) {const auto & mapping = mappings.at(weight->idx);+ uint8_t * data = (uint8_t *) mapping->addr() + weight->offs;++ // the tensors of a file can be spread across several buffers - find the one containing this tensor's dataggml_backend_buffer_t buf_mmap = nullptr;- if (bufs.count(weight->idx)) {- buf_mmap = bufs.at(weight->idx);+ for (const auto & buf : bufs) {+ const uint8_t * base = (const uint8_t *) ggml_backend_buffer_get_base(buf);+ if (base && base <= data && data + n_size <= base + ggml_backend_buffer_get_size(buf)) {+ buf_mmap = buf;+ break;+ }}- uint8_t * data = (uint8_t *) mapping->addr() + weight->offs;if (check_tensors) {validation_result.emplace_back(std::async(std::launch::async, [cur, data, n_size] {diff --git a/src/llama-model-loader.h b/src/llama-model-loader.hindex 9e51d0ce75..ed59079a4e 100644--- a/src/llama-model-loader.h+++ b/src/llama-model-loader.h@@ -16,8 +16,6 @@#include <stdexcept>#include <unordered_map>-using llama_buf_map = std::unordered_map<uint32_t, ggml_backend_buffer_t>;-// lists of buffer types used for each layerusing buft_list_t = std::vector<std::pair<ggml_backend_dev_t, ggml_backend_buffer_type_t>>;@@ -242,7 +240,16 @@ struct llama_model_loader {void init_mappings(bool prefetch = true, llama_mlocks * mlock_mmaps = nullptr);- void get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const;+ // byte range within a mapped file+ struct mapping_range {+ size_t first; // inclusive+ size_t last; // exclusive+ };++ // sorted, non-overlapping ranges of file 'idx' that back the tensors of 'ctx'+ // ranges owned by other contexts are excluded, so that a buffer wrapping them won't pin+ // data that this context never reads (lazy tensors, -cmoe tensors, ...)+ std::vector<mapping_range> get_mapping_ranges(uint32_t idx, ggml_context * ctx) const;// release a weight's mmap pagesvoid unmap_weight(const llama_tensor_weight & w) const;@@ -254,7 +261,7 @@ struct llama_model_loader {// Returns false if cancelled by progress_callbackbool load_all_data(struct ggml_context * ctx,- llama_buf_map & bufs,+ std::vector<ggml_backend_buffer_t> & bufs,llama_mlocks * lmlocks,llama_progress_callback progress_callback,void * progress_callback_user_data);diff --git a/src/llama-model.cpp b/src/llama-model.cppindex 7072d029c9..aeab7b75b3 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -1875,12 +1875,13 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {pimpl->mappings.reserve(ml.mappings.size());// create the backend buffers- std::vector<std::pair<ggml_context *, llama_buf_map>> ctx_buf_maps;- ctx_buf_maps.reserve(ml.ctx_map.size());+ // note: the buffers are owned by pimpl->ctxs_bufs, this is a raw-pointer view used to load the data+ std::vector<std::pair<ggml_context *, std::vector<ggml_backend_buffer_t>>> ctx_bufs;+ ctx_bufs.reserve(ml.ctx_map.size());- // Ensure we have enough capacity for the maximum backend buffer we will potentially create- const size_t n_max_backend_buffer = ml.ctx_map.size() * ml.files.size();- pimpl->ctxs_bufs.reserve(n_max_backend_buffer);+ // reserve hint: each context creates one buffer per file, or more when its tensors are spread+ // across disjoint ranges of a file (see get_mapping_ranges)+ pimpl->ctxs_bufs.reserve(ml.ctx_map.size() * ml.files.size());for (auto & [ctx_key, ctx_ptr] : ml.ctx_map) {ggml_backend_buffer_type_t buft = ctx_key.buft;@@ -1891,9 +1892,6 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {continue;}- llama_buf_map buf_map;- buf_map.reserve(n_max_backend_buffer);-// check if it is possible to use buffer_from_host_ptr with this buffer typeggml_backend_dev_t dev = ggml_backend_buft_get_device(buft);if (!dev) {@@ -1915,24 +1913,24 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {if ((ml.use_mmap || is_lazy_mapped) && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft) {GGML_ASSERT(!ml.no_alloc);+ const size_t max_size = ggml_get_max_tensor_size(ctx);+for (uint32_t idx = 0; idx < ml.files.size(); idx++) {- // only the mmap region containing the tensors in the model is mapped to the backend buffer- // this is important for metal with apple silicon: if the entire model could be mapped to a metal buffer,- // then we could just use metal for all layers- // this allows using partial offloading when the model size exceeds the metal buffer size, but not the RAM size- void * addr = nullptr;- size_t first, last; // NOLINT- ml.get_mapping_range(&first, &last, &addr, idx, ctx);- if (first >= last) {- continue;- }- const size_t max_size = ggml_get_max_tensor_size(ctx);- ggml_backend_buffer_t buf = ggml_backend_dev_buffer_from_host_ptr(dev, (char *) addr + first, last - first, max_size);- if (buf == nullptr) {- throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft)));+ // map the mmap regions that contain the tensors of this context+ // this is important for metal with apple silicon: a backend buffer pins everything it+ // wraps, so the regions owned by the other contexts (lazy tensors read on demand+ // from disk, MoE tensors kept on the CPU, etc) are left out and stay out of RAM+ // this also allows using partial offloading when the model size exceeds the metal buffer+ // size, but not the RAM size+ void * addr = ml.mappings.at(idx)->addr();++ for (const auto & range : ml.get_mapping_ranges(idx, ctx)) {+ ggml_backend_buffer_t buf = ggml_backend_dev_buffer_from_host_ptr(dev, (char *) addr + range.first, range.last - range.first, max_size);+ if (buf == nullptr) {+ throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft)));+ }+ bufs.emplace_back(buf);}- bufs.emplace_back(buf);- buf_map.emplace(idx, buf);}} else {ggml_backend_buffer_t buf;@@ -1954,20 +1952,22 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {mlock_buf->grow_to(ggml_backend_buffer_get_size(buf));}bufs.emplace_back(buf);- for (uint32_t idx = 0; idx < ml.files.size(); idx++) {- buf_map.emplace(idx, buf);- }}+ std::vector<ggml_backend_buffer_t> buf_ptrs;+ buf_ptrs.reserve(bufs.size());+for (auto & buf : bufs) {// indicate that this buffer contains weights// this is used by ggml_backend_sched to improve op scheduling: ops that use a weight are preferably scheduled to the backend that contains the weightggml_backend_buffer_set_usage(buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);++ buf_ptrs.emplace_back(buf.get());}pimpl->ctxs_bufs.emplace_back(std::move(ctx_ptr), std::move(bufs));- ctx_buf_maps.emplace_back(ctx, buf_map);+ ctx_bufs.emplace_back(ctx, std::move(buf_ptrs));}if (llama_supports_gpu_offload()) {@@ -2000,15 +2000,15 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {// without mmap, load non-host buffers first: their tensors go through a staging buffer, which is cheapest while the fewest weights are residentif (!ml.use_mmap) {- std::stable_partition(ctx_buf_maps.begin(), ctx_buf_maps.end(), [](const auto & ctx_buf_map) {- const auto & buf_map = ctx_buf_map.second;- return !buf_map.empty() && !ggml_backend_buffer_is_host(buf_map.begin()->second);+ std::stable_partition(ctx_bufs.begin(), ctx_bufs.end(), [](const auto & x) {+ const auto & bufs = x.second;+ return !bufs.empty() && !ggml_backend_buffer_is_host(bufs[0]);});}// load tensor data- for (auto & [ctx, buf_map] : ctx_buf_maps) {- if (!ml.load_all_data(ctx, buf_map, use_mlock ? &pimpl->mlock_mmaps : NULL, params.progress_callback, params.progress_callback_user_data)) {+ for (auto & [ctx, bufs] : ctx_bufs) {+ if (!ml.load_all_data(ctx, bufs, use_mlock ? &pimpl->mlock_mmaps : NULL, params.progress_callback, params.progress_callback_user_data)) {return false;}}