Skip to content

LLM inference in C/C++

C++ ggml

llama : split llama_prefetch() into merge and prefetch steps (#30532)

Adrien Gallouët68ac7657268d
diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
index 80a719670b..4d1cd31e98 100644
--- a/src/llama-mmap.cpp
+++ b/src/llama-mmap.cpp
@@ -865,23 +865,7 @@ void llama_prefetch(llama_memory_ranges mr) {
});
uintptr_t begin = 0, end = 0;
-#if defined(_WIN32)
- // collect the mr and prefetch them in one call, so the reads can be issued concurrently
- std::vector<WIN32_MEMORY_RANGE_ENTRY> entries;
- auto prefetch = [&]() {
- entries.push_back({ (PVOID) begin, (SIZE_T) (end - begin) });
- return true;
- };
-#else
- auto prefetch = [&]() {
- // TODO: in a follow-up, use posix_madvise(.., POSIX_MADV_WILLNEED) to match the rest of this file
- if (madvise((void *) begin, end - begin, MADV_WILLNEED) != 0) {
- LLAMA_LOG_WARN("llama_prefetch: madvise(MADV_WILLNEED) failed: %s\n", strerror(errno));
- return false;
- }
- return true;
- };
-#endif
+ size_t n_ranges = 0;
for (const auto & range : mr) {
if (!range.addr || range.size == 0) {
continue;
@@ -890,9 +874,7 @@ void llama_prefetch(llama_memory_ranges mr) {
const uintptr_t first = pointer / page * page;
const uintptr_t last = (pointer + range.size + page - 1) / page * page;
if (end && first > end) {
- if (!prefetch()) {
- return;
- }
+ mr[n_ranges++] = { (const void *) begin, end - begin };
end = 0;
}
if (!end) {
@@ -901,13 +883,28 @@ void llama_prefetch(llama_memory_ranges mr) {
end = std::max(end, last);
}
if (end) {
- prefetch();
+ mr[n_ranges++] = { (const void *) begin, end - begin };
}
+ mr.resize(n_ranges);
#if defined(_WIN32)
+ // prefetch the merged ranges in one call, so the reads can be issued concurrently
+ std::vector<WIN32_MEMORY_RANGE_ENTRY> entries;
+ entries.reserve(mr.size());
+ for (const auto & range : mr) {
+ entries.push_back({ const_cast<PVOID>(range.addr), (SIZE_T) range.size });
+ }
if (!entries.empty() && !pPrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) entries.size(), entries.data(), 0)) {
LLAMA_LOG_WARN("llama_prefetch: PrefetchVirtualMemory failed: %s\n",
llama_format_win_err(GetLastError()).c_str());
}
+#else
+ for (const auto & range : mr) {
+ // TODO: in a follow-up, use posix_madvise(.., POSIX_MADV_WILLNEED) to match the rest of this file
+ if (madvise(const_cast<void *>(range.addr), range.size, MADV_WILLNEED) != 0) {
+ LLAMA_LOG_WARN("llama_prefetch: madvise(MADV_WILLNEED) failed: %s\n", strerror(errno));
+ return;
+ }
+ }
#endif
#else
GGML_UNUSED(mr);