Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

llama.cpp: version bump b10729 (#18160)

Daniel Hiltgen5ec58043600a
diff --git a/LLAMA_CPP_VERSION b/LLAMA_CPP_VERSION
index b08661a43..9da8f62c7 100644
--- a/LLAMA_CPP_VERSION
+++ b/LLAMA_CPP_VERSION
@@ -1 +1 @@
-b10630
+b10729
diff --git a/llama/compat/001-llama-cpp-hooks.patch b/llama/compat/001-llama-cpp-hooks.patch
index 3ded2a5e5..661005209 100644
--- a/llama/compat/001-llama-cpp-hooks.patch
+++ b/llama/compat/001-llama-cpp-hooks.patch
@@ -1,5 +1,5 @@
diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp
-index df8313e81..29ccb0f85 100644
+index 7663797ba..4140399a5 100644
--- a/src/llama-model-loader.cpp
+++ b/src/llama-model-loader.cpp
@@ -5,6 +5,7 @@
@@ -10,7 +10,7 @@ index df8313e81..29ccb0f85 100644
#include <algorithm>
#include <array>
-@@ -561,6 +562,9 @@ llama_model_loader::llama_model_loader(
+@@ -574,6 +575,9 @@ llama_model_loader::llama_model_loader(
}
get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);
@@ -20,7 +20,7 @@ index df8313e81..29ccb0f85 100644
llm_kv = LLM_KV(llm_arch_from_string(arch_name));
files.emplace_back(new llama_file(fname.c_str(), "rb", use_direct_io));
-@@ -571,6 +575,9 @@ llama_model_loader::llama_model_loader(
+@@ -584,6 +588,9 @@ llama_model_loader::llama_model_loader(
// so we build a unified tensors index for weights.
for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
std::string tensor_name = std::string(cur->name);
@@ -30,7 +30,7 @@ index df8313e81..29ccb0f85 100644
// make sure there is no duplicated tensor names
if (weights_map.find(tensor_name) != weights_map.end()) {
throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));
-@@ -681,6 +688,9 @@ llama_model_loader::llama_model_loader(
+@@ -694,6 +701,9 @@ llama_model_loader::llama_model_loader(
// Save tensors data offset info of the main file.
for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
std::string tensor_name = std::string(cur->name);
@@ -40,24 +40,27 @@ index df8313e81..29ccb0f85 100644
// make sure there is no duplicated tensor names
if (weights_map.find(tensor_name) != weights_map.end()) {
throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));
-@@ -1380,6 +1390,7 @@ void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void *
+@@ -1464,6 +1474,10 @@ void llama_model_loader::unmap_weight(const llama_tensor_weight & w) const {
+ const void * llama_model_loader::load_data_range(const llama_tensor_weight & w, size_t offs, size_t size, void * buf) const {
+ GGML_ASSERT(offs + size <= ggml_nbytes(w.tensor));
- void llama_model_loader::load_data_for(struct ggml_tensor * cur) const {
- const auto & w = require_weight(ggml_get_name(cur));
-+ if (llama_ollama_compat::maybe_load_text_tensor(this, cur, w.offs)) return;
++ if (const void * compat_data = llama_ollama_compat::maybe_load_text_tensor_range(this, w.tensor, offs, size, buf)) {
++ return compat_data;
++ }
++
+ const void * data = buf;
if (use_mmap) {
- const auto & mapping = mappings.at(w.idx);
-@@ -1530,6 +1541,7 @@ bool llama_model_loader::load_all_data(
+@@ -1612,6 +1626,7 @@ bool llama_model_loader::load_all_data(
}
size_t n_size = ggml_nbytes(cur);
+ if (llama_ollama_compat::maybe_load_text_tensor(this, cur, weight->offs)) continue;
- if (use_mmap) {
- const auto & mapping = mappings.at(weight->idx);
+ const bool from_mapping = use_mmap || lazy.has(cur);
+
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
-index c1870813f..a1a923082 100644
+index 90de19575..5a8e4c675 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -10,6 +10,8 @@
@@ -69,7 +72,7 @@ index c1870813f..a1a923082 100644
#include <algorithm>
#include <cassert>
#include <cmath>
-@@ -1093,6 +1095,11 @@ struct clip_model_loader {
+@@ -1188,6 +1190,11 @@ struct clip_model_loader {
ctx_meta.reset(meta);
@@ -81,7 +84,7 @@ index c1870813f..a1a923082 100644
const int n_tensors = gguf_get_n_tensors(ctx_gguf.get());
// print gguf info
-@@ -3085,6 +3092,7 @@ struct clip_model_loader {
+@@ -3569,6 +3576,7 @@ struct clip_model_loader {
auto it_off = tensor_offset.find(t->name);
GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor");
const size_t offset = it_off->second;
@@ -89,7 +92,7 @@ index c1870813f..a1a923082 100644
fin.seekg(offset, std::ios::beg);
if (!fin) {
throw std::runtime_error(string_format("%s: failed to seek for tensor %s\n", __func__, t->name));
-@@ -4964,6 +4972,15 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
+@@ -5797,6 +5805,15 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
diff --git a/llama/compat/README.md b/llama/compat/README.md
index eafae60cc..caf3fbcfd 100644
--- a/llama/compat/README.md
+++ b/llama/compat/README.md
@@ -53,8 +53,15 @@ The layer runs at a small set of loader hook points:
vision, audio, MTP, or other tensors that the text loader should not claim.
3. Main model tensor reads: `maybe_load_text_tensor` applies registered
text-side load operations, such as FFN concat or dtype promotion, before
- the normal llama.cpp file read. This is wired into both full model loading
- and single-tensor reads used by tools such as `llama-quantize`.
+ the normal llama.cpp file read. This is wired into full model loading
+ (`load_all_data`) and single-tensor reads used by tools such as
+ `llama-quantize`. Since llama.cpp b10729 replaced the whole-tensor
+ `load_data_for` read with slabs via `load_data_range`, the quantize-style
+ slab path goes through `maybe_load_text_tensor_range`, which materializes
+ the op's full output for one active tensor at a time (single-slot cache —
+ evicted when the next tensor's first range arrives) and serves each
+ requested (offset, size) range from it, so quantize memory stays at one
+ op tensor, matching the whole-tensor `load_data_for` read it replaced.
4. `mtmd/clip` constructor: `translate_clip_metadata` rewrites a clip-facing
view of monolithic GGUFs into the mmproj form expected by llama.cpp.
5. `mtmd/clip` tensor load loop: `maybe_load_tensor` applies clip-side load
diff --git a/llama/compat/llama-ollama-compat.cpp b/llama/compat/llama-ollama-compat.cpp
index ce5abd59e..0888fef0e 100644
--- a/llama/compat/llama-ollama-compat.cpp
+++ b/llama/compat/llama-ollama-compat.cpp
@@ -3438,6 +3438,71 @@ bool maybe_load_text_tensor(const llama_model_loader * ml,
return load_tensor_with_op(cur, path.c_str(), buft, op);
}
+// Slab-read cache slot (maybe_load_text_tensor_range). Holds AT MOST ONE
+// materialized tensor per loader: quantize reads a tensor's slabs
+// contiguously, so the previous entry is evicted when the next tensor's
+// first range arrives. This keeps peak memory at one op tensor at a time —
+// the same profile as the whole-tensor read this hook replaced. Growing a
+// per-tensor map instead would accumulate every layer's output for per-layer
+// ops (gemma4 MoE gate/up, qwen3.5 norm-shift) — most of the model in RAM by
+// the end of a quantize run.
+std::mutex g_text_range_mutex;
+std::unordered_map<const llama_model_loader *,
+ std::pair<std::string, std::vector<uint8_t>>>
+ g_text_range_cache;
+
+const void * maybe_load_text_tensor_range(const llama_model_loader * ml,
+ ggml_tensor * cur,
+ size_t offs,
+ size_t size,
+ void * buf) {
+ if (compat_disabled()) return nullptr;
+ std::string path;
+ {
+ std::lock_guard<std::mutex> lk(g_loader_path_mutex);
+ auto it = g_loader_paths.find(ml);
+ if (it == g_loader_paths.end() || it->second.empty()) return nullptr;
+ path = it->second;
+ }
+
+ std::lock_guard<std::mutex> lk(g_text_range_mutex);
+ auto & slot = g_text_range_cache[ml];
+ auto cached = slot.first == ggml_get_name(cur) ? &slot.second : nullptr;
+ if (!cached) {
+ LoadOp op;
+ if (!take_load_op(ggml_get_name(cur), op)) return nullptr;
+
+ const auto start = std::chrono::steady_clock::now();
+ std::vector<uint8_t> full(ggml_nbytes(cur));
+ if (!op.apply(path.c_str(), full.data(), full.size())) {
+ OLLAMA_COMPAT_LOG_ERROR("%s: %s failed for %s after %.3f ms\n",
+ __func__, op.description, ggml_get_name(cur), elapsed_ms(start));
+ return nullptr;
+ }
+
+ const size_t dst_size = full.size();
+ slot = {std::string(ggml_get_name(cur)), std::move(full)};
+ cached = &slot.second;
+ const double ms = elapsed_ms(start);
+ const TransformTiming total = record_transform_timing(dst_size, ms);
+ OLLAMA_COMPAT_LOG_INFO("compat tensor transform: op=%s tensor=%s bytes=%zu duration_ms=%.3f total_ops=%llu total_bytes=%zu total_ms=%.3f\n",
+ op.description, ggml_get_name(cur), dst_size, ms,
+ (unsigned long long) total.count, total.bytes, total.ms);
+ }
+
+ if (offs + size > cached->size()) {
+ OLLAMA_COMPAT_LOG_ERROR("%s: range %zu+%zu out of bounds for %s (%zu bytes)\n",
+ __func__, offs, size, ggml_get_name(cur), cached->size());
+ return nullptr;
+ }
+
+ const void * out = buf ? buf : cached->data() + offs;
+ if (buf) {
+ std::memcpy(buf, cached->data() + offs, size);
+ }
+ return out;
+}
+
int maybe_clip_mmproj_embd(const char * projector_type, int projection_dim) {
if (compat_disabled() || projection_dim <= 0) return 0;
if (!clip_mmproj_embd_uses_projection_dim(projector_type)) return 0;
diff --git a/llama/compat/llama-ollama-compat.h b/llama/compat/llama-ollama-compat.h
index c97326df4..ba3f06096 100644
--- a/llama/compat/llama-ollama-compat.h
+++ b/llama/compat/llama-ollama-compat.h
@@ -79,6 +79,22 @@ bool maybe_load_text_tensor(const llama_model_loader * ml,
ggml_tensor * cur,
size_t file_offset);
+// Slab-serving variant for loaders that read tensor data in (offset, size)
+// byte ranges (llama_model_loader::load_data_range, used by llama-quantize
+// and other single-tensor read tools). Same registry and file-path lookup
+// as maybe_load_text_tensor, but serves one range of the op's destination
+// bytes per call: the first call for a tensor materializes the op's full
+// output, and later ranges are served from that cache. To keep quantize
+// memory at a single op tensor (the same profile as the whole-tensor read
+// this hook replaced), the cache holds one active tensor per loader.
+// Returns the requested range (copied into buf when buf is non-null) or
+// nullptr when no load op exists for this tensor.
+const void * maybe_load_text_tensor_range(const llama_model_loader * ml,
+ ggml_tensor * cur,
+ size_t offs,
+ size_t size,
+ void * buf);
+
// Called from clip_n_mmproj_embd() before the upstream switch. Returns a
// positive embedding size only for Ollama compatibility cases whose projector
// metadata already follows upstream naming, but whose legacy projector type
diff --git a/llama/server/CMakeLists.txt b/llama/server/CMakeLists.txt
index 7d88fcd7d..b994b14bd 100644
--- a/llama/server/CMakeLists.txt
+++ b/llama/server/CMakeLists.txt
@@ -187,6 +187,85 @@ FetchContent_Declare(
)
FetchContent_MakeAvailable(llama_cpp)
+if(NOT COMMAND license_add_file)
+ message(FATAL_ERROR "llama.cpp license registry (cmake/license.cmake) not found; check the llama.cpp update")
+endif()
+set(_llama_cpp_root "${llama_cpp_SOURCE_DIR}")
+
+file(GLOB_RECURSE _ollama_vendor_license_files
+ "${_llama_cpp_root}/vendor/*/LICENSE*")
+file(GLOB _llamacpp_repo_license_files
+ "${_llama_cpp_root}/licenses/LICENSE*")
+list(APPEND _ollama_vendor_license_files ${_llamacpp_repo_license_files})
+list(APPEND _ollama_vendor_license_files "${_llama_cpp_root}/LICENSE")
+foreach(_lic IN LISTS _ollama_vendor_license_files)
+ cmake_path(RELATIVE_PATH _lic BASE_DIRECTORY "${_llama_cpp_root}")
+ # upstream registers these itself; skip to avoid double entries
+ if(_lic STREQUAL "LICENSE" OR _lic STREQUAL "vendor/cpp-httplib/LICENSE")
+ continue()
+ endif()
+ set(_name "${_lic}")
+ if(_name MATCHES "^vendor/")
+ # registry name = component path without the vendor/ prefix
+ string(REGEX REPLACE "^vendor/" "" _name "${_name}")
+ string(REGEX REPLACE "/LICENSE[^/]*$" "" _name "${_name}")
+ endif()
+ string(REPLACE "/" "-" _name "${_name}")
+ license_add_file("${_name}" "${_llama_cpp_root}/${_lic}")
+endforeach()
+unset(_ollama_vendor_license_files)
+
+function(ollama_extract_vendor_license name src start_marker end_marker)
+ if(NOT EXISTS "${src}")
+ message(FATAL_ERROR
+ "llama.cpp vendored source missing: ${src}; check the llama.cpp update "
+ "and the embedded-license entry in llama/server/CMakeLists.txt")
+ endif()
+ file(READ "${src}" _txt)
+ string(FIND "${_txt}" "${start_marker}" _start)
+ if(_start EQUAL -1)
+ message(FATAL_ERROR
+ "llama.cpp embedded license anchor '${start_marker}' not found in "
+ "${src}; update llama/server/CMakeLists.txt")
+ endif()
+ string(FIND "${_txt}" "${end_marker}" _end)
+ if(_end EQUAL -1)
+ message(FATAL_ERROR
+ "llama.cpp embedded license end anchor '${end_marker}' not found in "
+ "${src}; update llama/server/CMakeLists.txt")
+ endif()
+ string(LENGTH "${end_marker}" _end_len)
+ math(EXPR _len "${_end} - ${_start} + ${_end_len}")
+ string(SUBSTRING "${_txt}" "${_start}" "${_len}" _lic)
+ set(_dst "${CMAKE_BINARY_DIR}/ollama-vendor-licenses/${name}.txt")
+ file(WRITE "${_dst}" "${_lic}")
+ license_add_file("${name}" "${_dst}")
+endfunction()
+
+ollama_extract_vendor_license(
+ "llamafile-sgemm"
+ "${_llama_cpp_root}/ggml/src/ggml-cpu/llamafile/sgemm.cpp"
+ "Copyright 2024 Mozilla Foundation"
+ "// SOFTWARE.")
+
+get_property(_reg_text GLOBAL PROPERTY LICENSE_TEXT)
+string(REPLACE "R\"=L=(" "" _reg_text "${_reg_text}")
+string(REPLACE ")=L=\"," "\n" _reg_text "${_reg_text}")
+set(_reg_ok TRUE)
+foreach(_anchor IN ITEMS "llama.cpp" "cpp-httplib" "llamafile-sgemm" "jsonhpp")
+ string(FIND "${_reg_text}" "${_anchor}" _pos)
+ if(_pos EQUAL -1)
+ set(_reg_ok FALSE)
+ endif()
+endforeach()
+string(LENGTH "${_reg_text}" _reg_len)
+if(NOT _reg_ok OR _reg_len LESS 256)
+ message(FATAL_ERROR "llama.cpp vendored license aggregate came out empty or incomplete; the registry mechanism changed upstream")
+endif()
+file(WRITE "${CMAKE_BINARY_DIR}/ollama-vendor-licenses/LLAMA_CPP_VENDORS_LICENSE" "${_reg_text}")
+unset(_reg_text)
+unset(_llama_cpp_root)
+
# Link the Ollama-compat source files into the fetched llama target.
# Kept separate from the hook patch so our .cpp/.h stay
# on-disk in llama/compat/ rather than being copied into _deps/.
@@ -539,6 +618,25 @@ else()
RUNTIME DESTINATION ${_base_dest} COMPONENT llama-server OPTIONAL
LIBRARY DESTINATION ${_base_dest} COMPONENT llama-server OPTIONAL)
+ get_target_property(_llama_cpp_license_source llama SOURCE_DIR)
+ if(NOT _llama_cpp_license_source OR _llama_cpp_license_source MATCHES "-NOTFOUND$")
+ message(FATAL_ERROR "llama.cpp source directory not found for license install")
+ endif()
+ get_filename_component(_llama_cpp_license_root "${_llama_cpp_license_source}/.." ABSOLUTE)
+ install(FILES
+ "${_llama_cpp_license_root}/LICENSE"
+ DESTINATION ${OLLAMA_LIB_DIR}
+ RENAME LLAMA_CPP_LICENSE
+ COMPONENT llama-server)
+ install(FILES
+ "${_llama_cpp_license_root}/vendor/cpp-httplib/LICENSE"
+ DESTINATION ${OLLAMA_LIB_DIR}
+ RENAME CPP_HTTPLIB_LICENSE
+ COMPONENT llama-server)
+ install(FILES
+ "${CMAKE_BINARY_DIR}/ollama-vendor-licenses/LLAMA_CPP_VENDORS_LICENSE"
+ DESTINATION ${OLLAMA_LIB_DIR}
+ COMPONENT llama-server)
# Bundle Windows CRT DLLs alongside the executables so zip installs
# do not depend on host-global redistributables.
ollama_install_windows_runtime_dlls("${_base_dest}")