Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
llama.cpp: version bump b10729 (#18160)
Daniel Hiltgen5ec58043600a
diff --git a/LLAMA_CPP_VERSION b/LLAMA_CPP_VERSIONindex b08661a43..9da8f62c7 100644--- a/LLAMA_CPP_VERSION+++ b/LLAMA_CPP_VERSION@@ -1 +1 @@-b10630+b10729diff --git a/llama/compat/001-llama-cpp-hooks.patch b/llama/compat/001-llama-cpp-hooks.patchindex 3ded2a5e5..661005209 100644--- a/llama/compat/001-llama-cpp-hooks.patch+++ b/llama/compat/001-llama-cpp-hooks.patch@@ -1,5 +1,5 @@diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp-index df8313e81..29ccb0f85 100644+index 7663797ba..4140399a5 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -5,6 +5,7 @@@@ -10,7 +10,7 @@ index df8313e81..29ccb0f85 100644#include <algorithm>#include <array>-@@ -561,6 +562,9 @@ llama_model_loader::llama_model_loader(+@@ -574,6 +575,9 @@ llama_model_loader::llama_model_loader(}get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);@@ -20,7 +20,7 @@ index df8313e81..29ccb0f85 100644llm_kv = LLM_KV(llm_arch_from_string(arch_name));files.emplace_back(new llama_file(fname.c_str(), "rb", use_direct_io));-@@ -571,6 +575,9 @@ llama_model_loader::llama_model_loader(+@@ -584,6 +588,9 @@ llama_model_loader::llama_model_loader(// so we build a unified tensors index for weights.for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {std::string tensor_name = std::string(cur->name);@@ -30,7 +30,7 @@ index df8313e81..29ccb0f85 100644// make sure there is no duplicated tensor namesif (weights_map.find(tensor_name) != weights_map.end()) {throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));-@@ -681,6 +688,9 @@ llama_model_loader::llama_model_loader(+@@ -694,6 +701,9 @@ llama_model_loader::llama_model_loader(// Save tensors data offset info of the main file.for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {std::string tensor_name = std::string(cur->name);@@ -40,24 +40,27 @@ index df8313e81..29ccb0f85 100644// make sure there is no duplicated tensor namesif (weights_map.find(tensor_name) != weights_map.end()) {throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));-@@ -1380,6 +1390,7 @@ void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void *+@@ -1464,6 +1474,10 @@ void llama_model_loader::unmap_weight(const llama_tensor_weight & w) const {+ const void * llama_model_loader::load_data_range(const llama_tensor_weight & w, size_t offs, size_t size, void * buf) const {+ GGML_ASSERT(offs + size <= ggml_nbytes(w.tensor));- void llama_model_loader::load_data_for(struct ggml_tensor * cur) const {- const auto & w = require_weight(ggml_get_name(cur));-+ if (llama_ollama_compat::maybe_load_text_tensor(this, cur, w.offs)) return;++ if (const void * compat_data = llama_ollama_compat::maybe_load_text_tensor_range(this, w.tensor, offs, size, buf)) {++ return compat_data;++ }+++ const void * data = buf;if (use_mmap) {- const auto & mapping = mappings.at(w.idx);-@@ -1530,6 +1541,7 @@ bool llama_model_loader::load_all_data(+@@ -1612,6 +1626,7 @@ bool llama_model_loader::load_all_data(}size_t n_size = ggml_nbytes(cur);+ if (llama_ollama_compat::maybe_load_text_tensor(this, cur, weight->offs)) continue;- if (use_mmap) {- const auto & mapping = mappings.at(weight->idx);+ const bool from_mapping = use_mmap || lazy.has(cur);+diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp-index c1870813f..a1a923082 100644+index 90de19575..5a8e4c675 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -10,6 +10,8 @@@@ -69,7 +72,7 @@ index c1870813f..a1a923082 100644#include <algorithm>#include <cassert>#include <cmath>-@@ -1093,6 +1095,11 @@ struct clip_model_loader {+@@ -1188,6 +1190,11 @@ struct clip_model_loader {ctx_meta.reset(meta);@@ -81,7 +84,7 @@ index c1870813f..a1a923082 100644const int n_tensors = gguf_get_n_tensors(ctx_gguf.get());// print gguf info-@@ -3085,6 +3092,7 @@ struct clip_model_loader {+@@ -3569,6 +3576,7 @@ struct clip_model_loader {auto it_off = tensor_offset.find(t->name);GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor");const size_t offset = it_off->second;@@ -89,7 +92,7 @@ index c1870813f..a1a923082 100644fin.seekg(offset, std::ios::beg);if (!fin) {throw std::runtime_error(string_format("%s: failed to seek for tensor %s\n", __func__, t->name));-@@ -4964,6 +4972,15 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32+@@ -5797,6 +5805,15 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {}int clip_n_mmproj_embd(const struct clip_ctx * ctx) {diff --git a/llama/compat/README.md b/llama/compat/README.mdindex eafae60cc..caf3fbcfd 100644--- a/llama/compat/README.md+++ b/llama/compat/README.md@@ -53,8 +53,15 @@ The layer runs at a small set of loader hook points:vision, audio, MTP, or other tensors that the text loader should not claim.3. Main model tensor reads: `maybe_load_text_tensor` applies registeredtext-side load operations, such as FFN concat or dtype promotion, before- the normal llama.cpp file read. This is wired into both full model loading- and single-tensor reads used by tools such as `llama-quantize`.+ the normal llama.cpp file read. This is wired into full model loading+ (`load_all_data`) and single-tensor reads used by tools such as+ `llama-quantize`. Since llama.cpp b10729 replaced the whole-tensor+ `load_data_for` read with slabs via `load_data_range`, the quantize-style+ slab path goes through `maybe_load_text_tensor_range`, which materializes+ the op's full output for one active tensor at a time (single-slot cache —+ evicted when the next tensor's first range arrives) and serves each+ requested (offset, size) range from it, so quantize memory stays at one+ op tensor, matching the whole-tensor `load_data_for` read it replaced.4. `mtmd/clip` constructor: `translate_clip_metadata` rewrites a clip-facingview of monolithic GGUFs into the mmproj form expected by llama.cpp.5. `mtmd/clip` tensor load loop: `maybe_load_tensor` applies clip-side loaddiff --git a/llama/compat/llama-ollama-compat.cpp b/llama/compat/llama-ollama-compat.cppindex ce5abd59e..0888fef0e 100644--- a/llama/compat/llama-ollama-compat.cpp+++ b/llama/compat/llama-ollama-compat.cpp@@ -3438,6 +3438,71 @@ bool maybe_load_text_tensor(const llama_model_loader * ml,return load_tensor_with_op(cur, path.c_str(), buft, op);}+// Slab-read cache slot (maybe_load_text_tensor_range). Holds AT MOST ONE+// materialized tensor per loader: quantize reads a tensor's slabs+// contiguously, so the previous entry is evicted when the next tensor's+// first range arrives. This keeps peak memory at one op tensor at a time —+// the same profile as the whole-tensor read this hook replaced. Growing a+// per-tensor map instead would accumulate every layer's output for per-layer+// ops (gemma4 MoE gate/up, qwen3.5 norm-shift) — most of the model in RAM by+// the end of a quantize run.+std::mutex g_text_range_mutex;+std::unordered_map<const llama_model_loader *,+ std::pair<std::string, std::vector<uint8_t>>>+ g_text_range_cache;++const void * maybe_load_text_tensor_range(const llama_model_loader * ml,+ ggml_tensor * cur,+ size_t offs,+ size_t size,+ void * buf) {+ if (compat_disabled()) return nullptr;+ std::string path;+ {+ std::lock_guard<std::mutex> lk(g_loader_path_mutex);+ auto it = g_loader_paths.find(ml);+ if (it == g_loader_paths.end() || it->second.empty()) return nullptr;+ path = it->second;+ }++ std::lock_guard<std::mutex> lk(g_text_range_mutex);+ auto & slot = g_text_range_cache[ml];+ auto cached = slot.first == ggml_get_name(cur) ? &slot.second : nullptr;+ if (!cached) {+ LoadOp op;+ if (!take_load_op(ggml_get_name(cur), op)) return nullptr;++ const auto start = std::chrono::steady_clock::now();+ std::vector<uint8_t> full(ggml_nbytes(cur));+ if (!op.apply(path.c_str(), full.data(), full.size())) {+ OLLAMA_COMPAT_LOG_ERROR("%s: %s failed for %s after %.3f ms\n",+ __func__, op.description, ggml_get_name(cur), elapsed_ms(start));+ return nullptr;+ }++ const size_t dst_size = full.size();+ slot = {std::string(ggml_get_name(cur)), std::move(full)};+ cached = &slot.second;+ const double ms = elapsed_ms(start);+ const TransformTiming total = record_transform_timing(dst_size, ms);+ OLLAMA_COMPAT_LOG_INFO("compat tensor transform: op=%s tensor=%s bytes=%zu duration_ms=%.3f total_ops=%llu total_bytes=%zu total_ms=%.3f\n",+ op.description, ggml_get_name(cur), dst_size, ms,+ (unsigned long long) total.count, total.bytes, total.ms);+ }++ if (offs + size > cached->size()) {+ OLLAMA_COMPAT_LOG_ERROR("%s: range %zu+%zu out of bounds for %s (%zu bytes)\n",+ __func__, offs, size, ggml_get_name(cur), cached->size());+ return nullptr;+ }++ const void * out = buf ? buf : cached->data() + offs;+ if (buf) {+ std::memcpy(buf, cached->data() + offs, size);+ }+ return out;+}+int maybe_clip_mmproj_embd(const char * projector_type, int projection_dim) {if (compat_disabled() || projection_dim <= 0) return 0;if (!clip_mmproj_embd_uses_projection_dim(projector_type)) return 0;diff --git a/llama/compat/llama-ollama-compat.h b/llama/compat/llama-ollama-compat.hindex c97326df4..ba3f06096 100644--- a/llama/compat/llama-ollama-compat.h+++ b/llama/compat/llama-ollama-compat.h@@ -79,6 +79,22 @@ bool maybe_load_text_tensor(const llama_model_loader * ml,ggml_tensor * cur,size_t file_offset);+// Slab-serving variant for loaders that read tensor data in (offset, size)+// byte ranges (llama_model_loader::load_data_range, used by llama-quantize+// and other single-tensor read tools). Same registry and file-path lookup+// as maybe_load_text_tensor, but serves one range of the op's destination+// bytes per call: the first call for a tensor materializes the op's full+// output, and later ranges are served from that cache. To keep quantize+// memory at a single op tensor (the same profile as the whole-tensor read+// this hook replaced), the cache holds one active tensor per loader.+// Returns the requested range (copied into buf when buf is non-null) or+// nullptr when no load op exists for this tensor.+const void * maybe_load_text_tensor_range(const llama_model_loader * ml,+ ggml_tensor * cur,+ size_t offs,+ size_t size,+ void * buf);+// Called from clip_n_mmproj_embd() before the upstream switch. Returns a// positive embedding size only for Ollama compatibility cases whose projector// metadata already follows upstream naming, but whose legacy projector typediff --git a/llama/server/CMakeLists.txt b/llama/server/CMakeLists.txtindex 7d88fcd7d..b994b14bd 100644--- a/llama/server/CMakeLists.txt+++ b/llama/server/CMakeLists.txt@@ -187,6 +187,85 @@ FetchContent_Declare()FetchContent_MakeAvailable(llama_cpp)+if(NOT COMMAND license_add_file)+ message(FATAL_ERROR "llama.cpp license registry (cmake/license.cmake) not found; check the llama.cpp update")+endif()+set(_llama_cpp_root "${llama_cpp_SOURCE_DIR}")++file(GLOB_RECURSE _ollama_vendor_license_files+ "${_llama_cpp_root}/vendor/*/LICENSE*")+file(GLOB _llamacpp_repo_license_files+ "${_llama_cpp_root}/licenses/LICENSE*")+list(APPEND _ollama_vendor_license_files ${_llamacpp_repo_license_files})+list(APPEND _ollama_vendor_license_files "${_llama_cpp_root}/LICENSE")+foreach(_lic IN LISTS _ollama_vendor_license_files)+ cmake_path(RELATIVE_PATH _lic BASE_DIRECTORY "${_llama_cpp_root}")+ # upstream registers these itself; skip to avoid double entries+ if(_lic STREQUAL "LICENSE" OR _lic STREQUAL "vendor/cpp-httplib/LICENSE")+ continue()+ endif()+ set(_name "${_lic}")+ if(_name MATCHES "^vendor/")+ # registry name = component path without the vendor/ prefix+ string(REGEX REPLACE "^vendor/" "" _name "${_name}")+ string(REGEX REPLACE "/LICENSE[^/]*$" "" _name "${_name}")+ endif()+ string(REPLACE "/" "-" _name "${_name}")+ license_add_file("${_name}" "${_llama_cpp_root}/${_lic}")+endforeach()+unset(_ollama_vendor_license_files)++function(ollama_extract_vendor_license name src start_marker end_marker)+ if(NOT EXISTS "${src}")+ message(FATAL_ERROR+ "llama.cpp vendored source missing: ${src}; check the llama.cpp update "+ "and the embedded-license entry in llama/server/CMakeLists.txt")+ endif()+ file(READ "${src}" _txt)+ string(FIND "${_txt}" "${start_marker}" _start)+ if(_start EQUAL -1)+ message(FATAL_ERROR+ "llama.cpp embedded license anchor '${start_marker}' not found in "+ "${src}; update llama/server/CMakeLists.txt")+ endif()+ string(FIND "${_txt}" "${end_marker}" _end)+ if(_end EQUAL -1)+ message(FATAL_ERROR+ "llama.cpp embedded license end anchor '${end_marker}' not found in "+ "${src}; update llama/server/CMakeLists.txt")+ endif()+ string(LENGTH "${end_marker}" _end_len)+ math(EXPR _len "${_end} - ${_start} + ${_end_len}")+ string(SUBSTRING "${_txt}" "${_start}" "${_len}" _lic)+ set(_dst "${CMAKE_BINARY_DIR}/ollama-vendor-licenses/${name}.txt")+ file(WRITE "${_dst}" "${_lic}")+ license_add_file("${name}" "${_dst}")+endfunction()++ollama_extract_vendor_license(+ "llamafile-sgemm"+ "${_llama_cpp_root}/ggml/src/ggml-cpu/llamafile/sgemm.cpp"+ "Copyright 2024 Mozilla Foundation"+ "// SOFTWARE.")++get_property(_reg_text GLOBAL PROPERTY LICENSE_TEXT)+string(REPLACE "R\"=L=(" "" _reg_text "${_reg_text}")+string(REPLACE ")=L=\"," "\n" _reg_text "${_reg_text}")+set(_reg_ok TRUE)+foreach(_anchor IN ITEMS "llama.cpp" "cpp-httplib" "llamafile-sgemm" "jsonhpp")+ string(FIND "${_reg_text}" "${_anchor}" _pos)+ if(_pos EQUAL -1)+ set(_reg_ok FALSE)+ endif()+endforeach()+string(LENGTH "${_reg_text}" _reg_len)+if(NOT _reg_ok OR _reg_len LESS 256)+ message(FATAL_ERROR "llama.cpp vendored license aggregate came out empty or incomplete; the registry mechanism changed upstream")+endif()+file(WRITE "${CMAKE_BINARY_DIR}/ollama-vendor-licenses/LLAMA_CPP_VENDORS_LICENSE" "${_reg_text}")+unset(_reg_text)+unset(_llama_cpp_root)+# Link the Ollama-compat source files into the fetched llama target.# Kept separate from the hook patch so our .cpp/.h stay# on-disk in llama/compat/ rather than being copied into _deps/.@@ -539,6 +618,25 @@ else()RUNTIME DESTINATION ${_base_dest} COMPONENT llama-server OPTIONALLIBRARY DESTINATION ${_base_dest} COMPONENT llama-server OPTIONAL)+ get_target_property(_llama_cpp_license_source llama SOURCE_DIR)+ if(NOT _llama_cpp_license_source OR _llama_cpp_license_source MATCHES "-NOTFOUND$")+ message(FATAL_ERROR "llama.cpp source directory not found for license install")+ endif()+ get_filename_component(_llama_cpp_license_root "${_llama_cpp_license_source}/.." ABSOLUTE)+ install(FILES+ "${_llama_cpp_license_root}/LICENSE"+ DESTINATION ${OLLAMA_LIB_DIR}+ RENAME LLAMA_CPP_LICENSE+ COMPONENT llama-server)+ install(FILES+ "${_llama_cpp_license_root}/vendor/cpp-httplib/LICENSE"+ DESTINATION ${OLLAMA_LIB_DIR}+ RENAME CPP_HTTPLIB_LICENSE+ COMPONENT llama-server)+ install(FILES+ "${CMAKE_BINARY_DIR}/ollama-vendor-licenses/LLAMA_CPP_VENDORS_LICENSE"+ DESTINATION ${OLLAMA_LIB_DIR}+ COMPONENT llama-server)# Bundle Windows CRT DLLs alongside the executables so zip installs# do not depend on host-global redistributables.ollama_install_windows_runtime_dlls("${_base_dest}")