Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
llama.cpp update (#17545)
Daniel Hiltgenc82ebbd5bfb9
diff --git a/LLAMA_CPP_VERSION b/LLAMA_CPP_VERSIONindex 1c1f72d52..667a5805a 100644--- a/LLAMA_CPP_VERSION+++ b/LLAMA_CPP_VERSION@@ -1 +1 @@-b10091+b10242diff --git a/llama/compat/001-llama-cpp-hooks.patch b/llama/compat/001-llama-cpp-hooks.patchindex 5fc5b15d6..3ded2a5e5 100644--- a/llama/compat/001-llama-cpp-hooks.patch+++ b/llama/compat/001-llama-cpp-hooks.patch@@ -1,24 +1,26 @@diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp-index 474cabdfc..2bfe43a28 100644+index df8313e81..29ccb0f85 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp-@@ -4,6 +4,7 @@- #include "ggml.h"+@@ -5,6 +5,7 @@#include "gguf.h"#include "llama-hparams.h"+ #include "llama.h"+#include "llama-ollama-compat.h"#include <algorithm>#include <array>-@@ -551,6 +552,7 @@ llama_model_loader::llama_model_loader(+@@ -561,6 +562,9 @@ llama_model_loader::llama_model_loader(}get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);-+ if (llama_ollama_compat::translate_metadata(this, metadata, ctx, arch_name, fname.c_str())) use_mmap = false;++ if (llama_ollama_compat::translate_metadata(this, metadata, ctx, arch_name, fname.c_str())) {++ this->use_mmap = false;++ }llm_kv = LLM_KV(llm_arch_from_string(arch_name));files.emplace_back(new llama_file(fname.c_str(), "rb", use_direct_io));-@@ -575,6 +577,9 @@ llama_model_loader::llama_model_loader(+@@ -571,6 +575,9 @@ llama_model_loader::llama_model_loader(// so we build a unified tensors index for weights.for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {std::string tensor_name = std::string(cur->name);@@ -28,7 +30,7 @@ index 474cabdfc..2bfe43a28 100644// make sure there is no duplicated tensor namesif (weights_map.find(tensor_name) != weights_map.end()) {throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));-@@ -685,6 +690,9 @@ llama_model_loader::llama_model_loader(+@@ -681,6 +688,9 @@ llama_model_loader::llama_model_loader(// Save tensors data offset info of the main file.for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {std::string tensor_name = std::string(cur->name);@@ -38,7 +40,7 @@ index 474cabdfc..2bfe43a28 100644// make sure there is no duplicated tensor namesif (weights_map.find(tensor_name) != weights_map.end()) {throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));-@@ -1384,6 +1392,7 @@ void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void *+@@ -1380,6 +1390,7 @@ void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void *void llama_model_loader::load_data_for(struct ggml_tensor * cur) const {const auto & w = require_weight(ggml_get_name(cur));@@ -46,7 +48,7 @@ index 474cabdfc..2bfe43a28 100644if (use_mmap) {const auto & mapping = mappings.at(w.idx);-@@ -1534,6 +1543,7 @@ bool llama_model_loader::load_all_data(+@@ -1530,6 +1541,7 @@ bool llama_model_loader::load_all_data(}size_t n_size = ggml_nbytes(cur);@@ -55,7 +57,7 @@ index 474cabdfc..2bfe43a28 100644if (use_mmap) {const auto & mapping = mappings.at(weight->idx);diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp-index 7bd486030..6c3b23da0 100644+index c1870813f..a1a923082 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -10,6 +10,8 @@@@ -67,7 +69,7 @@ index 7bd486030..6c3b23da0 100644#include <algorithm>#include <cassert>#include <cmath>-@@ -1070,6 +1072,11 @@ struct clip_model_loader {+@@ -1093,6 +1095,11 @@ struct clip_model_loader {ctx_meta.reset(meta);@@ -79,7 +81,7 @@ index 7bd486030..6c3b23da0 100644const int n_tensors = gguf_get_n_tensors(ctx_gguf.get());// print gguf info-@@ -2822,6 +2829,7 @@ struct clip_model_loader {+@@ -3085,6 +3092,7 @@ struct clip_model_loader {auto it_off = tensor_offset.find(t->name);GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor");const size_t offset = it_off->second;@@ -87,7 +89,7 @@ index 7bd486030..6c3b23da0 100644fin.seekg(offset, std::ios::beg);if (!fin) {throw std::runtime_error(string_format("%s: failed to seek for tensor %s\n", __func__, t->name));-@@ -4489,6 +4497,15 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32+@@ -4964,6 +4972,15 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32}int clip_n_mmproj_embd(const struct clip_ctx * ctx) {diff --git a/llama/server/CMakeLists.txt b/llama/server/CMakeLists.txtindex bec100df2..7d88fcd7d 100644--- a/llama/server/CMakeLists.txt+++ b/llama/server/CMakeLists.txt@@ -176,6 +176,7 @@ set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)set(LLAMA_TOOLS_INSTALL OFF CACHE BOOL "" FORCE)set(LLAMA_CURL OFF CACHE BOOL "" FORCE)set(LLAMA_OPENSSL OFF CACHE BOOL "" FORCE)+set(LLAMA_SUBPROCESS OFF CACHE BOOL "" FORCE)FetchContent_Declare(llama_cppdiff --git a/llm/llama_server.go b/llm/llama_server.goindex e529cfc13..c99c29556 100644--- a/llm/llama_server.go+++ b/llm/llama_server.go@@ -385,19 +385,7 @@ func startLlamaServer(launch llamaServerLaunchConfig, out io.Writer) (cmd *exec.params = append(params, "--lora", adapter)}- // UseMmap- if launch.opts.UseMMap != nil && !*launch.opts.UseMMap {- params = append(params, "--no-mmap")- }-- // Direct I/O skips the page cache on load for integrated CUDA/ROCm GPUs, which- // share system memory with the CPU and would otherwise double-buffer weights.- for _, g := range launch.gpus {- if runtime.GOOS == "linux" && g.Integrated && (strings.EqualFold(g.Library, "CUDA") || strings.EqualFold(g.Library, "ROCm")) {- params = append(params, "--direct-io")- break- }- }+ params = appendLoadModeArgs(params, launch.opts, launch.gpus)// KV cache typeif launch.kvCacheType != "" {@@ -630,6 +618,23 @@ func appendFlashAttentionArgs(params []string, gpus []ml.DeviceInfo) []string {}}+// appendLoadModeArgs selects llama-server's single model loading mode. Direct I/O+// skips the page cache on load for integrated CUDA/ROCm GPUs, which share system+// memory with the CPU and would otherwise double-buffer weights.+func appendLoadModeArgs(params []string, opts api.Options, gpus []ml.DeviceInfo) []string {+ for _, g := range gpus {+ if runtime.GOOS == "linux" && g.Integrated && (strings.EqualFold(g.Library, "CUDA") || strings.EqualFold(g.Library, "ROCm")) {+ return append(params, "--load-mode", "dio")+ }+ }++ if opts.UseMMap != nil && !*opts.UseMMap {+ return append(params, "--load-mode", "none")+ }++ return params+}+func appendMainGPUArgs(params []string, opts api.Options) []string {if opts.MainGPU == nil {return paramsdiff --git a/llm/llama_server_test.go b/llm/llama_server_test.goindex d39b5b085..744b4322e 100644--- a/llm/llama_server_test.go+++ b/llm/llama_server_test.go@@ -2069,6 +2069,97 @@ func TestAppendMainGPUArgs(t *testing.T) {}}+func TestAppendLoadModeArgs(t *testing.T) {+ mmapOff := api.DefaultOptions()+ mmapOff.UseMMap = testBoolPtr(false)+ mmapOn := api.DefaultOptions()+ mmapOn.UseMMap = testBoolPtr(true)++ integratedCUDA := []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, Integrated: true}}+ integratedROCm := []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "rocm"}, Integrated: true}}+ discreteCUDA := []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}}}+ integratedMetal := []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "Metal"}, Integrated: true}}++ // Direct I/O is only selected on Linux, so the expectation depends on the host.+ dio := []string{"base"}+ dioWithMMapOff := []string{"base", "--load-mode", "none"}+ if runtime.GOOS == "linux" {+ dio = []string{"base", "--load-mode", "dio"}+ dioWithMMapOff = []string{"base", "--load-mode", "dio"}+ }++ tests := []struct {+ name string+ opts api.Options+ gpus []ml.DeviceInfo+ want []string+ }{+ {+ name: "defaults leave llama-server load mode alone",+ opts: api.DefaultOptions(),+ gpus: discreteCUDA,+ want: []string{"base"},+ },+ {+ name: "explicit mmap enabled leaves default",+ opts: mmapOn,+ gpus: discreteCUDA,+ want: []string{"base"},+ },+ {+ name: "mmap disabled selects none",+ opts: mmapOff,+ gpus: discreteCUDA,+ want: []string{"base", "--load-mode", "none"},+ },+ {+ name: "integrated cuda selects dio",+ opts: api.DefaultOptions(),+ gpus: integratedCUDA,+ want: dio,+ },+ {+ name: "integrated rocm selects dio case insensitively",+ opts: api.DefaultOptions(),+ gpus: integratedROCm,+ want: dio,+ },+ {+ name: "integrated metal does not select dio",+ opts: api.DefaultOptions(),+ gpus: integratedMetal,+ want: []string{"base"},+ },+ {+ name: "discrete cuda does not select dio",+ opts: api.DefaultOptions(),+ gpus: discreteCUDA,+ want: []string{"base"},+ },+ {+ name: "dio wins over disabled mmap",+ opts: mmapOff,+ gpus: integratedCUDA,+ want: dioWithMMapOff,+ },+ {+ name: "no gpus leaves default",+ opts: api.DefaultOptions(),+ gpus: nil,+ want: []string{"base"},+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := appendLoadModeArgs([]string{"base"}, tt.opts, tt.gpus)+ if !slices.Equal(got, tt.want) {+ t.Fatalf("appendLoadModeArgs = %v, want %v", got, tt.want)+ }+ })+ }+}+func TestAppendMMProjArgs(t *testing.T) {defaultOpts := api.DefaultOptions()partialOpts := api.DefaultOptions()@@ -2466,6 +2557,10 @@ func testIntPtr(v int) *int {return &v}+func testBoolPtr(v bool) *bool {+ return &v+}+func setFlashAttentionEnv(t *testing.T, value string, set bool) {t.Helper()