Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

llama.cpp update (#17545)

Daniel Hiltgenc82ebbd5bfb9
diff --git a/LLAMA_CPP_VERSION b/LLAMA_CPP_VERSION
index 1c1f72d52..667a5805a 100644
--- a/LLAMA_CPP_VERSION
+++ b/LLAMA_CPP_VERSION
@@ -1 +1 @@
-b10091
+b10242
diff --git a/llama/compat/001-llama-cpp-hooks.patch b/llama/compat/001-llama-cpp-hooks.patch
index 5fc5b15d6..3ded2a5e5 100644
--- a/llama/compat/001-llama-cpp-hooks.patch
+++ b/llama/compat/001-llama-cpp-hooks.patch
@@ -1,24 +1,26 @@
diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp
-index 474cabdfc..2bfe43a28 100644
+index df8313e81..29ccb0f85 100644
--- a/src/llama-model-loader.cpp
+++ b/src/llama-model-loader.cpp
-@@ -4,6 +4,7 @@
- #include "ggml.h"
+@@ -5,6 +5,7 @@
#include "gguf.h"
#include "llama-hparams.h"
+ #include "llama.h"
+#include "llama-ollama-compat.h"
#include <algorithm>
#include <array>
-@@ -551,6 +552,7 @@ llama_model_loader::llama_model_loader(
+@@ -561,6 +562,9 @@ llama_model_loader::llama_model_loader(
}
get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);
-+ if (llama_ollama_compat::translate_metadata(this, metadata, ctx, arch_name, fname.c_str())) use_mmap = false;
++ if (llama_ollama_compat::translate_metadata(this, metadata, ctx, arch_name, fname.c_str())) {
++ this->use_mmap = false;
++ }
llm_kv = LLM_KV(llm_arch_from_string(arch_name));
files.emplace_back(new llama_file(fname.c_str(), "rb", use_direct_io));
-@@ -575,6 +577,9 @@ llama_model_loader::llama_model_loader(
+@@ -571,6 +575,9 @@ llama_model_loader::llama_model_loader(
// so we build a unified tensors index for weights.
for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
std::string tensor_name = std::string(cur->name);
@@ -28,7 +30,7 @@ index 474cabdfc..2bfe43a28 100644
// make sure there is no duplicated tensor names
if (weights_map.find(tensor_name) != weights_map.end()) {
throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));
-@@ -685,6 +690,9 @@ llama_model_loader::llama_model_loader(
+@@ -681,6 +688,9 @@ llama_model_loader::llama_model_loader(
// Save tensors data offset info of the main file.
for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
std::string tensor_name = std::string(cur->name);
@@ -38,7 +40,7 @@ index 474cabdfc..2bfe43a28 100644
// make sure there is no duplicated tensor names
if (weights_map.find(tensor_name) != weights_map.end()) {
throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));
-@@ -1384,6 +1392,7 @@ void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void *
+@@ -1380,6 +1390,7 @@ void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void *
void llama_model_loader::load_data_for(struct ggml_tensor * cur) const {
const auto & w = require_weight(ggml_get_name(cur));
@@ -46,7 +48,7 @@ index 474cabdfc..2bfe43a28 100644
if (use_mmap) {
const auto & mapping = mappings.at(w.idx);
-@@ -1534,6 +1543,7 @@ bool llama_model_loader::load_all_data(
+@@ -1530,6 +1541,7 @@ bool llama_model_loader::load_all_data(
}
size_t n_size = ggml_nbytes(cur);
@@ -55,7 +57,7 @@ index 474cabdfc..2bfe43a28 100644
if (use_mmap) {
const auto & mapping = mappings.at(weight->idx);
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
-index 7bd486030..6c3b23da0 100644
+index c1870813f..a1a923082 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -10,6 +10,8 @@
@@ -67,7 +69,7 @@ index 7bd486030..6c3b23da0 100644
#include <algorithm>
#include <cassert>
#include <cmath>
-@@ -1070,6 +1072,11 @@ struct clip_model_loader {
+@@ -1093,6 +1095,11 @@ struct clip_model_loader {
ctx_meta.reset(meta);
@@ -79,7 +81,7 @@ index 7bd486030..6c3b23da0 100644
const int n_tensors = gguf_get_n_tensors(ctx_gguf.get());
// print gguf info
-@@ -2822,6 +2829,7 @@ struct clip_model_loader {
+@@ -3085,6 +3092,7 @@ struct clip_model_loader {
auto it_off = tensor_offset.find(t->name);
GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor");
const size_t offset = it_off->second;
@@ -87,7 +89,7 @@ index 7bd486030..6c3b23da0 100644
fin.seekg(offset, std::ios::beg);
if (!fin) {
throw std::runtime_error(string_format("%s: failed to seek for tensor %s\n", __func__, t->name));
-@@ -4489,6 +4497,15 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
+@@ -4964,6 +4972,15 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
}
int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
diff --git a/llama/server/CMakeLists.txt b/llama/server/CMakeLists.txt
index bec100df2..7d88fcd7d 100644
--- a/llama/server/CMakeLists.txt
+++ b/llama/server/CMakeLists.txt
@@ -176,6 +176,7 @@ set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
set(LLAMA_TOOLS_INSTALL OFF CACHE BOOL "" FORCE)
set(LLAMA_CURL OFF CACHE BOOL "" FORCE)
set(LLAMA_OPENSSL OFF CACHE BOOL "" FORCE)
+set(LLAMA_SUBPROCESS OFF CACHE BOOL "" FORCE)
FetchContent_Declare(
llama_cpp
diff --git a/llm/llama_server.go b/llm/llama_server.go
index e529cfc13..c99c29556 100644
--- a/llm/llama_server.go
+++ b/llm/llama_server.go
@@ -385,19 +385,7 @@ func startLlamaServer(launch llamaServerLaunchConfig, out io.Writer) (cmd *exec.
params = append(params, "--lora", adapter)
}
- // UseMmap
- if launch.opts.UseMMap != nil && !*launch.opts.UseMMap {
- params = append(params, "--no-mmap")
- }
-
- // Direct I/O skips the page cache on load for integrated CUDA/ROCm GPUs, which
- // share system memory with the CPU and would otherwise double-buffer weights.
- for _, g := range launch.gpus {
- if runtime.GOOS == "linux" && g.Integrated && (strings.EqualFold(g.Library, "CUDA") || strings.EqualFold(g.Library, "ROCm")) {
- params = append(params, "--direct-io")
- break
- }
- }
+ params = appendLoadModeArgs(params, launch.opts, launch.gpus)
// KV cache type
if launch.kvCacheType != "" {
@@ -630,6 +618,23 @@ func appendFlashAttentionArgs(params []string, gpus []ml.DeviceInfo) []string {
}
}
+// appendLoadModeArgs selects llama-server's single model loading mode. Direct I/O
+// skips the page cache on load for integrated CUDA/ROCm GPUs, which share system
+// memory with the CPU and would otherwise double-buffer weights.
+func appendLoadModeArgs(params []string, opts api.Options, gpus []ml.DeviceInfo) []string {
+ for _, g := range gpus {
+ if runtime.GOOS == "linux" && g.Integrated && (strings.EqualFold(g.Library, "CUDA") || strings.EqualFold(g.Library, "ROCm")) {
+ return append(params, "--load-mode", "dio")
+ }
+ }
+
+ if opts.UseMMap != nil && !*opts.UseMMap {
+ return append(params, "--load-mode", "none")
+ }
+
+ return params
+}
+
func appendMainGPUArgs(params []string, opts api.Options) []string {
if opts.MainGPU == nil {
return params
diff --git a/llm/llama_server_test.go b/llm/llama_server_test.go
index d39b5b085..744b4322e 100644
--- a/llm/llama_server_test.go
+++ b/llm/llama_server_test.go
@@ -2069,6 +2069,97 @@ func TestAppendMainGPUArgs(t *testing.T) {
}
}
+func TestAppendLoadModeArgs(t *testing.T) {
+ mmapOff := api.DefaultOptions()
+ mmapOff.UseMMap = testBoolPtr(false)
+ mmapOn := api.DefaultOptions()
+ mmapOn.UseMMap = testBoolPtr(true)
+
+ integratedCUDA := []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, Integrated: true}}
+ integratedROCm := []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "rocm"}, Integrated: true}}
+ discreteCUDA := []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}}}
+ integratedMetal := []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "Metal"}, Integrated: true}}
+
+ // Direct I/O is only selected on Linux, so the expectation depends on the host.
+ dio := []string{"base"}
+ dioWithMMapOff := []string{"base", "--load-mode", "none"}
+ if runtime.GOOS == "linux" {
+ dio = []string{"base", "--load-mode", "dio"}
+ dioWithMMapOff = []string{"base", "--load-mode", "dio"}
+ }
+
+ tests := []struct {
+ name string
+ opts api.Options
+ gpus []ml.DeviceInfo
+ want []string
+ }{
+ {
+ name: "defaults leave llama-server load mode alone",
+ opts: api.DefaultOptions(),
+ gpus: discreteCUDA,
+ want: []string{"base"},
+ },
+ {
+ name: "explicit mmap enabled leaves default",
+ opts: mmapOn,
+ gpus: discreteCUDA,
+ want: []string{"base"},
+ },
+ {
+ name: "mmap disabled selects none",
+ opts: mmapOff,
+ gpus: discreteCUDA,
+ want: []string{"base", "--load-mode", "none"},
+ },
+ {
+ name: "integrated cuda selects dio",
+ opts: api.DefaultOptions(),
+ gpus: integratedCUDA,
+ want: dio,
+ },
+ {
+ name: "integrated rocm selects dio case insensitively",
+ opts: api.DefaultOptions(),
+ gpus: integratedROCm,
+ want: dio,
+ },
+ {
+ name: "integrated metal does not select dio",
+ opts: api.DefaultOptions(),
+ gpus: integratedMetal,
+ want: []string{"base"},
+ },
+ {
+ name: "discrete cuda does not select dio",
+ opts: api.DefaultOptions(),
+ gpus: discreteCUDA,
+ want: []string{"base"},
+ },
+ {
+ name: "dio wins over disabled mmap",
+ opts: mmapOff,
+ gpus: integratedCUDA,
+ want: dioWithMMapOff,
+ },
+ {
+ name: "no gpus leaves default",
+ opts: api.DefaultOptions(),
+ gpus: nil,
+ want: []string{"base"},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := appendLoadModeArgs([]string{"base"}, tt.opts, tt.gpus)
+ if !slices.Equal(got, tt.want) {
+ t.Fatalf("appendLoadModeArgs = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
func TestAppendMMProjArgs(t *testing.T) {
defaultOpts := api.DefaultOptions()
partialOpts := api.DefaultOptions()
@@ -2466,6 +2557,10 @@ func testIntPtr(v int) *int {
return &v
}
+func testBoolPtr(v bool) *bool {
+ return &v
+}
+
func setFlashAttentionEnv(t *testing.T, value string, set bool) {
t.Helper()