Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

llama-server: allow GPU offload for projectors (#16473)

Daniel Hiltgen01557ff313ad
diff --git a/llm/llama_server.go b/llm/llama_server.go
index b38d887cb..85701351f 100644
--- a/llm/llama_server.go
+++ b/llm/llama_server.go
@@ -631,7 +631,7 @@ func shouldDisableMMProjOffload(opts api.Options, gpus []ml.DeviceInfo, modelLay
}
for _, gpu := range gpus {
- if gpu.Integrated {
+ if gpu.Integrated && gpu.Library != "Metal" {
return true, "shared-memory-gpu"
}
memory := gpu.FreeMemory
diff --git a/llm/llama_server_test.go b/llm/llama_server_test.go
index d5915c4df..b870bea28 100644
--- a/llm/llama_server_test.go
+++ b/llm/llama_server_test.go
@@ -1839,13 +1839,21 @@ func TestAppendMMProjArgs(t *testing.T) {
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
},
{
- name: "integrated gpu disables projector offload",
+ name: "integrated rocm gpu disables projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "ROCm"}, Integrated: true, FreeMemory: 32 << 30}},
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
},
+ {
+ name: "integrated metal gpu keeps projector offload",
+ projectors: []string{"model.gguf"},
+ opts: defaultOpts,
+ gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "Metal"}, Integrated: true, FreeMemory: 32 << 30}},
+ modelLayers: 81,
+ want: []string{"base", "--mmproj", "model.gguf"},
+ },
{
name: "cpu only request disables projector offload",
projectors: []string{"model.gguf"},