Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
llama-server: allow GPU offload for projectors (#16473)
Daniel Hiltgen01557ff313ad
diff --git a/llm/llama_server.go b/llm/llama_server.goindex b38d887cb..85701351f 100644--- a/llm/llama_server.go+++ b/llm/llama_server.go@@ -631,7 +631,7 @@ func shouldDisableMMProjOffload(opts api.Options, gpus []ml.DeviceInfo, modelLay}for _, gpu := range gpus {- if gpu.Integrated {+ if gpu.Integrated && gpu.Library != "Metal" {return true, "shared-memory-gpu"}memory := gpu.FreeMemorydiff --git a/llm/llama_server_test.go b/llm/llama_server_test.goindex d5915c4df..b870bea28 100644--- a/llm/llama_server_test.go+++ b/llm/llama_server_test.go@@ -1839,13 +1839,21 @@ func TestAppendMMProjArgs(t *testing.T) {want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},},{- name: "integrated gpu disables projector offload",+ name: "integrated rocm gpu disables projector offload",projectors: []string{"model.gguf"},opts: defaultOpts,gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "ROCm"}, Integrated: true, FreeMemory: 32 << 30}},modelLayers: 81,want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},},+ {+ name: "integrated metal gpu keeps projector offload",+ projectors: []string{"model.gguf"},+ opts: defaultOpts,+ gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "Metal"}, Integrated: true, FreeMemory: 32 << 30}},+ modelLayers: 81,+ want: []string{"base", "--mmproj", "model.gguf"},+ },{name: "cpu only request disables projector offload",projectors: []string{"model.gguf"},