Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
llama: use dynamic backend loading for mllama and clip (#8835)
Jeffrey Morgancd3fbf1c49aa
diff --git a/llama/llama.cpp/examples/llava/clip.cpp b/llama/llama.cpp/examples/llava/clip.cppindex 718052e16..86b91d5cb 100644--- a/llama/llama.cpp/examples/llava/clip.cpp+++ b/llama/llama.cpp/examples/llava/clip.cpp@@ -1235,35 +1235,15 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {}}-#ifdef GGML_USE_CUDA- new_clip->backend = ggml_backend_cuda_init(0);- LOG_INF("%s: CLIP using CUDA backend\n", __func__);-#endif--#ifdef GGML_USE_METAL- new_clip->backend = ggml_backend_metal_init();- LOG_INF("%s: CLIP using Metal backend\n", __func__);-#endif--#ifdef GGML_USE_CANN- new_clip->backend = ggml_backend_cann_init(0);- LOG_INF("%s: CLIP using CANN backend\n", __func__);-#endif--#ifdef GGML_USE_VULKAN- new_clip->backend = ggml_backend_vk_init(0);- LOG_INF("%s: CLIP using Vulkan backend\n", __func__);-#endif--#ifdef GGML_USE_SYCL- new_clip->backend = ggml_backend_sycl_init(0);- LOG_INF("%s: CLIP using SYCL backend\n", __func__);-#endif-- if (!new_clip->backend) {- new_clip->backend = ggml_backend_cpu_init();- LOG_INF("%s: CLIP using CPU backend\n", __func__);+ ggml_backend_t backend = ggml_backend_init_best();+ if (backend == nullptr) {+ LOG_ERR("%s: failed to initialize backend\n", __func__);+ clip_free(new_clip);+ gguf_free(ctx);+ return nullptr;}+ LOG_INF("%s: using %s backend\n", __func__, ggml_backend_name(backend));+ new_clip->backend = backend;// model size and capabilities{diff --git a/llama/mllama.cpp b/llama/mllama.cppindex df5bd6a98..4e84c60ae 100644--- a/llama/mllama.cpp+++ b/llama/mllama.cpp@@ -558,30 +558,15 @@ struct mllama_ctx *mllama_model_load(const char *fname, const int verbosity = 1)mllama_ctx *new_mllama = new mllama_ctx{};-#ifdef GGML_USE_CUDA- new_mllama->backend = ggml_backend_cuda_init(0);- LOG("vision using CUDA backend");-#endif--#ifdef GGML_USE_METAL- new_mllama->backend = ggml_backend_metal_init();- LOG("vision using Metal backend");-#endif--#ifdef GGML_USE_CANN- new_mllama->backend = ggml_backend_cann_init(0);- LOG("vision using CANN backend");-#endif--#ifdef GGML_USE_VULKAN- new_mllama->backend = ggml_backend_vk_init(0);- LOG("vision using Vulkan backend");-#endif-- if (!new_mllama->backend) {- new_mllama->backend = ggml_backend_cpu_init();- LOG("vision using CPU backend");+ ggml_backend_t backend = ggml_backend_init_best();+ if (backend == nullptr) {+ LOG("%s: failed to initialize backend\n", __func__);+ mllama_free(new_mllama);+ gguf_free(ctx);+ return nullptr;}+ LOG("%s: using %s backend\n", __func__, ggml_backend_name(backend));+ new_mllama->backend = backend;// load tensors{diff --git a/llama/patches/0013-re-enable-gpu-for-clip.patch b/llama/patches/0013-use-dynamic-backend-loading-for-clip.patchsimilarity index 64%rename from llama/patches/0013-re-enable-gpu-for-clip.patchrename to llama/patches/0013-use-dynamic-backend-loading-for-clip.patchindex a38d08841..e283a857f 100644--- a/llama/patches/0013-re-enable-gpu-for-clip.patch+++ b/llama/patches/0013-use-dynamic-backend-loading-for-clip.patch@@ -1,14 +1,14 @@From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001From: jmorganca <[email protected]>Date: Sat, 4 Jan 2025 22:52:48 -0800-Subject: [PATCH] re-enable gpu for clip+Subject: [PATCH] use dynamic backend loading for clip---- examples/llava/clip.cpp | 86 ++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 43 deletions(-)+ examples/llava/clip.cpp | 74 +++++++++++++++--------------------------+ 1 file changed, 27 insertions(+), 47 deletions(-)diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp-index b3c1829f..718052e1 100644+index b3c1829f..86b91d5c 100644--- a/examples/llava/clip.cpp+++ b/examples/llava/clip.cpp@@ -8,25 +8,25 @@@@ -56,7 +56,7 @@ index b3c1829f..718052e1 100644#define STB_IMAGE_IMPLEMENTATION#include "stb_image.h"-@@ -1235,30 +1235,30 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {+@@ -1235,35 +1235,15 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {}}@@ -84,30 +84,19 @@ index b3c1829f..718052e1 100644-// new_clip->backend = ggml_backend_sycl_init(0);-// LOG_INF("%s: CLIP using SYCL backend\n", __func__);-//#endif-+#ifdef GGML_USE_CUDA-+ new_clip->backend = ggml_backend_cuda_init(0);-+ LOG_INF("%s: CLIP using CUDA backend\n", __func__);-+#endif-+-+#ifdef GGML_USE_METAL-+ new_clip->backend = ggml_backend_metal_init();-+ LOG_INF("%s: CLIP using Metal backend\n", __func__);-+#endif-+-+#ifdef GGML_USE_CANN-+ new_clip->backend = ggml_backend_cann_init(0);-+ LOG_INF("%s: CLIP using CANN backend\n", __func__);-+#endif-+-+#ifdef GGML_USE_VULKAN-+ new_clip->backend = ggml_backend_vk_init(0);-+ LOG_INF("%s: CLIP using Vulkan backend\n", __func__);-+#endif-+-+#ifdef GGML_USE_SYCL-+ new_clip->backend = ggml_backend_sycl_init(0);-+ LOG_INF("%s: CLIP using SYCL backend\n", __func__);-+#endif+-+- if (!new_clip->backend) {+- new_clip->backend = ggml_backend_cpu_init();+- LOG_INF("%s: CLIP using CPU backend\n", __func__);++ ggml_backend_t backend = ggml_backend_init_best();++ if (backend == nullptr) {++ LOG_ERR("%s: failed to initialize backend\n", __func__);++ clip_free(new_clip);++ gguf_free(ctx);++ return nullptr;+ }++ LOG_INF("%s: using %s backend\n", __func__, ggml_backend_name(backend));++ new_clip->backend = backend;- if (!new_clip->backend) {- new_clip->backend = ggml_backend_cpu_init();+ // model size and capabilities+ {