Skip to content

Distribute and run LLMs with a single file.

Make LLaVA fast again

Justine Tunney54d3c727b4d6
diff --git a/llama.cpp/llava/clip.cpp b/llama.cpp/llava/clip.cpp
index cbf68ab8..85a8db0e 100644
--- a/llama.cpp/llava/clip.cpp
+++ b/llama.cpp/llava/clip.cpp
@@ -1114,15 +1114,19 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
}
}
-#ifdef GGML_USE_CUDA
- new_clip->backend = ggml_backend_cuda_init(0);
- LOG_TEE("%s: CLIP using CUDA backend\n", __func__);
-#endif
+// #ifdef GGML_USE_CUDA
+ if (!new_clip->backend) { // [jart]
+ if ((new_clip->backend = ggml_backend_cuda_init(0)))
+ LOG_TEE("%s: CLIP using CUDA backend\n", __func__);
+ }
+// #endif
-#ifdef GGML_USE_METAL
- new_clip->backend = ggml_backend_metal_init();
- LOG_TEE("%s: CLIP using Metal backend\n", __func__);
-#endif
+// #ifdef GGML_USE_METAL
+ if (!new_clip->backend) { // [jart]
+ if ((new_clip->backend = ggml_backend_metal_init()))
+ LOG_TEE("%s: CLIP using Metal backend\n", __func__);
+ }
+// #endif
#ifdef GGML_USE_CANN
new_clip->backend = ggml_backend_cann_init(0);
diff --git a/llama.cpp/llava/llava-cli.cpp b/llama.cpp/llava/llava-cli.cpp
index cc4d449f..bc4032d8 100644
--- a/llama.cpp/llava/llava-cli.cpp
+++ b/llama.cpp/llava/llava-cli.cpp
@@ -143,14 +143,14 @@ static struct llava_image_embed * load_image(llava_context * ctx_llava, gpt_para
if (!params->image.empty()) {
LOG_TEE("using base64 encoded image instead of command line image path\n");
}
- embed = llava_image_embed_make_with_prompt_base64(ctx_llava->ctx_clip, params->n_threads, prompt);
+ embed = llava_image_embed_make_with_prompt_base64(ctx_llava->ctx_clip, params->n_threads_batch, prompt); // [jart] batch
if (!embed) {
LOG_TEE("%s: can't load image from prompt\n", __func__);
return NULL;
}
params->prompt = remove_image_from_prompt(prompt);
} else {
- embed = llava_image_embed_make_with_filename(ctx_llava->ctx_clip, params->n_threads, fname.c_str());
+ embed = llava_image_embed_make_with_filename(ctx_llava->ctx_clip, params->n_threads_batch, fname.c_str()); // [jart] batch
if (!embed) {
fprintf(stderr, "%s: is %s really an image file?\n", __func__, fname.c_str());
return NULL;