Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Make LLaVA fast again
Justine Tunney54d3c727b4d6
diff --git a/llama.cpp/llava/clip.cpp b/llama.cpp/llava/clip.cppindex cbf68ab8..85a8db0e 100644--- a/llama.cpp/llava/clip.cpp+++ b/llama.cpp/llava/clip.cpp@@ -1114,15 +1114,19 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {}}-#ifdef GGML_USE_CUDA- new_clip->backend = ggml_backend_cuda_init(0);- LOG_TEE("%s: CLIP using CUDA backend\n", __func__);-#endif+// #ifdef GGML_USE_CUDA+ if (!new_clip->backend) { // [jart]+ if ((new_clip->backend = ggml_backend_cuda_init(0)))+ LOG_TEE("%s: CLIP using CUDA backend\n", __func__);+ }+// #endif-#ifdef GGML_USE_METAL- new_clip->backend = ggml_backend_metal_init();- LOG_TEE("%s: CLIP using Metal backend\n", __func__);-#endif+// #ifdef GGML_USE_METAL+ if (!new_clip->backend) { // [jart]+ if ((new_clip->backend = ggml_backend_metal_init()))+ LOG_TEE("%s: CLIP using Metal backend\n", __func__);+ }+// #endif#ifdef GGML_USE_CANNnew_clip->backend = ggml_backend_cann_init(0);diff --git a/llama.cpp/llava/llava-cli.cpp b/llama.cpp/llava/llava-cli.cppindex cc4d449f..bc4032d8 100644--- a/llama.cpp/llava/llava-cli.cpp+++ b/llama.cpp/llava/llava-cli.cpp@@ -143,14 +143,14 @@ static struct llava_image_embed * load_image(llava_context * ctx_llava, gpt_paraif (!params->image.empty()) {LOG_TEE("using base64 encoded image instead of command line image path\n");}- embed = llava_image_embed_make_with_prompt_base64(ctx_llava->ctx_clip, params->n_threads, prompt);+ embed = llava_image_embed_make_with_prompt_base64(ctx_llava->ctx_clip, params->n_threads_batch, prompt); // [jart] batchif (!embed) {LOG_TEE("%s: can't load image from prompt\n", __func__);return NULL;}params->prompt = remove_image_from_prompt(prompt);} else {- embed = llava_image_embed_make_with_filename(ctx_llava->ctx_clip, params->n_threads, fname.c_str());+ embed = llava_image_embed_make_with_filename(ctx_llava->ctx_clip, params->n_threads_batch, fname.c_str()); // [jart] batchif (!embed) {fprintf(stderr, "%s: is %s really an image file?\n", __func__, fname.c_str());return NULL;