Skip to content

Distribute and run LLMs with a single file.

Don't schlep weights into CPU when using GPU

Justine Tunneyf0e86e1a061f
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index 5ee41a0a..c857e58d 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -893,8 +893,11 @@ struct llama_mmap {
}
}
- // report terminal progress of loading weights
- llamafile_schlep(addr, size);
+ // report terminal progress of loading weights off the disk into
+ // the cpu. if we're using gpu inference, then don't even bother
+ if (!ggml_metal_supported() && !ggml_cublas_loaded()) {
+ llamafile_schlep(addr, size);
+ }
}
~llama_mmap() {