Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Disable GGML_CUDA_FORCE_MMQ
Justine Tunney698de9147dfd
diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cuindex 800f5021..de814da4 100644--- a/llama.cpp/ggml-cuda.cu+++ b/llama.cpp/ggml-cuda.cu@@ -341,8 +341,15 @@ void ggml_abort(const char * file, int line, const char * fmt, ...) {// - 13B quantum model: +200-400 MB//// [jart] https://github.com/Mozilla-Ocho/llamafile/issues/403#issuecomment-2103687594+//+// TODO(jart): oops looks like we can't use this anymore, because my+// five year old NVIDIA GeForce RTX 2080 Ti card stopped+// working with "ggml-cuda.cu:11460: ERROR: CUDA kernel+// mul_mat_q has no device code compatible with CUDA arch+// 700. ggml-cuda.cu was compiled for: 600,700,800,900"!+//#ifdef GGML_USE_TINYBLAS-#define GGML_CUDA_FORCE_MMQ // [jart] want this+// #define GGML_CUDA_FORCE_MMQ // [jart] want this#endifGGML_CALL bool ggml_cuda_link(const struct ggml_backend_api *backend_api) {