Skip to content

Distribute and run LLMs with a single file.

Enable GGML_CUDA_FORCE_MMQ in tinyBLAS mode

Justine Tunney00e4f72b85b8
diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cu
index 3020013c..599fe31f 100644
--- a/llama.cpp/ggml-cuda.cu
+++ b/llama.cpp/ggml-cuda.cu
@@ -322,7 +322,10 @@ GGML_CALL bool ggml_cuda_link(const struct ggml_backend_api *backend_api) {
// - 7B quantum model: +100-200 MB
// - 13B quantum model: +200-400 MB
//
-//#define GGML_CUDA_FORCE_MMQ
+// [jart] https://github.com/Mozilla-Ocho/llamafile/issues/403#issuecomment-2103687594
+#ifdef GGML_USE_TINYBLAS
+#define GGML_CUDA_FORCE_MMQ // [jart] want this
+#endif
// TODO: improve this to be correct for more hardware
// for example, currently fails for GeForce GTX 1660 which is TURING arch (> VOLTA) but does not have tensor cores