LLM inference in C/C++
C++ ggml
ggml-cuda : disable MMQ on devices with less than 48 KiB shared memory (#26141)
Kakarucaa596ab3f0f
diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cuindex 8a0f4d3b5c..8cf1a3eb94 100644--- a/ggml/src/ggml-cuda/mmq.cu+++ b/ggml/src/ggml-cuda/mmq.cu@@ -296,6 +296,15 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_treturn false;}+ // MMQ tiles require at least 48 KiB per-block shared memory; fall back to BLAS otherwise.+ {+ const int id = ggml_cuda_get_device();+ const size_t smpbo = ggml_cuda_info().devices[id].smpbo;+ if (smpbo < 48 * 1024) {+ return false;+ }+ }+if (turing_mma_available(cc)) {return true;}