LLM inference in C/C++
C++ ggml
CUDA: adding switch points per HW and quant type to tune the mvq->MMQ decode crossover (#26079)
Pranesh Gonegandla2b5621094ef3
diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cuindex c999238045..9705348098 100644--- a/ggml/src/ggml-cuda/mmvq.cu+++ b/ggml/src/ggml-cuda/mmvq.cu@@ -290,6 +290,42 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {if (!ggml_is_quantized(type)) {return false;}+ // k-quants cost more to decode and mvq redoes that per column, so MMQ wins sooner.+ // Only list quant-types MMQ supports, others would fall back to cuBLAS.+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ADA_LOVELACE) {+ switch (type) { // tuned on RTX 4090+ case GGML_TYPE_Q2_K:+ return ne11 <= 4;+ case GGML_TYPE_Q3_K:+ return ne11 <= 6;+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ return ne11 <= 7;+ default:+ return ne11 <= MMVQ_MAX_BATCH_SIZE;+ }+ }+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_BLACKWELL) {+ switch (type) { // tuned on RTX 5090+ case GGML_TYPE_Q2_K:+ case GGML_TYPE_Q3_K:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ return ne11 <= 5;+ case GGML_TYPE_Q6_K:+ return ne11 <= 7;+ default:+ return ne11 <= MMVQ_MAX_BATCH_SIZE;+ }+ }+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {+ switch (type) { // tuned on DGX Spark GB10+ case GGML_TYPE_Q2_K:+ return ne11 <= 6;+ default:+ return ne11 <= MMVQ_MAX_BATCH_SIZE;+ }+ }if (GGML_CUDA_CC_IS_CDNA(cc)) {if (GGML_CUDA_CC_IS_CDNA1(cc)) {switch (type) {