Skip to content

LLM inference in C/C++

C++ ggml

CUDA: adding switch points per HW and quant type to tune the mvq->MMQ decode crossover (#26079)

Pranesh Gonegandla2b5621094ef3
diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu
index c999238045..9705348098 100644
--- a/ggml/src/ggml-cuda/mmvq.cu
+++ b/ggml/src/ggml-cuda/mmvq.cu
@@ -290,6 +290,42 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
if (!ggml_is_quantized(type)) {
return false;
}
+ // k-quants cost more to decode and mvq redoes that per column, so MMQ wins sooner.
+ // Only list quant-types MMQ supports, others would fall back to cuBLAS.
+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ADA_LOVELACE) {
+ switch (type) { // tuned on RTX 4090
+ case GGML_TYPE_Q2_K:
+ return ne11 <= 4;
+ case GGML_TYPE_Q3_K:
+ return ne11 <= 6;
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ return ne11 <= 7;
+ default:
+ return ne11 <= MMVQ_MAX_BATCH_SIZE;
+ }
+ }
+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_BLACKWELL) {
+ switch (type) { // tuned on RTX 5090
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ return ne11 <= 5;
+ case GGML_TYPE_Q6_K:
+ return ne11 <= 7;
+ default:
+ return ne11 <= MMVQ_MAX_BATCH_SIZE;
+ }
+ }
+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
+ switch (type) { // tuned on DGX Spark GB10
+ case GGML_TYPE_Q2_K:
+ return ne11 <= 6;
+ default:
+ return ne11 <= MMVQ_MAX_BATCH_SIZE;
+ }
+ }
if (GGML_CUDA_CC_IS_CDNA(cc)) {
if (GGML_CUDA_CC_IS_CDNA1(cc)) {
switch (type) {