LLM inference in C/C++
C++ ggml
cuda: always use MMVQ for MUL_MAT_ID on sm_60 (#27828)
David Friehs1623d8ce47bc
diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cuindex 251577d2fd..0714ae3bbd 100644--- a/ggml/src/ggml-cuda/mmvq.cu+++ b/ggml/src/ggml-cuda/mmvq.cu@@ -283,9 +283,9 @@ static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna4(ggml_type// Host function: returns the max batch size for the current arch+type at runtime.int get_mmvq_mmid_max_batch(ggml_type type, int cc) {- // NVIDIA: Volta, Ada Lovelace, and Blackwell always use MMVQ for MUL_MAT_ID.+ // NVIDIA: P100, Volta, Ada Lovelace, and Blackwell always use MMVQ for MUL_MAT_ID.if (GGML_CUDA_CC_IS_NVIDIA(cc)) {- if (cc == GGML_CUDA_CC_VOLTA || cc >= GGML_CUDA_CC_ADA_LOVELACE) {+ if (cc == GGML_CUDA_CC_PASCAL || cc == GGML_CUDA_CC_VOLTA || cc >= GGML_CUDA_CC_ADA_LOVELACE) {return MMVQ_MAX_BATCH_SIZE;}if (cc >= GGML_CUDA_CC_TURING) {@@ -440,7 +440,7 @@ static constexpr __device__ int get_mmvq_mmid_max_batch_for_device() {return get_mmvq_mmid_max_batch_cdna(type);#elif defined(GCN)return get_mmvq_mmid_max_batch_gcn(type);-#elif !defined(GGML_USE_MUSA) && (__CUDA_ARCH__ == GGML_CUDA_CC_VOLTA || __CUDA_ARCH__ >= GGML_CUDA_CC_ADA_LOVELACE)+#elif !defined(GGML_USE_MUSA) && (__CUDA_ARCH__ == GGML_CUDA_CC_PASCAL || __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA || __CUDA_ARCH__ >= GGML_CUDA_CC_ADA_LOVELACE)return MMVQ_MAX_BATCH_SIZE;#elif !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURINGreturn get_mmvq_mmid_max_batch_turing_plus(type);