LLM inference in C/C++
C++ ggml
cuda : tune MMVQ to MMQ crossover for SM70 (Volta) (#28912)
Yangyu Chen68d9053afd4f
diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cuindex 6305230b1f..a85155360a 100644--- a/ggml/src/ggml-cuda/mmvq.cu+++ b/ggml/src/ggml-cuda/mmvq.cu@@ -365,6 +365,22 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {return ne11 <= MMVQ_MAX_BATCH_SIZE;}}+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_VOLTA) {+ switch (type) {+ case GGML_TYPE_Q2_K:+ return ne11 <= 4;+ case GGML_TYPE_Q3_K:+ return ne11 <= 6;+ case GGML_TYPE_Q4_K:+ return ne11 <= 5;+ case GGML_TYPE_Q5_K:+ return ne11 <= 6;+ case GGML_TYPE_Q6_K:+ return ne11 <= 7;+ default:+ return ne11 <= MMVQ_MAX_BATCH_SIZE;+ }+ }if (GGML_CUDA_CC_IS_CDNA(cc)) {if (GGML_CUDA_CC_IS_CDNA1(cc)) {switch (type) {