Skip to content

LLM inference in C/C++

C++ ggml

cuda : tune MMVQ to MMQ crossover for SM70 (Volta) (#28912)

Yangyu Chen68d9053afd4f
diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu
index 6305230b1f..a85155360a 100644
--- a/ggml/src/ggml-cuda/mmvq.cu
+++ b/ggml/src/ggml-cuda/mmvq.cu
@@ -365,6 +365,22 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
return ne11 <= MMVQ_MAX_BATCH_SIZE;
}
}
+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_VOLTA) {
+ switch (type) {
+ case GGML_TYPE_Q2_K:
+ return ne11 <= 4;
+ case GGML_TYPE_Q3_K:
+ return ne11 <= 6;
+ case GGML_TYPE_Q4_K:
+ return ne11 <= 5;
+ case GGML_TYPE_Q5_K:
+ return ne11 <= 6;
+ case GGML_TYPE_Q6_K:
+ return ne11 <= 7;
+ default:
+ return ne11 <= MMVQ_MAX_BATCH_SIZE;
+ }
+ }
if (GGML_CUDA_CC_IS_CDNA(cc)) {
if (GGML_CUDA_CC_IS_CDNA1(cc)) {
switch (type) {