LLM inference in C/C++
C++ ggml
tune MMVQ to MMQ crossover for SM87 (#28285)
kbenkhaled8c1a25166b6b
diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuhindex e5ccd1feab..9918c03947 100644--- a/ggml/src/ggml-cuda/common.cuh+++ b/ggml/src/ggml-cuda/common.cuh@@ -52,6 +52,7 @@#define GGML_CUDA_CC_VOLTA 700#define GGML_CUDA_CC_TURING 750#define GGML_CUDA_CC_AMPERE 800+#define GGML_CUDA_CC_ORIN 870#define GGML_CUDA_CC_ADA_LOVELACE 890#define GGML_CUDA_CC_HOPPER 900// While BW spans CC 1000, 1100 & 1200, we are integrating Tensor Core instructions available to 1200 family, seediff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cuindex 2be2f24910..f65e0fbcd7 100644--- a/ggml/src/ggml-cuda/mmvq.cu+++ b/ggml/src/ggml-cuda/mmvq.cu@@ -326,6 +326,18 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {return ne11 <= MMVQ_MAX_BATCH_SIZE;}}+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ORIN) {+ switch (type) { // tuned for Jetson Orin+ case GGML_TYPE_Q2_K:+ case GGML_TYPE_Q3_K:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ case GGML_TYPE_Q6_K:+ return ne11 <= 1;+ default:+ return ne11 <= MMVQ_MAX_BATCH_SIZE;+ }+ }if (GGML_CUDA_CC_IS_CDNA(cc)) {if (GGML_CUDA_CC_IS_CDNA1(cc)) {switch (type) {