Skip to content

LLM inference in C/C++

C++ ggml

HIP: use hipBLAS for dense prefill on gfx900, keep MMQ for MoE (#24588)

zdufordd9df11006f6d
diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
index e1add5e033..6b3b0d064a 100644
--- a/ggml/src/ggml-cuda/mmq.cu
+++ b/ggml/src/ggml-cuda/mmq.cu
@@ -368,5 +368,12 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
return true;
}
+ // gfx900 (Vega 10) lacks native dp4a, loses to dequant + hipBLAS
+ // for dense matrices; keep MMQ only for MoE, where the
+ // hipBLAS path is much slower.
+ if (cc == GGML_CUDA_CC_VEGA) {
+ return n_experts > 0;
+ }
+
return (!GGML_CUDA_CC_IS_CDNA(cc)) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;
}