LLM inference in C/C++
C++ ggml
CUDA: fix external compilation of q1_0 MMQ (#25778)
Johannes Gäßler1425386fd996
diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cuindex 1f0747456d..8a0f4d3b5c 100644--- a/ggml/src/ggml-cuda/mmq.cu+++ b/ggml/src/ggml-cuda/mmq.cu@@ -25,12 +25,7 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, concase GGML_TYPE_Q8_0:mul_mat_q_case<GGML_TYPE_Q8_0>(ctx, args, stream);break;- case GGML_TYPE_MXFP4:- mul_mat_q_case<GGML_TYPE_MXFP4>(ctx, args, stream);- break;- case GGML_TYPE_NVFP4:- mul_mat_q_case<GGML_TYPE_NVFP4>(ctx, args, stream);- break;+// -----------------------------------------------------------------------case GGML_TYPE_Q2_K:mul_mat_q_case<GGML_TYPE_Q2_K>(ctx, args, stream);break;@@ -46,6 +41,10 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, concase GGML_TYPE_Q6_K:mul_mat_q_case<GGML_TYPE_Q6_K>(ctx, args, stream);break;+// -----------------------------------------------------------------------+ case GGML_TYPE_IQ1_S:+ mul_mat_q_case<GGML_TYPE_IQ1_S>(ctx, args, stream);+ break;case GGML_TYPE_IQ2_XXS:mul_mat_q_case<GGML_TYPE_IQ2_XXS>(ctx, args, stream);break;@@ -61,15 +60,19 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, concase GGML_TYPE_IQ3_S:mul_mat_q_case<GGML_TYPE_IQ3_S>(ctx, args, stream);break;- case GGML_TYPE_IQ1_S:- mul_mat_q_case<GGML_TYPE_IQ1_S>(ctx, args, stream);- break;case GGML_TYPE_IQ4_XS:mul_mat_q_case<GGML_TYPE_IQ4_XS>(ctx, args, stream);break;case GGML_TYPE_IQ4_NL:mul_mat_q_case<GGML_TYPE_IQ4_NL>(ctx, args, stream);break;+// -----------------------------------------------------------------------+ case GGML_TYPE_MXFP4:+ mul_mat_q_case<GGML_TYPE_MXFP4>(ctx, args, stream);+ break;+ case GGML_TYPE_NVFP4:+ mul_mat_q_case<GGML_TYPE_NVFP4>(ctx, args, stream);+ break;default:GGML_ABORT("fatal error");break;@@ -264,21 +267,24 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_tcase GGML_TYPE_Q5_0:case GGML_TYPE_Q5_1:case GGML_TYPE_Q8_0:- case GGML_TYPE_MXFP4:- case GGML_TYPE_NVFP4:+// -------------------------------------------------case GGML_TYPE_Q2_K:case GGML_TYPE_Q3_K:case GGML_TYPE_Q4_K:case GGML_TYPE_Q5_K:case GGML_TYPE_Q6_K:+// -------------------------------------------------+ case GGML_TYPE_IQ1_S:case GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:case GGML_TYPE_IQ2_S:case GGML_TYPE_IQ3_XXS:case GGML_TYPE_IQ3_S:- case GGML_TYPE_IQ1_S:case GGML_TYPE_IQ4_XS:case GGML_TYPE_IQ4_NL:+// -------------------------------------------------+ case GGML_TYPE_MXFP4:+ case GGML_TYPE_NVFP4:mmq_supported = true;break;default:diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuhindex f25835aa37..71e3b2647a 100644--- a/ggml/src/ggml-cuda/mmq.cuh+++ b/ggml/src/ggml-cuda/mmq.cuh@@ -1537,26 +1537,30 @@ void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cuda#define DECL_MMQ_CASE(type) \template void mul_mat_q_case<type>(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \+extern DECL_MMQ_CASE(GGML_TYPE_Q1_0);extern DECL_MMQ_CASE(GGML_TYPE_Q4_0);extern DECL_MMQ_CASE(GGML_TYPE_Q4_1);extern DECL_MMQ_CASE(GGML_TYPE_Q5_0);extern DECL_MMQ_CASE(GGML_TYPE_Q5_1);extern DECL_MMQ_CASE(GGML_TYPE_Q8_0);-extern DECL_MMQ_CASE(GGML_TYPE_MXFP4);-extern DECL_MMQ_CASE(GGML_TYPE_NVFP4);+// -----------------------------------------extern DECL_MMQ_CASE(GGML_TYPE_Q2_K);extern DECL_MMQ_CASE(GGML_TYPE_Q3_K);extern DECL_MMQ_CASE(GGML_TYPE_Q4_K);extern DECL_MMQ_CASE(GGML_TYPE_Q5_K);extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);+// -----------------------------------------+extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);extern DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS);extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);-extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);extern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);+// -----------------------------------------+extern DECL_MMQ_CASE(GGML_TYPE_MXFP4);+extern DECL_MMQ_CASE(GGML_TYPE_NVFP4);// -------------------------------------------------------------------------------------------------------------------------