LLM inference in C/C++
C++ ggml
CUDA: Handle compute type for NVFP4 on cublass path (#29173)
ynankanib56f34ab130b
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex ebbb5c1061..f9fb46c2a3 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -1616,6 +1616,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, conststatic void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const int cc = ggml_cuda_info().devices[ctx.device].cc;+ const ggml_prec prec = (ggml_prec) ggml_get_op_params_i32(dst, 0);ggml_type compute_type = src0->type;if (ggml_is_quantized(compute_type)) {compute_type = fast_fp16_hardware_available(cc) ? GGML_TYPE_F16 : GGML_TYPE_F32;@@ -1629,7 +1630,10 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggmlcompute_type = GGML_TYPE_F32;}}- if (dst->op_params[0] == GGML_PREC_F32) {+ // F16 is the only compute type that can not satisfy a request for BF16+ if (prec == GGML_PREC_BF16 && compute_type == GGML_TYPE_F16) {+ compute_type = fast_bf16_hardware_available(cc) ? GGML_TYPE_BF16 : GGML_TYPE_F32;+ } else if (prec == GGML_PREC_F32) {compute_type = GGML_TYPE_F32;}@@ -2033,6 +2037,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *ggml_tensor dst_slice;memset(&dst_slice, 0, sizeof(dst_slice));+ memcpy(dst_slice.op_params, dst->op_params, sizeof(dst_slice.op_params));dst_slice.buffer = dst->buffer;dst_slice.type = type_dst_sorted;dst_slice.ne[0] = ne0;diff --git a/src/llama-graph.cpp b/src/llama-graph.cppindex 2df063f955..ae477c08de 100644--- a/src/llama-graph.cpp+++ b/src/llama-graph.cpp@@ -1527,6 +1527,10 @@ ggml_tensor * llm_graph_context::build_lora_mm(prec_policy->apply(res);}+ if (w->type == GGML_TYPE_NVFP4) {+ ggml_prec_set_acc(res, GGML_PREC_BF16);+ }+if (w_s) {res = ggml_mul(ctx0, res, w_s);}@@ -1563,6 +1567,10 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(prec_policy->apply(res);}+ if (w->type == GGML_TYPE_NVFP4) {+ ggml_prec_set_acc(res, GGML_PREC_BF16);+ }+if (w_s) {const int64_t n_expert = w_s->ne[0];const int64_t n_tokens = cur->ne[2];