LLM inference in C/C++
C++ ggml
CUDA: use MMVF for thin f16/bf16 mul_mat at small batch size (#29633)
ynankani9d3aba6b5e10
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 303c9f463d..303d67e4c1 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -1778,8 +1778,9 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor) {(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16) &&src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;- const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;- use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, is_mul_mat_id ? src1->ne[2] : src1->ne[1]);+ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;+ const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size;+ use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, is_mul_mat_id ? src1->ne[2] : src1->ne[1]);//we only support fusion for ncols_dst = 1if (tensor->op == GGML_OP_MUL_MAT && dst->ne[1] != 1) {@@ -1892,7 +1893,7 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensorconst int cc = ggml_cuda_info().devices[ctx.device].cc;const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size;- if (ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, ne11)) {+ if (ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11)) {// The custom F16 vector kernel can be used over batched cuBLAS GEMM.// But this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention)ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst);@@ -1902,7 +1903,7 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensorif (ne01 == 1 && ne11 > MMVF_MAX_BATCH_SIZE && ne2 == 1 && ne3 == 1&& src0->type == GGML_TYPE_F32&& ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst)- && ggml_cuda_should_use_mmvf(src1->type, cc, src1->ne, src1->nb, /*ne11 =*/ 1)) {+ && ggml_cuda_should_use_mmvf(src1->type, cc, warp_size, src1->ne, src1->nb, /*ne11 =*/ 1)) {ggml_tensor dst_vec = *dst;dst_vec.ne[0] = ne11;dst_vec.ne[1] = 1;diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cuindex bd5c5d421a..d0a572e158 100644--- a/ggml/src/ggml-cuda/mmvf.cu+++ b/ggml/src/ggml-cuda/mmvf.cu@@ -2,6 +2,7 @@#include "common.cuh"#include "unary.cuh"#include "mmvf.cuh"+#include "mmf.cuh"#include "convert.cuh"template <typename T, typename type_acc, int ncols_dst, int block_size, bool has_fusion = false, bool is_multi_token_id = false>@@ -789,7 +790,7 @@ void ggml_cuda_op_mul_mat_vec_f(GGML_UNUSED_VARS(ctx, src1, dst, src1_ddq_i, src1_ncols, src1_padded_row_size);}-bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11) {+bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11) {if (src0_ne[0] % 2 != 0) {return false;}@@ -820,12 +821,16 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0if (fp32_mma_hardware_available(cc)) {return ne11 <= 3;}- return ne11 <= 8;+ return ne11 <= MMVF_MAX_BATCH_SIZE;}- return ne11 <= 8;+ return ne11 <= MMVF_MAX_BATCH_SIZE;case GGML_TYPE_F16:if (GGML_CUDA_CC_IS_NVIDIA(cc)) {const bool src0_small = (src0_ne[1] <= 512 || src0_ne[2]*src0_ne[3] == 1);+ // MMF needs full row tiles, for other row counts MMVF still beats cuBLAS at small batch size+ if (src0_small && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) {+ return ne11 <= MMVF_MAX_BATCH_SIZE;+ }if (ampere_mma_available(cc)) {return src0_small && ne11 == 1;}@@ -835,8 +840,11 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0if (fp16_mma_hardware_available(cc)) {return src0_small && ne11 <= 3;}- return ne11 <= 8;+ return ne11 <= MMVF_MAX_BATCH_SIZE;} else if (GGML_CUDA_CC_IS_AMD(cc)) {+ if (GGML_CUDA_CC_IS_RDNA(cc) && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) {+ return ne11 <= MMVF_MAX_BATCH_SIZE;+ }if (fp16_mma_hardware_available(cc)) {if (GGML_CUDA_CC_IS_RDNA3(cc)) {return ne11 <= 3;@@ -846,12 +854,16 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0}return ne11 <= 2;}- return ne11 <= 8;+ return ne11 <= MMVF_MAX_BATCH_SIZE;}- return ne11 <= 8;+ return ne11 <= MMVF_MAX_BATCH_SIZE;case GGML_TYPE_BF16:if (GGML_CUDA_CC_IS_NVIDIA(cc)) {const bool src0_small = (src0_ne[1] <= 512 || src0_ne[2]*src0_ne[3] == 1);+ // MMF needs full row tiles, for other row counts MMVF still beats cuBLAS at small batch size+ if (src0_small && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) {+ return ne11 <= MMVF_MAX_BATCH_SIZE;+ }if (ampere_mma_available(cc)) {return src0_small && ne11 == 1;}@@ -861,14 +873,17 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0if (bf16_mma_hardware_available(cc)) {return src0_small && ne11 <= 3;}- return ne11 <= 8;+ return ne11 <= MMVF_MAX_BATCH_SIZE;} else if (GGML_CUDA_CC_IS_AMD(cc)) {+ if (GGML_CUDA_CC_IS_RDNA(cc) && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) {+ return ne11 <= MMVF_MAX_BATCH_SIZE;+ }if (bf16_mma_hardware_available(cc)) {return ne11 <= 3;}- return ne11 <= 8;+ return ne11 <= MMVF_MAX_BATCH_SIZE;}- return ne11 <= 8;+ return ne11 <= MMVF_MAX_BATCH_SIZE;default:return false;}diff --git a/ggml/src/ggml-cuda/mmvf.cuh b/ggml/src/ggml-cuda/mmvf.cuhindex a50f7c0218..350b5cc064 100644--- a/ggml/src/ggml-cuda/mmvf.cuh+++ b/ggml/src/ggml-cuda/mmvf.cuh@@ -11,4 +11,4 @@ void ggml_cuda_op_mul_mat_vec_f(const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,const int64_t src1_padded_row_size, cudaStream_t stream);-bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11);+bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11);