LLM inference in C/C++
C++ ggml
CUDA: fuse RMS_NORM + SCALE into one kernel (#29393)
InflexCZE1ab7e5ad2d4e
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 38b8714d5f..36c6a93b18 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -3316,6 +3316,19 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph,return true;}+ if (ops.size() == 2 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_SCALE) {+ const ggml_tensor * rms_norm = cgraph->nodes[node_idx];+ const ggml_tensor * scale = cgraph->nodes[node_idx+1];++ GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32);+ GGML_ASSERT(rms_norm->type == GGML_TYPE_F32);++ float bias;+ memcpy(&bias, (const float *) scale->op_params + 1, sizeof(float));++ return bias == 0.0f && scale->type == GGML_TYPE_F32;+ }+if (ops.size() == 2 && ops.begin()[0] == GGML_OP_SSM_CONV && ops.begin()[1] == GGML_OP_UNARY&& unary_ops.size() == 1 && unary_ops.begin()[0] == GGML_UNARY_OP_SILU) {const ggml_tensor * ssm_conv = cgraph->nodes[node_idx];@@ -4157,6 +4170,11 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraphreturn 1;}+ if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_SCALE }, {})) {+ ggml_cuda_op_rms_norm_scale_fused(*cuda_ctx, node, cgraph->nodes[i + 1]);+ return 1;+ }+if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_SSM_CONV, GGML_OP_ADD, GGML_OP_UNARY }, { GGML_UNARY_OP_SILU })) {ggml_cuda_op_ssm_conv(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]);return 2;diff --git a/ggml/src/ggml-cuda/norm.cu b/ggml/src/ggml-cuda/norm.cuindex c3758cd50c..5543307be6 100644--- a/ggml/src/ggml-cuda/norm.cu+++ b/ggml/src/ggml-cuda/norm.cu@@ -73,7 +73,7 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int gr}}-template <int block_size, bool do_multiply = false, bool do_add = false>+template <int block_size, bool do_multiply = false, bool do_add = false, bool do_scale = false>static __global__ void rms_norm_f32(const float * x,float * dst,const int ncols,@@ -96,7 +96,8 @@ static __global__ void rms_norm_f32(const float * x,const uint3 add_ncols_packed = make_uint3(0, 0, 0),const uint3 add_nrows_packed = make_uint3(0, 0, 0),const uint3 add_nchannels_packed = make_uint3(0, 0, 0),- const uint3 add_nsamples_packed = make_uint3(0, 0, 0)) {+ const uint3 add_nsamples_packed = make_uint3(0, 0, 0),+ const float scale_out = 1.0f) {ggml_cuda_pdl_lc();const int nrows = gridDim.x;const int nchannels = gridDim.y;@@ -107,6 +108,7 @@ static __global__ void rms_norm_f32(const float * x,const int tid = threadIdx.x;static_assert(!do_add || do_multiply, "fusing add is not supported without multiplying");+ static_assert(!do_scale || !do_multiply, "fusing scale is not supported with multiplying");x += sample*stride_sample + channel*stride_channel + row*stride_row;dst += ((sample*nchannels + channel)*nrows + row)*ncols;@@ -148,6 +150,8 @@ static __global__ void rms_norm_f32(const float * x,} else if constexpr (do_multiply) {const int mul_col = fastmodulo(col, mul_ncols_packed);dst[col] = scale * x[col] * mul[mul_col];+ } else if constexpr (do_scale) {+ dst[col] = scale_out * (scale * x[col]);} else {dst[col] = scale * x[col];}@@ -301,25 +305,27 @@ static void group_norm_f32_cuda(}}+template <bool do_scale = false>static void rms_norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples,- const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) {+ const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream,+ const float scale_out = 1.0f) {const dim3 blocks_num(nrows, nchannels, nsamples);if (ncols < 1024) {const dim3 block_dims(256, 1, 1);const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream};- ggml_cuda_kernel_launch(rms_norm_f32<256, false>, launch_params,+ ggml_cuda_kernel_launch(rms_norm_f32<256, false, false, do_scale>, launch_params,x, dst, ncols, stride_row, stride_channel, stride_sample, eps,// underlying cudaLaunchKernelEx does not support default paramsnullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0),- nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0));+ nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), scale_out);} else {const dim3 block_dims(1024, 1, 1);const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream};- ggml_cuda_kernel_launch(rms_norm_f32<1024, false>, launch_params, x, dst, ncols, stride_row, stride_channel, stride_sample, eps,+ ggml_cuda_kernel_launch(rms_norm_f32<1024, false, false, do_scale>, launch_params, x, dst, ncols, stride_row, stride_channel, stride_sample, eps,// underlying cudaLaunchKernelEx does not support default paramsnullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0),- nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0));+ nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), scale_out);}}@@ -367,7 +373,7 @@ static void rms_norm_mul_f32_cuda(const float * x,x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel,mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,// underlying cudaLaunchKernelEx does not support default params- nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0));+ nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), 1.0f);} else {const dim3 block_dims(1024, 1, 1);const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream};@@ -375,7 +381,7 @@ static void rms_norm_mul_f32_cuda(const float * x,x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel,mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,// underlying cudaLaunchKernelEx does not support default params- nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0));+ nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), 1.0f);}} else {const uint3 mul_ncols_packed = init_fastdiv_values(mul_ncols);@@ -394,7 +400,7 @@ static void rms_norm_mul_f32_cuda(const float * x,x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel,mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, add,add_stride_row, add_stride_channel, add_stride_sample, add_ncols_packed, add_nrows_packed,- add_nchannels_packed, add_nsamples_packed);+ add_nchannels_packed, add_nsamples_packed, 1.0f);} else {const dim3 block_dims(1024, 1, 1);const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream};@@ -402,7 +408,7 @@ static void rms_norm_mul_f32_cuda(const float * x,x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel,mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, add,add_stride_row, add_stride_channel, add_stride_sample, add_ncols_packed, add_nrows_packed,- add_nchannels_packed, add_nsamples_packed);+ add_nchannels_packed, add_nsamples_packed, 1.0f);}}}@@ -499,6 +505,33 @@ void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {rms_norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream);}+void ggml_cuda_op_rms_norm_scale_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * scale_tensor) {+ const ggml_tensor * src0 = dst->src[0];+ const float * src0_d = (const float *) src0->data;+ float * dst_d = (float *) scale_tensor->data;+ cudaStream_t stream = ctx.stream();++ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT(scale_tensor->type == GGML_TYPE_F32);++ GGML_TENSOR_UNARY_OP_LOCALS;++ float eps;+ memcpy(&eps, dst->op_params, sizeof(float));+ GGML_ASSERT(eps >= 0.0f);++ float scale;+ memcpy(&scale, (const float *) scale_tensor->op_params + 0, sizeof(float));++ const size_t ts0 = ggml_type_size(src0->type);+ GGML_ASSERT(nb00 == ts0);+ const int64_t s01 = nb01 / ts0;+ const int64_t s02 = nb02 / ts0;+ const int64_t s03 = nb03 / ts0;++ rms_norm_f32_cuda<true>(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream, scale);+}+void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor) {const ggml_tensor * rms_norm_src = (ggml_tensor *) dst->src[0];float eps = 0.0f;diff --git a/ggml/src/ggml-cuda/norm.cuh b/ggml/src/ggml-cuda/norm.cuhindex a74f637672..95618df5fd 100644--- a/ggml/src/ggml-cuda/norm.cuh+++ b/ggml/src/ggml-cuda/norm.cuh@@ -8,6 +8,8 @@ void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor);+void ggml_cuda_op_rms_norm_scale_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * scale_tensor);+void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx,ggml_tensor * dst,ggml_tensor * mul_tensor,