LLM inference in C/C++
C++ ggml
cuda : support arbitrary striding for unary ops on f16, f32, and bf16 (#29781)
Muhammad Saad08246a28f600
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 20775a9219..995037f1b1 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -5314,9 +5314,10 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const gcase GGML_UNARY_OP_CEIL:case GGML_UNARY_OP_ROUND:case GGML_UNARY_OP_TRUNC:- // TODO: should become:- //return ggml_is_contiguous_rows(op->src[0]);- return ggml_is_contiguous(op->src[0]);+ if (op->src[0]->type == GGML_TYPE_BF16 && ggml_get_unary_op(op) == GGML_UNARY_OP_XIELU) {+ return false;+ }+ return op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16;default:return false;}diff --git a/ggml/src/ggml-cuda/unary.cu b/ggml/src/ggml-cuda/unary.cuindex c9a8cf6b00..436fdfffb9 100644--- a/ggml/src/ggml-cuda/unary.cu+++ b/ggml/src/ggml-cuda/unary.cu@@ -134,24 +134,67 @@ static void unary_cuda(const T * x, T * dst, const int k, cudaStream_t stream) {ggml_cuda_kernel_launch(unary_op_kernel<op, T>, launch_params, x, dst, k);}+template <float (*op)(float), typename T>+static __global__ void unary_op_kernel_strided(const T * x, T * dst, const int k,const int64_t ne00,const int64_t ne01,const int64_t ne02,const size_t nb00,const size_t nb01,const size_t nb02,const size_t nb03) {+ ggml_cuda_pdl_lc();+ const int i = blockDim.x*blockIdx.x + threadIdx.x;++ if (i >= k) {+ return;+ }++ int64_t rem = i;+ const int64_t i0 = rem % ne00; rem /= ne00;+ const int64_t i1 = rem % ne01; rem /= ne01;+ const int64_t i2 = rem % ne02;+ const int64_t i3 = rem / ne02;+ const size_t src_byte_offset = i0 * nb00 + i1 * nb01 + i2 * nb02 + i3 * nb03;+ const T * src_ptr = (const T *)((const char *)x + src_byte_offset);++ ggml_cuda_pdl_sync();+ dst[i] = ggml_cuda_cast<T>(op(ggml_cuda_cast<float>(*src_ptr)));+}++template <float (*op)(float), typename T>+static void unary_cuda_strided(const T * x, T * dst, const int k,const int64_t ne00,const int64_t ne01,const int64_t ne02,const size_t nb00,const size_t nb01,const size_t nb02,const size_t nb03, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_NEG_BLOCK_SIZE - 1) / CUDA_NEG_BLOCK_SIZE;+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params((dim3)num_blocks, CUDA_NEG_BLOCK_SIZE, 0, stream);+ ggml_cuda_kernel_launch(unary_op_kernel_strided<op, T>, launch_params, x, dst, k, ne00,ne01,ne02,nb00,nb01,nb02,nb03);+}+template <float (*op)(float)>void ggml_cuda_op_unary(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {const ggml_tensor * src0 = dst->src[0];const void * src0_d = src0->data;void * dst_d = dst->data;- cudaStream_t stream = ctx.stream();- GGML_ASSERT(ggml_is_contiguous(src0));+ cudaStream_t stream = ctx.stream();GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);GGML_ASSERT(src0->type == dst->type);- if (src0->type == GGML_TYPE_F16) {- unary_cuda<op>((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), stream);- } else if (src0->type == GGML_TYPE_BF16) {- unary_cuda<op>((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), stream);+ if (ggml_is_contiguous(src0)) {+ if (src0->type == GGML_TYPE_F16) {+ unary_cuda<op>((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), stream);+ } else if (src0->type == GGML_TYPE_BF16) {+ unary_cuda<op>((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), stream);+ } else {+ unary_cuda<op>((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), stream);+ }} else {- unary_cuda<op>((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), stream);+ if (src0->type == GGML_TYPE_F16) {+ unary_cuda_strided<op>((const half *)src0_d, (half *)dst_d, ggml_nelements(src0),+ src0->ne[0], src0->ne[1], src0->ne[2],+ src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream);+ } else if (src0->type == GGML_TYPE_BF16) {+ unary_cuda_strided<op>((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0),+ src0->ne[0], src0->ne[1], src0->ne[2],+ src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream);+ } else {+ unary_cuda_strided<op>((const float *)src0_d, (float *)dst_d, ggml_nelements(src0),+ src0->ne[0], src0->ne[1], src0->ne[2],+ src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream);+ }}}