LLM inference in C/C++
C++ ggml
ggml-cuda : convert contiguous tensors four elements at a time (#29155)
Piotr Wilkin (ilintar)f4e276a2066a
diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cuindex 360c614a44..0619f4760e 100644--- a/ggml/src/ggml-cuda/convert.cu+++ b/ggml/src/ggml-cuda/convert.cu@@ -439,6 +439,29 @@ static __global__ void convert_unary(}}+template <typename T> struct alignas(sizeof(T)*4) cvt_vec4 { T v[4]; };++// four elements per thread, so a warp moves 512B (RDNA) / 1k (CDNA) per load+template <typename src_t, typename dst_t>+static __global__ void convert_unary_cont_vec4(+ const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k4) {+ const int64_t i = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;++ if (i >= k4) {+ return;+ }++ const cvt_vec4<src_t> xv = ((const cvt_vec4<src_t> *) vx)[i];++ cvt_vec4<dst_t> yv;+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ yv.v[j] = ggml_cuda_cast<dst_t>(xv.v[j]);+ }++ ((cvt_vec4<dst_t> *) y)[i] = yv;+}+template <typename src_t, typename dst_t>static void convert_unary_cuda(const void * vx, dst_t * y,const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,@@ -452,6 +475,15 @@ static void convert_unary_cuda(const void * vx, dst_t * y,template <typename src_t, typename dst_t>static void convert_unary_cont_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ if (k % 4 == 0 &&+ (uintptr_t) vx % alignof(cvt_vec4<src_t>) == 0 &&+ (uintptr_t) y % alignof(cvt_vec4<dst_t>) == 0) {+ const int64_t k4 = k/4;+ const int64_t num_blocks = (k4 + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;+ convert_unary_cont_vec4<src_t, dst_t><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k4);+ return;+ }+convert_unary_cuda<src_t>(vx, y, k, 1, 1, 1, k, k, k, stream);}