Skip to content

LLM inference in C/C++

C++ ggml

ggml-cuda : convert contiguous tensors four elements at a time (#29155)

Piotr Wilkin (ilintar)f4e276a2066a
diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu
index 360c614a44..0619f4760e 100644
--- a/ggml/src/ggml-cuda/convert.cu
+++ b/ggml/src/ggml-cuda/convert.cu
@@ -439,6 +439,29 @@ static __global__ void convert_unary(
}
}
+template <typename T> struct alignas(sizeof(T)*4) cvt_vec4 { T v[4]; };
+
+// four elements per thread, so a warp moves 512B (RDNA) / 1k (CDNA) per load
+template <typename src_t, typename dst_t>
+static __global__ void convert_unary_cont_vec4(
+ const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k4) {
+ const int64_t i = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k4) {
+ return;
+ }
+
+ const cvt_vec4<src_t> xv = ((const cvt_vec4<src_t> *) vx)[i];
+
+ cvt_vec4<dst_t> yv;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ yv.v[j] = ggml_cuda_cast<dst_t>(xv.v[j]);
+ }
+
+ ((cvt_vec4<dst_t> *) y)[i] = yv;
+}
+
template <typename src_t, typename dst_t>
static void convert_unary_cuda(const void * vx, dst_t * y,
const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,
@@ -452,6 +475,15 @@ static void convert_unary_cuda(const void * vx, dst_t * y,
template <typename src_t, typename dst_t>
static void convert_unary_cont_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ if (k % 4 == 0 &&
+ (uintptr_t) vx % alignof(cvt_vec4<src_t>) == 0 &&
+ (uintptr_t) y % alignof(cvt_vec4<dst_t>) == 0) {
+ const int64_t k4 = k/4;
+ const int64_t num_blocks = (k4 + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;
+ convert_unary_cont_vec4<src_t, dst_t><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k4);
+ return;
+ }
+
convert_unary_cuda<src_t>(vx, y, k, 1, 1, 1, k, k, k, stream);
}