LLM inference in C/C++
C++ ggml
hip : optimize Q2_0 dot-product path for gfx1201 (#26753)
LunalFreshe42214804794
diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuhindex 8ed704c281..7f00bad943 100644--- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh+++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh@@ -138,12 +138,20 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_for (int j = 0; j < 4; ++j) {const int q = qxi[j];+#if defined(GGML_USE_HIP)+ const uint32_t qx_indices = (q & 0x03) | ((q & 0x0C) << 6) | ((q & 0x30) << 12) | ((q & 0xC0) << 18);+ const uint32_t qy_bits = q >> 8;+ const uint32_t qy_indices = (qy_bits & 0x03) | ((qy_bits & 0x0C) << 6) | ((qy_bits & 0x30) << 12) | ((qy_bits & 0xC0) << 18);+ const int qx = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qx_indices);+ const int qy = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qy_indices);+#else// unpack even and odd crumbs into byte valuesconst int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0);const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2);// unshuffle valuesconst int qx = __byte_perm(qe, qo, 0x5140);const int qy = __byte_perm(qe, qo, 0x7362);+#endif // defined(GGML_USE_HIP)#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)x_qs[i*sram_stride + dst_offset + j*2+0] = qx;diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuhindex 0f039c735b..ec117c57df 100644--- a/ggml/src/ggml-cuda/vecdotq.cuh+++ b/ggml/src/ggml-cuda/vecdotq.cuh@@ -747,12 +747,20 @@ static __device__ __forceinline__ float vec_dot_q2_0_q8_1(const int u = get_int_b4(bq8_1_chunk->qs, j*2+0);const int v = get_int_b4(bq8_1_chunk->qs, j*2+1);+#if defined(GGML_USE_HIP)+ const uint32_t qx_indices = (q & 0x03) | ((q & 0x0C) << 6) | ((q & 0x30) << 12) | ((q & 0xC0) << 18);+ const uint32_t qy_bits = q >> 8;+ const uint32_t qy_indices = (qy_bits & 0x03) | ((qy_bits & 0x0C) << 6) | ((qy_bits & 0x30) << 12) | ((qy_bits & 0xC0) << 18);+ const int qx = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qx_indices);+ const int qy = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qy_indices);+#else// unpack even and odd crumbs into byte valuesconst int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0);const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2);// unshuffle valuesconst int qx = __byte_perm(qe, qo, 0x5140);const int qy = __byte_perm(qe, qo, 0x7362);+#endif // defined(GGML_USE_HIP)sumi = ggml_cuda_dp4a(u, qx, sumi);sumi = ggml_cuda_dp4a(v, qy, sumi);