Skip to content

LLM inference in C/C++

C++ ggml

hip : optimize Q2_0 dot-product path for gfx1201 (#26753)

LunalFreshe42214804794
diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuh
index 8ed704c281..7f00bad943 100644
--- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh
+++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh
@@ -138,12 +138,20 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
for (int j = 0; j < 4; ++j) {
const int q = qxi[j];
+#if defined(GGML_USE_HIP)
+ const uint32_t qx_indices = (q & 0x03) | ((q & 0x0C) << 6) | ((q & 0x30) << 12) | ((q & 0xC0) << 18);
+ const uint32_t qy_bits = q >> 8;
+ const uint32_t qy_indices = (qy_bits & 0x03) | ((qy_bits & 0x0C) << 6) | ((qy_bits & 0x30) << 12) | ((qy_bits & 0xC0) << 18);
+ const int qx = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qx_indices);
+ const int qy = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qy_indices);
+#else
// unpack even and odd crumbs into byte values
const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0);
const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2);
// unshuffle values
const int qx = __byte_perm(qe, qo, 0x5140);
const int qy = __byte_perm(qe, qo, 0x7362);
+#endif // defined(GGML_USE_HIP)
#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
x_qs[i*sram_stride + dst_offset + j*2+0] = qx;
diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuh
index 0f039c735b..ec117c57df 100644
--- a/ggml/src/ggml-cuda/vecdotq.cuh
+++ b/ggml/src/ggml-cuda/vecdotq.cuh
@@ -747,12 +747,20 @@ static __device__ __forceinline__ float vec_dot_q2_0_q8_1(
const int u = get_int_b4(bq8_1_chunk->qs, j*2+0);
const int v = get_int_b4(bq8_1_chunk->qs, j*2+1);
+#if defined(GGML_USE_HIP)
+ const uint32_t qx_indices = (q & 0x03) | ((q & 0x0C) << 6) | ((q & 0x30) << 12) | ((q & 0xC0) << 18);
+ const uint32_t qy_bits = q >> 8;
+ const uint32_t qy_indices = (qy_bits & 0x03) | ((qy_bits & 0x0C) << 6) | ((qy_bits & 0x30) << 12) | ((qy_bits & 0xC0) << 18);
+ const int qx = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qx_indices);
+ const int qy = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qy_indices);
+#else
// unpack even and odd crumbs into byte values
const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0);
const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2);
// unshuffle values
const int qx = __byte_perm(qe, qo, 0x5140);
const int qy = __byte_perm(qe, qo, 0x7362);
+#endif // defined(GGML_USE_HIP)
sumi = ggml_cuda_dp4a(u, qx, sumi);
sumi = ggml_cuda_dp4a(v, qy, sumi);