Skip to content

LLM inference in C/C++

C++ ggml

opencl: tune the quant paths for Intel Xe-LP GPUs to improve its TG and PP performance (#26438)

Hongqiang Wang010be9683afa
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 90635cc858..34d58f4ee8 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -20002,7 +20002,8 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
}
kernel = backend_ctx->kernel_mul_mm_q4_k_f32_l4_lm;
- nth0 = 128; // calculated as (BM*BN)/(TM*TN)
+ // (BM*BN)/(TM*TN): Intel uses an 8x8 microtile (WG=64), others 4x8 (WG=128)
+ nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128;
int batch_stride_a = ne00*ne01;
int batch_stride_b = ne10*ne11;
@@ -20046,7 +20047,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
}
kernel = backend_ctx->kernel_mul_mm_q5_k_f32_l4_lm;
- nth0 = 128; // calculated as (BM*BN)/(TM*TN)
+ nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; // Intel 8x8 microtile
int batch_stride_a = ne00*ne01;
int batch_stride_b = ne10*ne11;
@@ -20860,7 +20861,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
if (backend_ctx->gpu_family == INTEL) {
nth0 = 16;
nth1 = 1;
- ndst = 4;
+ ndst = 16; // 8->16 rows per subgroup — matches N_DST in mul_mv_q4_k_f32_flat.cl (32 spills)
} else if (backend_ctx->gpu_family == ADRENO) {
nth0 = 64;
nth1 = 2;
@@ -20934,7 +20935,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
if (backend_ctx->gpu_family == INTEL) {
nth0 = 16;
nth1 = 1;
- ndst = 4;
+ ndst = 8; // 4->8 rows per subgroup (2x activation reuse)
} else if (backend_ctx->gpu_family == ADRENO) {
nth0 = 64;
nth1 = 2;
diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl
index 2235b1ae83..a9c649a521 100644
--- a/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl
+++ b/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl
@@ -1,13 +1,23 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#ifdef cl_intel_required_subgroup_size
+#define INTEL_GPU 1
+#endif
+
#define LOAD_VEC_A 4
#define LOAD_VEC_B 4
#define BM 64
#define BN 64
#define BK 32
+#ifdef INTEL_GPU
+// Intel Xe iGPU: 8x8 microtile (WG = BM*BN/(TM*TN) = 64) — ~+12% pp512 vs 4x8
+#define TM 8
+#define TN 8
+#else
#define TM 4
#define TN 8
+#endif
kernel void kernel_mul_mm_q4_k_f32_l4_lm(
global uchar4 * src0_q,
diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl
index 8e191f57e8..a343b5c4c6 100644
--- a/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl
+++ b/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl
@@ -1,13 +1,23 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#ifdef cl_intel_required_subgroup_size
+#define INTEL_GPU 1
+#endif
+
#define LOAD_VEC_A 4
#define LOAD_VEC_B 4
#define BM 64
#define BN 64
#define BK 32
+#ifdef INTEL_GPU
+// Intel Xe iGPU: 8x8 microtile (WG=64)
+#define TM 8
+#define TN 8
+#else
#define TM 4
#define TN 8
+#endif
kernel void kernel_mul_mm_q5_k_f32_l4_lm(
global uchar4 * src0_q,
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl
index 70391866ca..5316bd3636 100644
--- a/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl
@@ -40,7 +40,7 @@ typedef struct {
#undef N_SIMDWIDTH
#ifdef INTEL_GPU
-#define N_DST 4 // number of rows each SIMD group works on
+#define N_DST 16 // number of rows each SIMD group works on (Intel: 8->16, 2x further activation reuse; 32 spills registers)
#define N_SIMDGROUP 1 // number of SIMD groups in a thread group
#define N_SIMDWIDTH 16 // SIMD group size
#elif defined (ADRENO_GPU)
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl
index 6020364b5c..ab2e1fab8b 100644
--- a/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl
@@ -38,7 +38,7 @@ typedef struct {
#undef N_SIMDWIDTH
#ifdef INTEL_GPU
-#define N_DST 4
+#define N_DST 8 // Intel: 4->8 for 2x activation reuse (see mul_mv_q4_k_f32_flat.cl)
#define N_SIMDGROUP 1
#define N_SIMDWIDTH 16
#elif defined(ADRENO_GPU)