LLM inference in C/C++
C++ ggml
opencl: tune the quant paths for Intel Xe-LP GPUs to improve its TG and PP performance (#26438)
Hongqiang Wang010be9683afa
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 90635cc858..34d58f4ee8 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -20002,7 +20002,8 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co}kernel = backend_ctx->kernel_mul_mm_q4_k_f32_l4_lm;- nth0 = 128; // calculated as (BM*BN)/(TM*TN)+ // (BM*BN)/(TM*TN): Intel uses an 8x8 microtile (WG=64), others 4x8 (WG=128)+ nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128;int batch_stride_a = ne00*ne01;int batch_stride_b = ne10*ne11;@@ -20046,7 +20047,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co}kernel = backend_ctx->kernel_mul_mm_q5_k_f32_l4_lm;- nth0 = 128; // calculated as (BM*BN)/(TM*TN)+ nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; // Intel 8x8 microtileint batch_stride_a = ne00*ne01;int batch_stride_b = ne10*ne11;@@ -20860,7 +20861,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, coif (backend_ctx->gpu_family == INTEL) {nth0 = 16;nth1 = 1;- ndst = 4;+ ndst = 16; // 8->16 rows per subgroup — matches N_DST in mul_mv_q4_k_f32_flat.cl (32 spills)} else if (backend_ctx->gpu_family == ADRENO) {nth0 = 64;nth1 = 2;@@ -20934,7 +20935,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, coif (backend_ctx->gpu_family == INTEL) {nth0 = 16;nth1 = 1;- ndst = 4;+ ndst = 8; // 4->8 rows per subgroup (2x activation reuse)} else if (backend_ctx->gpu_family == ADRENO) {nth0 = 64;nth1 = 2;diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.clindex 2235b1ae83..a9c649a521 100644--- a/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl+++ b/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl@@ -1,13 +1,23 @@#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#ifdef cl_intel_required_subgroup_size+#define INTEL_GPU 1+#endif+#define LOAD_VEC_A 4#define LOAD_VEC_B 4#define BM 64#define BN 64#define BK 32+#ifdef INTEL_GPU+// Intel Xe iGPU: 8x8 microtile (WG = BM*BN/(TM*TN) = 64) — ~+12% pp512 vs 4x8+#define TM 8+#define TN 8+#else#define TM 4#define TN 8+#endifkernel void kernel_mul_mm_q4_k_f32_l4_lm(global uchar4 * src0_q,diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.clindex 8e191f57e8..a343b5c4c6 100644--- a/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl+++ b/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl@@ -1,13 +1,23 @@#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#ifdef cl_intel_required_subgroup_size+#define INTEL_GPU 1+#endif+#define LOAD_VEC_A 4#define LOAD_VEC_B 4#define BM 64#define BN 64#define BK 32+#ifdef INTEL_GPU+// Intel Xe iGPU: 8x8 microtile (WG=64)+#define TM 8+#define TN 8+#else#define TM 4#define TN 8+#endifkernel void kernel_mul_mm_q5_k_f32_l4_lm(global uchar4 * src0_q,diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.clindex 70391866ca..5316bd3636 100644--- a/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl+++ b/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl@@ -40,7 +40,7 @@ typedef struct {#undef N_SIMDWIDTH#ifdef INTEL_GPU-#define N_DST 4 // number of rows each SIMD group works on+#define N_DST 16 // number of rows each SIMD group works on (Intel: 8->16, 2x further activation reuse; 32 spills registers)#define N_SIMDGROUP 1 // number of SIMD groups in a thread group#define N_SIMDWIDTH 16 // SIMD group size#elif defined (ADRENO_GPU)diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.clindex 6020364b5c..ab2e1fab8b 100644--- a/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl+++ b/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl@@ -38,7 +38,7 @@ typedef struct {#undef N_SIMDWIDTH#ifdef INTEL_GPU-#define N_DST 4+#define N_DST 8 // Intel: 4->8 for 2x activation reuse (see mul_mv_q4_k_f32_flat.cl)#define N_SIMDGROUP 1#define N_SIMDWIDTH 16#elif defined(ADRENO_GPU)