LLM inference in C/C++
C++ ggml
hexagon: Q6_K weight dequant speedup (#30121)
kurquhar06cad0b9e773
diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.hindex 698d6a33b1..40404bf102 100644--- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h+++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h@@ -631,17 +631,40 @@ static void dequantize_tiled_weight_to_fp16_task_q6_k(HVX_Vector v_scale_k16 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc_k16, v_sc_k16, -2));#pragma unroll- for (int g = 0; g < 8; g++) {+ for (int g = 0; g < 8; g += 4) {const HVX_Vector v_scale = (g < 4) ? v_scale_k0 : v_scale_k16;- HVX_Vector v_q = unpack_q6_k_group(vptr, g, mask_0f, mask_03, i32);- HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q);- HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4);-- hvx_vmem(dst_ptr + (2 * g + 0) * 64) =- Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale));- hvx_vmem(dst_ptr + (2 * g + 1) * 64) =- Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale));+ HVX_Vector v_q0 = unpack_q6_k_group(vptr, g + 0, mask_0f, mask_03, i32);+ HVX_Vector v_q1 = unpack_q6_k_group(vptr, g + 1, mask_0f, mask_03, i32);+ HVX_Vector v_q2 = unpack_q6_k_group(vptr, g + 2, mask_0f, mask_03, i32);+ HVX_Vector v_q3 = unpack_q6_k_group(vptr, g + 3, mask_0f, mask_03, i32);++ HVX_VectorPair vp16_0 = Q6_Wh_vunpack_Vb(v_q0);+ HVX_VectorPair vp16_1 = Q6_Wh_vunpack_Vb(v_q1);+ HVX_VectorPair vp16_2 = Q6_Wh_vunpack_Vb(v_q2);+ HVX_VectorPair vp16_3 = Q6_Wh_vunpack_Vb(v_q3);+ HVX_VectorPair vp_k0 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_0), Q6_V_lo_W(vp16_0), -4);+ HVX_VectorPair vp_k1 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_1), Q6_V_lo_W(vp16_1), -4);+ HVX_VectorPair vp_k2 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_2), Q6_V_lo_W(vp16_2), -4);+ HVX_VectorPair vp_k3 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_3), Q6_V_lo_W(vp16_3), -4);++ HVX_Vector v_out00 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k0)), v_scale));+ HVX_Vector v_out01 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k0)), v_scale));+ HVX_Vector v_out10 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k1)), v_scale));+ HVX_Vector v_out11 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k1)), v_scale));+ HVX_Vector v_out20 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k2)), v_scale));+ HVX_Vector v_out21 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k2)), v_scale));+ HVX_Vector v_out30 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k3)), v_scale));+ HVX_Vector v_out31 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k3)), v_scale));++ hvx_vmem(dst_ptr + (2 * g + 0) * 64) = v_out00;+ hvx_vmem(dst_ptr + (2 * g + 1) * 64) = v_out01;+ hvx_vmem(dst_ptr + (2 * g + 2) * 64) = v_out10;+ hvx_vmem(dst_ptr + (2 * g + 3) * 64) = v_out11;+ hvx_vmem(dst_ptr + (2 * g + 4) * 64) = v_out20;+ hvx_vmem(dst_ptr + (2 * g + 5) * 64) = v_out21;+ hvx_vmem(dst_ptr + (2 * g + 6) * 64) = v_out30;+ hvx_vmem(dst_ptr + (2 * g + 7) * 64) = v_out31;}}}