LLM inference in C/C++
C++ ggml
hexagon: support tiled Q4_K and Q6_K GET_ROWS (#30115)
kurquharaa5e0092fdb8
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex f0d7683887..26b79582e8 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -7232,7 +7232,7 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *const struct ggml_tensor * src1 = op->src[1]; // indicesconst struct ggml_tensor * dst = op;- if (src0->type == GGML_TYPE_Q4_0 && src0->view_src) {+ if ((src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_K || src0->type == GGML_TYPE_Q6_K) && src0->view_src) {return false;}@@ -7241,7 +7241,7 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *if (src0_base->buffer && ggml_backend_buffer_is_hexagon(src0_base->buffer) && src0_base->extra) {const auto * extra = (const ggml_hexagon_tensor_extra *) src0_base->extra;is_repacked = (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0;- if (is_repacked && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0) {+ if (is_repacked && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_Q8_0) {return false;}}@@ -7252,7 +7252,7 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *return false;}- if (src0->type == GGML_TYPE_Q4_0 && src0->buffer && !is_repacked) {+ if ((src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_K || src0->type == GGML_TYPE_Q6_K) && src0->buffer && ggml_backend_buffer_get_size(src0->buffer) != 0 && !is_repacked) {return false;}@@ -7261,7 +7261,11 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *}if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 &&- src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) {+ src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) {+ return false;+ }++ if ((src0->type == GGML_TYPE_Q4_K || src0->type == GGML_TYPE_Q6_K) && (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || src0->ne[0] % QK_K)) {return false;}@@ -7290,8 +7294,8 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *return false;}- // Q4_0 has no raw fallback. Mark only accepted tensors for repacking.- if (src0->type == GGML_TYPE_Q4_0 && !src0->buffer) {+ // Tiled quantized weights have no raw fallback. Mark only accepted tensors for repacking.+ if ((src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_K || src0->type == GGML_TYPE_Q6_K) && !src0->buffer) {sess->needs_repack.insert(src0);}diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.cindex f354b813a0..90f38fa56b 100644--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c@@ -217,7 +217,7 @@ GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { compute_get_rows_q8_0((floatGET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); })-static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const uint8_t * tile, uint32_t row, bool q4) {+static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const uint8_t * tile, uint32_t row, bool q4, bool q4_k) {const HVX_VectorPred first2 = Q6_Q_vsetq_R(2);const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);HVX_Vector vq = Q6_V_vzero();@@ -235,7 +235,9 @@ static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, constconst HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F));const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4);vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1));- vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8));+ if (!q4_k) {+ vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8));+ }} else {for (int group = 7; group >= 0; --group) {const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), 2 * row);@@ -245,14 +247,46 @@ static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const}}const HVX_Vector scales = hvx_vmem(tile + (q4 ? 512 : 1024));- const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));+ const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, (q4_k ? 4 : 2) * row));const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));- const HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);+ HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);+ if (q4_k) {+ const HVX_Vector offset_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row + 2));+ const HVX_Vector offset = Q6_V_lo_W(hvx_vec_f16_to_f32(offset_hf));+ values = hvx_vec_add_f32_f32(values, offset);+ }*(HVX_Vector *) dst = values;}+static __attribute__((noinline)) void compute_get_rows_q6_k(float * dst, const uint8_t * tile, uint32_t row) {+ const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);+ const HVX_VectorPred first16 = Q6_Q_vsetq_R(16 * sizeof(float));+ const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F);+ const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);+ HVX_Vector vq = Q6_V_vzero();++ for (int group = 7; group >= 0; --group) {+ const HVX_Vector lo_plane = Q6_V_vror_VR(hvx_vmem(tile + (group >> 1) * VLEN), 4 * row);+ const HVX_Vector hi_plane = Q6_V_vror_VR(hvx_vmem(tile + 512 + (group >> 2) * VLEN), 4 * row);+ const HVX_Vector lo = (group & 1) ? Q6_Vub_vlsr_VubR(lo_plane, 4) : Q6_V_vand_VV(lo_plane, mask_0f);+ const HVX_Vector hi = Q6_Vub_vlsr_VubR(hi_plane, 2 * (group & 3));+ const HVX_Vector packed = Q6_V_vor_VV(lo, Q6_Vw_vasl_VwR(Q6_V_vand_VV(hi, mask_03), 4));+ vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));+ }++ const HVX_Vector scales = hvx_vmem(tile + 768);+ const HVX_Vector scale_lo_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));+ const HVX_Vector scale_hi_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 64 + 2 * row));+ const HVX_Vector scale_lo = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_lo_hf));+ const HVX_Vector scale_hi = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hi_hf));+ const HVX_Vector scale = Q6_V_vmux_QVV(first16, scale_lo, scale_hi);+ const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(32)));+ const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));+ *(HVX_Vector *) dst = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);+}+struct get_rows_tiled_task {dma_addr_t tile_src_base;dma_addr_t dst_data;@@ -315,7 +349,9 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * datconst uint32_t tile_size = grctx->tile_size;const uint32_t tile_stride = grctx->tile_stride;const uint32_t dst_bytes = ne00 * sizeof(float);- const bool is_q4 = (octx->src[0]->type == HTP_TYPE_Q4_0);+ const bool is_q4 = octx->src[0]->type == HTP_TYPE_Q4_0 || octx->src[0]->type == HTP_TYPE_Q4_K;+ const bool is_q4_k = octx->src[0]->type == HTP_TYPE_Q4_K;+ const bool is_q6_k = octx->src[0]->type == HTP_TYPE_Q6_K;for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, ++spad_idx) {const uint32_t i = ir0 + step;@@ -343,7 +379,11 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * datfor (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {const uint8_t * tile = src_spad + k_tile * tile_stride;float * dst_block = dst_spad + k_tile * HTP_MM_HMX_TILE_N_COLS;- compute_get_rows_tiled(dst_block, tile, task.row, is_q4);+ if (is_q6_k) {+ compute_get_rows_q6_k(dst_block, tile, task.row);+ } else {+ compute_get_rows_tiled(dst_block, tile, task.row, is_q4, is_q4_k);+ }}htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);@@ -369,10 +409,12 @@ int op_get_rows(struct htp_ops_context * octx) {const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params;if (octx->src[0]->type != HTP_TYPE_F32 &&- octx->src[0]->type != HTP_TYPE_F16 &&- octx->src[0]->type != HTP_TYPE_Q4_0 &&- octx->src[0]->type != HTP_TYPE_Q8_0 &&- octx->src[0]->type != HTP_TYPE_I32) {+ octx->src[0]->type != HTP_TYPE_F16 &&+ octx->src[0]->type != HTP_TYPE_Q4_0 &&+ octx->src[0]->type != HTP_TYPE_Q4_K &&+ octx->src[0]->type != HTP_TYPE_Q6_K &&+ octx->src[0]->type != HTP_TYPE_Q8_0 &&+ octx->src[0]->type != HTP_TYPE_I32) {return HTP_STATUS_NO_SUPPORT;}@@ -426,7 +468,7 @@ int op_get_rows(struct htp_ops_context * octx) {grctx.task_start = task_start;grctx.tasks = tasks;grctx.tasks_per_thread = octx->ctx->mdev.count == 1 ? kparams->tasks_per_thread : fastdiv(tasks + n_threads - 1, &octx->n_threads_div);- grctx.tile_size = octx->src[0]->type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0;+ grctx.tile_size = htp_mm_get_weight_tile_size(octx->src[0]->type);grctx.tile_stride = (grctx.tile_size + 127) & ~127;grctx.index_i32 = octx->src[1]->type == HTP_TYPE_I32;diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.hindex 06ca1ea74d..69fa544649 100644--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h@@ -55,7 +55,7 @@ static inline void htp_get_rows_vtcm_layout_build(}if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) {- const size_t tile_size = type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0;+ const size_t tile_size = htp_mm_get_weight_tile_size(type);const size_t tile_stride = (tile_size + 127) & ~127;const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS;const size_t row_tiles_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride;