LLM inference in C/C++
C++ ggml
hexagon: support tiled Q4_0 and Q8_0 GET_ROWS (#29511)
kurquhar7ac59a6e3ad8
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex d95687e345..521a97a730 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -5208,49 +5208,51 @@ static void ggml_hexagon_precompute_get_rows_params(const uint32_t ne12 = src1->ne[2];const uint32_t nr = ne10 * ne11 * ne12;- const size_t nb01 = src0->nb[1];- const size_t nb1 = dst->nb[1];-- const bool can_use_dma = (src0->type == dst->type) && (nb01 == nb1);- const bool use_dma = can_use_dma && (ne00 >= 2048);+ const ggml_tensor * src0_base = src0->view_src ? src0->view_src : src0;+ const auto * extra = src0_base->buffer && ggml_backend_buffer_is_hexagon(src0_base->buffer) ?+ (const ggml_hexagon_tensor_extra *) src0_base->extra : nullptr;+ const bool tiled = src0->type == GGML_TYPE_Q4_0 || (extra && (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0) ||+ sess->needs_repack.count(src0_base) || sess->needs_repack.count(src0);++ if (src0->type == dst->type) {+ kparams->kernel_type = HTP_GET_ROWS_KERNEL_SAMETYPE;+ } else if (tiled) {+ kparams->kernel_type = HTP_GET_ROWS_KERNEL_TILED;+ } else {+ kparams->kernel_type = HTP_GET_ROWS_KERNEL_FLAT;+ }- kparams->use_dma = use_dma ? 1 : 0;+ const uint32_t chunks_per_row = 1;+ const uint32_t chunk_size = ne00;+ const uint32_t total_tasks = nr;- uint32_t chunks_per_row = 1;- uint32_t chunk_size = ne00;- uint32_t total_tasks = nr;+ kparams->n_threads = (std::min)((uint32_t)sess->n_threads, total_tasks);- if (use_dma) {- kparams->n_threads = (std::min)((uint32_t)sess->n_threads, nr);- kparams->tasks_per_thread = (nr + kparams->n_threads - 1) / kparams->n_threads;- } else {- if (src0->type == GGML_TYPE_F32 && nr < sess->n_threads) {- const uint32_t min_chunk_size = 1024;- uint32_t max_chunks = ne00 / min_chunk_size;- if (max_chunks == 0) {- max_chunks = 1;- }- chunks_per_row = (std::min)((sess->n_threads + nr - 1) / nr, max_chunks);- chunk_size = (ne00 + chunks_per_row - 1) / chunks_per_row;- total_tasks = nr * chunks_per_row;+ struct htp_get_rows_vtcm_layout vtcm_layout = {};+ while (kparams->n_threads > 0) {+ htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, src0->type, ne00, kparams->n_threads);+ if (vtcm_layout.total_bytes <= sess->vtcm_size) {+ break;}- kparams->n_threads = (std::min)(total_tasks, (uint32_t)sess->n_threads);- kparams->tasks_per_thread = (total_tasks + kparams->n_threads - 1) / kparams->n_threads;+ --kparams->n_threads;+ }++ if (kparams->n_threads == 0 && total_tasks > 0) {+ htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, src0->type, ne00, 1);}+ kparams->vtcm_size = (total_tasks == 0) ? 0 : vtcm_layout.total_bytes;+ kparams->tasks_per_thread = kparams->n_threads > 0 ? (total_tasks + kparams->n_threads - 1) / kparams->n_threads : 0;+kparams->chunks_per_row = chunks_per_row;kparams->chunk_size = chunk_size;kparams->total_tasks = total_tasks;- kparams->div_ne10 = init_fastdiv_values(ne10);- kparams->div_ne10_ne11 = init_fastdiv_values(ne10 * ne11);- kparams->div_chunks_per_row = init_fastdiv_values(chunks_per_row);- kparams->div_ne02 = init_fastdiv_values(ne02);- kparams->div_ne03 = init_fastdiv_values(ne03);-- struct htp_get_rows_vtcm_layout vtcm_layout;- htp_get_rows_vtcm_layout_build(&vtcm_layout, src0->type, ne00, kparams->n_threads);- kparams->vtcm_size = vtcm_layout.total_bytes;+ kparams->div_ne10 = ne10 > 0 ? init_fastdiv_values(ne10) : fastdiv_values{0, 0};+ kparams->div_ne10_ne11 = (ne10 * ne11) > 0 ? init_fastdiv_values(ne10 * ne11) : fastdiv_values{0, 0};+ kparams->div_chunks_per_row = chunks_per_row > 0 ? init_fastdiv_values(chunks_per_row) : fastdiv_values{0, 0};+ kparams->div_ne02 = ne02 > 0 ? init_fastdiv_values(ne02) : fastdiv_values{0, 0};+ kparams->div_ne03 = ne03 > 0 ? init_fastdiv_values(ne03) : fastdiv_values{0, 0};}static void ggml_hexagon_precompute_set_rows_params(@@ -6176,19 +6178,36 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *const struct ggml_tensor * src1 = op->src[1]; // indicesconst struct ggml_tensor * dst = op;- if (src0->extra) {- const auto * extra = (const ggml_hexagon_tensor_extra *) src0->extra;- if (extra->flags & GGML_HEXAGON_TENSOR_REPACK) {+ if (src0->type == GGML_TYPE_Q4_0 && src0->view_src) {+ return false;+ }++ const ggml_tensor * src0_base = src0->view_src ? src0->view_src : src0;+ bool is_repacked = false;+ if (src0_base->buffer && ggml_backend_buffer_is_hexagon(src0_base->buffer) && src0_base->extra) {+ const auto * extra = (const ggml_hexagon_tensor_extra *) src0_base->extra;+ is_repacked = (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0;+ if (is_repacked && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0) {return false;}}+ is_repacked = is_repacked || sess->needs_repack.count(src0_base) || sess->needs_repack.count(src0);- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->ne[0] < 32) {+ // View offsets use the raw quantized layout and cannot address a tiled allocation.+ if (src0->view_src && is_repacked) {+ return false;+ }++ if (src0->type == GGML_TYPE_Q4_0 && src0->buffer && !is_repacked) {+ return false;+ }++ if (src0->type != dst->type && src0->ne[0] < 32) {return false;}if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 &&- src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) {+ src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) {return false;}@@ -6196,15 +6215,32 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *return false;}- if (src0->type == GGML_TYPE_I32) {- if (dst->type != GGML_TYPE_I32) {+ if (src0->type == dst->type) {+ if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->type != GGML_TYPE_F16) {return false;}+ } else if (src0->type == GGML_TYPE_I32) {+ return false;+ } else if (dst->type != GGML_TYPE_F32) {+ return false;+ }++ // Empty recurrent-state gathers are skipped at execution; do not split the graph for them.+ if (ggml_is_empty(op)) {+ return true;}- else if (dst->type != GGML_TYPE_F32) {++ struct htp_get_rows_kernel_params kparams;+ ggml_hexagon_precompute_get_rows_params(sess, src0, src1, dst, &kparams);+ if (kparams.n_threads == 0 || (size_t) kparams.vtcm_size > sess->vtcm_size) {return false;}+ // Q4_0 has no raw fallback. Mark only accepted tensors for repacking.+ if (src0->type == GGML_TYPE_Q4_0 && !src0->buffer) {+ sess->needs_repack.insert(src0);+ }+return true;GGML_UNUSED(sess);diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.hindex e3677b2671..8ff38545c7 100644--- a/ggml/src/ggml-hexagon/htp-opnode.h+++ b/ggml/src/ggml-hexagon/htp-opnode.h@@ -20,6 +20,7 @@#include "htp/gated-delta-net-ops.h"#include "htp/softmax-ops.h"#include "htp/argsort-ops.h"+#include "htp/get-rows-ops.h"struct htp_opnode {ggml_tensor * node { nullptr };@@ -360,9 +361,7 @@ struct htp_opformat {} else if (node.opcode == HTP_OP_GATED_DELTA_NET) {const auto * kparams = (const struct htp_gdn_kernel_params *) node.kernel_params;const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent";- snprintf(str, max_size, "%s-%s vtcm %u",- path,- kparams->kda ? "kda" : "scalar",+ snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar",(unsigned int) (kparams->vtcm_size ? kparams->vtcm_size : kparams->vtcm_per_thread * kparams->n_threads));} else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID ||node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {@@ -374,9 +373,19 @@ struct htp_opformat {node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort",(int) kparams->n_threads, (int) kparams->n_chunks,(int) kparams->chunk_elems, (int) kparams->vtcm_size);+ } else if (node.opcode == HTP_OP_GET_ROWS) {+ const auto * kparams = (const struct htp_get_rows_kernel_params *) node.kernel_params;+ const char * ktype_str = "unknown";+ switch (kparams->kernel_type) {+ case HTP_GET_ROWS_KERNEL_SAMETYPE: ktype_str = "sametype"; break;+ case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break;+ case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break;+ }+ snprintf(str, max_size, "%s%s vtcm %u", ktype_str, kparams->n_threads > 1 ? "-multi" : "", (unsigned int) kparams->vtcm_size);} else {snprintf(str, max_size, "----");}+}void format(const htp_opnode & node) {diff --git a/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake b/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmakeindex 10840185f8..3eff2a3986 100644--- a/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake+++ b/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake@@ -136,7 +136,7 @@ set(CMAKE_SHARED_LIBRARY_SONAME_C_FLAG "-Wl,-soname,")set(CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG "-Wl,-soname,")# Compiler Options-set(COMMON_FLAGS "${ARCH_FLAGS} -fno-vectorize -fno-slp-vectorize -flto -Wall -Werror -fno-zero-initialized-in-bss -G0 -fdata-sections -fpic ${XQF_ARGS}")+set(COMMON_FLAGS "${ARCH_FLAGS} -fvectorize -flto -Wall -Werror -fno-zero-initialized-in-bss -G0 -fdata-sections -fpic ${XQF_ARGS}")set(CMAKE_CXX_FLAGS_DEBUG "${COMMON_FLAGS} -O0 -D_DEBUG -g")set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${COMMON_FLAGS} -O2 -g")diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.cindex f51e00c15f..f354b813a0 100644--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c@@ -17,6 +17,7 @@#include "htp-tensor.h"#include "hvx-utils.h"#include "hvx-quant.h"+#include "matmul-ops.h"#include "get-rows-ops.h"#include "work-queue.h"@@ -28,6 +29,9 @@ struct get_rows_context {uint32_t task_start;uint32_t tasks;uint32_t tasks_per_thread;+ uint32_t tile_size;+ uint32_t tile_stride;+ bool index_i32;};#define get_rows_preamble \@@ -195,32 +199,191 @@ static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigneddma_queue_flush(dma_q); \}-#define F32_BYTES(n) ((n) * sizeof(float))#define F16_BYTES(n) ((n) * sizeof(__fp16))#define Q8_0_BYTES(n) (((n) / 32) * sizeof(block_q8_0))-GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int32_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); })-GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int64_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); })+static __attribute__((noinline)) void compute_get_rows_f16(float * dst_spad, const void * src_spad, uint32_t cur_elems) {+ hvx_dequantize_row_f16_f32(dst_spad, src_spad, cur_elems);+}++static __attribute__((noinline)) void compute_get_rows_q8_0(float * dst_spad, const void * src_spad, uint32_t cur_elems) {+ hvx_dequantize_row_q8_0_f32(dst_spad, src_spad, cur_elems);+}++GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int32_t, { compute_get_rows_f16((float *)dst_spad, src_spad, cur_elems); })+GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int64_t, { compute_get_rows_f16((float *)dst_spad, src_spad, cur_elems); })++GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); })+GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); })+++static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const uint8_t * tile, uint32_t row, bool q4) {+ const HVX_VectorPred first2 = Q6_Q_vsetq_R(2);+ const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);+ HVX_Vector vq = Q6_V_vzero();+ if (q4) {+ const HVX_VectorPred first1 = Q6_Q_vsetq_R(1);+ const HVX_VectorPred first3 = Q6_Q_vsetq_R(3);+ for (int group = 3; group >= 0; --group) {+ const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), row);+ // Four planes contribute bytes at 0, 32, 64 and 96 after rotation.+ HVX_Vector packed = Q6_V_vmux_QVV(first1, v, Q6_V_vror_VR(v, 31));+ packed = Q6_V_vmux_QVV(first2, packed, Q6_V_vror_VR(v, 62));+ packed = Q6_V_vmux_QVV(first3, packed, Q6_V_vror_VR(v, 93));+ vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));+ }+ const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F));+ const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4);+ vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1));+ vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8));+ } else {+ for (int group = 7; group >= 0; --group) {+ const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), 2 * row);+ // Two planes contribute halfwords at 0 and 64 after rotation.+ const HVX_Vector packed = Q6_V_vmux_QVV(first2, v, Q6_V_vror_VR(v, 62));+ vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));+ }+ }+ const HVX_Vector scales = hvx_vmem(tile + (q4 ? 512 : 1024));+ const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));+ const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));+ const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);+ const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));+ const HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);+ *(HVX_Vector *) dst = values;+}++struct get_rows_tiled_task {+ dma_addr_t tile_src_base;+ dma_addr_t dst_data;+ uint32_t row;+};++static inline struct get_rows_tiled_task get_rows_tiled_calc_task(+ const struct htp_ops_context * octx,+ const struct get_rows_context * grctx,+ uint32_t i,+ uint32_t n_k_tiles,+ uint32_t tile_size+) {+ const struct htp_get_rows_kernel_params * kparams = grctx->kparams;+ get_rows_preamble;++ const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11);+ const uint32_t rem = i - i12 * ne11 * ne10;+ const uint32_t i11 = fastdiv(rem, &kparams->div_ne10);+ const uint32_t i10 = rem - i11 * ne10;+ const dma_addr_t src1_data = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12;+ const uint32_t i01 = grctx->index_i32 ? *(const int32_t *)(uintptr_t) src1_data : (uint32_t) *(const int64_t *)(uintptr_t) src1_data;+ assert(i01 < ne01);++ const uint32_t q02 = fastdiv(i11, &kparams->div_ne02);+ const uint32_t i02 = i11 - q02 * ne02;+ const uint32_t q03 = fastdiv(i12, &kparams->div_ne03);+ const uint32_t i03 = i12 - q03 * ne03;+ const uint32_t column_tile = i01 / HTP_MM_HMX_TILE_N_ROWS;+ const uint32_t row = i01 % HTP_MM_HMX_TILE_N_ROWS;+ const dma_addr_t matrix = octx->src[0]->data + i02*nb02 + i03*nb03;++ struct get_rows_tiled_task task;+ task.tile_src_base = matrix + (column_tile * n_k_tiles) * tile_size;+ task.dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3;+ task.row = row;+ return task;+}++static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * data) {+ struct get_rows_context * grctx = (struct get_rows_context *) data;+ struct htp_ops_context * octx = grctx->octx;+ const struct htp_get_rows_kernel_params * kparams = grctx->kparams;+ get_rows_preamble;++ const uint32_t dr = grctx->tasks_per_thread;+ const uint32_t ir0 = grctx->task_start + dr * ith;+ if (ir0 >= grctx->task_start + grctx->tasks) {+ return;+ }++ const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);+ const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS;+ const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout;+ uint8_t * src_spad_base = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread;+ uint8_t * dst_spad_base = grctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread;+ dma_queue * dma_q = octx->ctx->dma[ith];+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];-GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int32_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); })-GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int64_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); })+ const uint32_t tile_size = grctx->tile_size;+ const uint32_t tile_stride = grctx->tile_stride;+ const uint32_t dst_bytes = ne00 * sizeof(float);+ const bool is_q4 = (octx->src[0]->type == HTP_TYPE_Q4_0);-GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); })-GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); })+ for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, ++spad_idx) {+ const uint32_t i = ir0 + step;+ struct get_rows_tiled_task task = get_rows_tiled_calc_task(octx, grctx, i, n_k_tiles, tile_size);++ // Dummy writeback to prime the queue with dst descriptor+ dma_queue_push(dma_q,+ dma_make_data(task.dst_data, dst_spad_base + spad_idx * vtcm_layout->dst_spad_half_size),+ dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 0);++ // Prefetch row tiles+ dma_queue_push(dma_q,+ dma_make_data(src_spad_base + spad_idx * vtcm_layout->src0_spad_half_size, task.tile_src_base),+ tile_stride, tile_size, tile_size, n_k_tiles);+ }++ for (uint32_t step = 0; step < ir1 - ir0; ++step) {+ const uint32_t i = ir0 + step;+ float * dst_spad = (float *) dma_queue_pop(dma_q).src;+ uint8_t * src_spad = (uint8_t *) dma_queue_pop(dma_q).dst;++ struct get_rows_tiled_task task = get_rows_tiled_calc_task(octx, grctx, i, n_k_tiles, tile_size);++ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);+ for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {+ const uint8_t * tile = src_spad + k_tile * tile_stride;+ float * dst_block = dst_spad + k_tile * HTP_MM_HMX_TILE_N_COLS;+ compute_get_rows_tiled(dst_block, tile, task.row, is_q4);+ }+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);++ // Real writeback of dst_spad+ dma_queue_push(dma_q,+ dma_make_data(task.dst_data, dst_spad),+ dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 1);++ const uint32_t next_step = step + 2;+ if (next_step < ir1 - ir0) {+ const uint32_t ni = ir0 + next_step;+ struct get_rows_tiled_task next_task = get_rows_tiled_calc_task(octx, grctx, ni, n_k_tiles, tile_size);+ dma_queue_push(dma_q,+ dma_make_data(src_spad, next_task.tile_src_base),+ tile_stride, tile_size, tile_size, n_k_tiles);+ }+ }++ dma_queue_flush(dma_q);+}int op_get_rows(struct htp_ops_context * octx) {const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params;if (octx->src[0]->type != HTP_TYPE_F32 &&- octx->src[0]->type != HTP_TYPE_F16 &&- octx->src[0]->type != HTP_TYPE_Q8_0 &&- octx->src[0]->type != HTP_TYPE_I32) {+ octx->src[0]->type != HTP_TYPE_F16 &&+ octx->src[0]->type != HTP_TYPE_Q4_0 &&+ octx->src[0]->type != HTP_TYPE_Q8_0 &&+ octx->src[0]->type != HTP_TYPE_I32) {return HTP_STATUS_NO_SUPPORT;}- if ((octx->src[0]->type == HTP_TYPE_I32 && octx->dst->type != HTP_TYPE_I32) ||- (octx->src[0]->type != HTP_TYPE_I32 && octx->dst->type != HTP_TYPE_F32)) {- return HTP_STATUS_NO_SUPPORT;+ if (kparams->kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) {+ if (octx->src[0]->type != octx->dst->type) {+ return HTP_STATUS_NO_SUPPORT;+ }+ } else {+ if (octx->dst->type != HTP_TYPE_F32) {+ return HTP_STATUS_NO_SUPPORT;+ }}if (octx->src[1]->type != HTP_TYPE_I32 && octx->src[1]->type != HTP_TYPE_I64) {@@ -262,33 +425,48 @@ int op_get_rows(struct htp_ops_context * octx) {grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base;grctx.task_start = task_start;grctx.tasks = tasks;- grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div);+ grctx.tasks_per_thread = octx->ctx->mdev.count == 1 ? kparams->tasks_per_thread : fastdiv(tasks + n_threads - 1, &octx->n_threads_div);+ grctx.tile_size = octx->src[0]->type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0;+ grctx.tile_stride = (grctx.tile_size + 127) & ~127;+ grctx.index_i32 = octx->src[1]->type == HTP_TYPE_I32;const uint32_t ne00 = octx->src[0]->ne[0];- htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, n_threads);+ htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, n_threads);++ if (grctx.vtcm_layout.total_bytes > octx->ctx->vtcm_size) {+ FARF(ERROR, "get-rows: VTCM reservation %zu is too small, needed %zu\n",+ octx->ctx->vtcm_size, grctx.vtcm_layout.total_bytes);+ return HTP_STATUS_INVAL_PARAMS;+ }const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);work_queue_func_t q_func = NULL;- if (kparams->use_dma) {- q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t);- } else {- switch (octx->src[0]->type) {- case HTP_TYPE_F32: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f32_int32_t : get_rows_thread_f32_int64_t); break;- case HTP_TYPE_F16: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t : get_rows_thread_f16_int64_t); break;- case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break;- case HTP_TYPE_I32: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t); break;- default: return HTP_STATUS_NO_SUPPORT;- }+ switch (kparams->kernel_type) {+ case HTP_GET_ROWS_KERNEL_SAMETYPE:+ q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t);+ break;+ case HTP_GET_ROWS_KERNEL_TILED:+ q_func = get_rows_thread_tiled;+ break;+ case HTP_GET_ROWS_KERNEL_FLAT:+ switch (octx->src[0]->type) {+ case HTP_TYPE_F16: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t : get_rows_thread_f16_int64_t); break;+ case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break;+ default: return HTP_STATUS_NO_SUPPORT;+ }+ break;+ default:+ return HTP_STATUS_NO_SUPPORT;}- FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu use-dma %d n-threads %d\n",+ FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-threads %d\n",octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3],octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3],octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3],grctx.vtcm_layout.src0_bytes_per_thread * n_threads,grctx.vtcm_layout.dst_bytes_per_thread * n_threads,- kparams->use_dma, n_threads);+ kparams->kernel_type, n_threads);work_queue_run(octx->ctx->work_queue, q_func, &grctx, n_threads);return HTP_STATUS_OK;diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.hindex 0e7c2ca8cf..06ca1ea74d 100644--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h@@ -1,11 +1,21 @@#ifndef HTP_GET_ROWS_OPS_H#define HTP_GET_ROWS_OPS_H+#include <stdbool.h>+#include <string.h>+#include "hex-fastdiv.h"+#include "matmul-ops.h"++enum htp_get_rows_kernel_type {+ HTP_GET_ROWS_KERNEL_SAMETYPE = 0,+ HTP_GET_ROWS_KERNEL_TILED,+ HTP_GET_ROWS_KERNEL_FLAT,+};struct htp_get_rows_kernel_params {int32_t n_threads;- int32_t use_dma;+ int32_t kernel_type;int32_t chunks_per_row;int32_t chunk_size;int32_t total_tasks;@@ -34,19 +44,37 @@ struct htp_get_rows_vtcm_layout {static inline void htp_get_rows_vtcm_layout_build(struct htp_get_rows_vtcm_layout * vtcm_layout,+ int kernel_type,int type,uint32_t ne00,uint32_t n_threads) {+ if (kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) {+ memset(vtcm_layout, 0, sizeof(*vtcm_layout));+ return;+ }++ if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) {+ const size_t tile_size = type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0;+ const size_t tile_stride = (tile_size + 127) & ~127;+ const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS;+ const size_t row_tiles_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride;+ vtcm_layout->src0_spad_half_size = (row_tiles_size + 255) & ~255;+ vtcm_layout->dst_spad_half_size = (ne00 * sizeof(float) + 255) & ~255;+ vtcm_layout->src0_bytes_per_thread = 2 * vtcm_layout->src0_spad_half_size;+ vtcm_layout->dst_bytes_per_thread = 2 * vtcm_layout->dst_spad_half_size;+ vtcm_layout->off_src0 = 0;+ vtcm_layout->off_dst = vtcm_layout->src0_bytes_per_thread * n_threads;+ vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads;+ return;+ }+uint32_t src0_row_size = 0;switch (type) {- case 0: // HTP_TYPE_F32- src0_row_size = ne00 * 4;- break;- case 1: // HTP_TYPE_F16+ case HTP_TYPE_F16:src0_row_size = ne00 * 2;break;- case 8: // HTP_TYPE_Q8_0+ case HTP_TYPE_Q8_0:src0_row_size = (ne00 / 32) * 34;break;default: