LLM inference in C/C++
C++ ggml
hexagon: add support for hadamard MUL_MAT and GET_ROW for flat and tiled Q1_0 and Q2_0 (#30627)
Todor Boinovski848bbda9041d
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex 6e714ff3b2..605c23c68c 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -5359,10 +5359,38 @@ static void ggml_hexagon_precompute_fused_mmidnx_params(kparams->n_weights = n_weights;}+static bool ggml_hexagon_op_mul_mat_use_fwht(const struct ggml_tensor * dst) {+ if (dst->op != GGML_OP_MUL_MAT || ggml_get_op_params_i32(dst, 1) != GGML_HINT_SRC0_IS_HADAMARD) {+ return false;+ }++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ const int64_t n = src1->ne[0];+ if (n < 32 || (n & (n - 1)) != 0) {+ return false;+ }++ return src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 &&+ (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) &&+ ggml_is_contiguous_rows(src1) && ggml_is_contiguous(dst) &&+ ggml_are_same_shape(src1, dst);+}+static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];+ if (ggml_get_op_params_i32(dst, 1) == GGML_HINT_SRC0_IS_HADAMARD) {+ if (!ggml_hexagon_op_mul_mat_use_fwht(dst)) {+ return false;+ }+ struct htp_unary_kernel_params ukparams;+ ggml_hexagon_precompute_unary_params(sess, HTP_OP_HADAMARD, src1, nullptr, dst, &ukparams);+ return ukparams.vtcm_row_per_thread > 0 && (size_t) ukparams.vtcm_size <= sess->vtcm_size;+ }+if (dst->type != GGML_TYPE_F32) {return false;}@@ -5921,8 +5949,9 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *const struct ggml_tensor * src1 = op->src[1]; // indicesconst struct ggml_tensor * dst = op;- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16 &&- src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) {+ if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16 && src0->type != GGML_TYPE_I32 &&+ src0->type != GGML_TYPE_Q1_0 && src0->type != GGML_TYPE_Q2_0 && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 &&+ src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K ) {return false;}@@ -6666,7 +6695,11 @@ static void ggml_backend_hexagon_free(ggml_backend_t backend) {static htp_op_code op_remap_to_htp(const ggml_tensor * t) {switch (t->op) {case GGML_OP_FLASH_ATTN_EXT: return HTP_OP_FLASH_ATTN_EXT;- case GGML_OP_MUL_MAT: return HTP_OP_MUL_MAT;+ case GGML_OP_MUL_MAT:+ if (ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD) {+ return HTP_OP_HADAMARD;+ }+ return HTP_OP_MUL_MAT;case GGML_OP_MUL_MAT_ID: return HTP_OP_MUL_MAT_ID;case GGML_OP_MUL: return HTP_OP_MUL;case GGML_OP_ADD: return HTP_OP_ADD;@@ -6795,6 +6828,7 @@ static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const ststatic bool is_mergeable_mul_mat(const ggml_tensor * t) {if (t->op != GGML_OP_MUL_MAT) return false;+ if (ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD) return false;const ggml_tensor * src0 = t->src[0];const ggml_tensor * src1 = t->src[1];@@ -6897,7 +6931,9 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggif (graph->nodes[i]->op == GGML_OP_RMS_NORM && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE;- } else if (graph->nodes[i]->op == GGML_OP_MUL_MAT || graph->nodes[i]->op == GGML_OP_MUL_MAT_ID) {+ } else if ((graph->nodes[i]->op == GGML_OP_MUL_MAT &&+ ggml_get_op_params_i32(graph->nodes[i], 1) != GGML_HINT_SRC0_IS_HADAMARD) ||+ graph->nodes[i]->op == GGML_OP_MUL_MAT_ID) {if ((i + 1 < graph->n_nodes && graph->nodes[i + 1]->op == GGML_OP_ADD && ggml_can_fuse(graph, i, { graph->nodes[i]->op, GGML_OP_ADD })) ||ggml_node_has_n_uses(graph, i, 1)) {extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE;diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.hindex d62ce1d000..57cd5f208d 100644--- a/ggml/src/ggml-hexagon/htp-opnode.h+++ b/ggml/src/ggml-hexagon/htp-opnode.h@@ -47,13 +47,19 @@ struct htp_opnode {void init(ggml_tensor * node) {this->node = node;if (this->node) {- this->name = ggml_op_desc(this->node);-- // Build inputs (preserving optional nullptrs)- int n_inputs = n_active_src(this->node);- this->inputs.resize(n_inputs, nullptr);- for (int i = 0; i < n_inputs; i++) {- this->inputs[i] = this->node->src[i];+ if (this->node->op == GGML_OP_MUL_MAT &&+ ggml_get_op_params_i32(this->node, 1) == GGML_HINT_SRC0_IS_HADAMARD) {+ this->name = "MUL_MAT_HADAMARD";+ this->inputs.push_back(this->node->src[1]);+ } else {+ this->name = ggml_op_desc(this->node);++ // Build inputs (preserving optional nullptrs)+ int n_inputs = n_active_src(this->node);+ this->inputs.resize(n_inputs, nullptr);+ for (int i = 0; i < n_inputs; i++) {+ this->inputs[i] = this->node->src[i];+ }}// Build outputs@@ -124,6 +130,10 @@ struct htp_opnode {bool stackable() const {switch (this->op()) {case GGML_OP_MUL_MAT:+ if (ggml_get_op_params_i32(this->node, 1) == GGML_HINT_SRC0_IS_HADAMARD) {+ return false;+ }+ return ggml_is_quantized(this->src0()->type);case GGML_OP_MUL_MAT_ID:return ggml_is_quantized(this->src0()->type);default:diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.cindex 27948c37ac..78eb1bb500 100644--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c@@ -288,7 +288,7 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat}const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);- const uint32_t n_k_tiles = ne00 / HTP_TENSOR_TILE_N_COLS;+ const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(octx->src[0]->type);const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout;uint8_t * src_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread;uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_thread;@@ -450,7 +450,9 @@ int op_get_rows(struct htp_ops_context * octx) {break;case HTP_GET_ROWS_KERNEL_TILED:switch (octx->src[0]->type) {- case HTP_TYPE_F16: grctx.compute_tiled = hvx_dequantize_row_tiled_f16_f32; break;+ case HTP_TYPE_F16: grctx.compute_tiled = hvx_dequantize_row_tiled_f16_f32; break;+ case HTP_TYPE_Q1_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q1_0_f32; break;+ case HTP_TYPE_Q2_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q2_0_f32; break;case HTP_TYPE_Q8_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q8_0_f32; break;case HTP_TYPE_Q4_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_0_f32; break;case HTP_TYPE_Q4_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_k_f32; break;@@ -466,6 +468,8 @@ int op_get_rows(struct htp_ops_context * octx) {switch (octx->src[0]->type) {case HTP_TYPE_F16: grctx.compute_flat = hvx_dequantize_row_flat_f16_f32; break;case HTP_TYPE_BF16: grctx.compute_flat = hvx_dequantize_row_flat_bf16_f32; break;+ case HTP_TYPE_Q1_0: grctx.compute_flat = hvx_dequantize_row_flat_q1_0_f32; break;+ case HTP_TYPE_Q2_0: grctx.compute_flat = hvx_dequantize_row_flat_q2_0_f32; break;case HTP_TYPE_Q8_0: grctx.compute_flat = hvx_dequantize_row_flat_q8_0_f32; break;case HTP_TYPE_Q4_0: grctx.compute_flat = hvx_dequantize_row_flat_q4_0_f32; break;case HTP_TYPE_Q4_K: grctx.compute_flat = hvx_dequantize_row_flat_q4_k_f32; break;diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.hindex f44e58d164..94f3c15f29 100644--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h@@ -64,7 +64,7 @@ static inline void htp_get_rows_vtcm_layout_build(size_t src0_row_size = 0;if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) {const size_t tile_stride = htp_tensor_get_tile_padded_size(type);- const uint32_t n_k_tiles = ne00 / HTP_TENSOR_TILE_N_COLS;+ const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(type);src0_row_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride;} else {src0_row_size = htp_tensor_get_row_size(type, ne00);diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.hindex 3fc5581192..9b4b86d3b9 100644--- a/ggml/src/ggml-hexagon/htp/htp-ops.h+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h@@ -114,6 +114,7 @@ enum htp_op_code {HTP_OP_POOL_1D,HTP_OP_UNARY_GELU_QUICK,HTP_OP_IM2COL_3D,+ HTP_OP_HADAMARD,HTP_OP_INVALID};diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.hindex 842001c851..9b2e13133f 100644--- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h+++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h@@ -29,29 +29,27 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero);HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero);- HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero);- HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero);-HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero);HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero);- HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero);- HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero);- HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf)));- HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf)));-- HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf)));- HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf)));+ HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf)));+ HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf)));+ HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0+ HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16);+ HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf);+ vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf));- HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0- HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0- HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16);- HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16);+ HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero);+ HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero);+ HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero);+ HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero);- HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf);+ HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf)));+ HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf)));+ HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0+ HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16);HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf);- vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf));- vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf));+ vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf));HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf);HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf);@@ -81,25 +79,10 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_HVX_Vector voff01_hf = hvx_vec_f32_to_f16(voff0_sf, voff1_sf);HVX_Vector voff23_hf = hvx_vec_f32_to_f16(voff2_sf, voff3_sf);- HVX_Vector r_scale[4] = {- hvx_vec_repl_f16(vd01_hf),- hvx_vec_repl_f16(Q6_V_vror_VR(vd01_hf, 64)),- hvx_vec_repl_f16(vd23_hf),- hvx_vec_repl_f16(Q6_V_vror_VR(vd23_hf, 64)),- };- HVX_Vector r_offset[4] = {- hvx_vec_repl_f16(voff01_hf),- hvx_vec_repl_f16(Q6_V_vror_VR(voff01_hf, 64)),- hvx_vec_repl_f16(voff23_hf),- hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 64)),- };+ int32_t __attribute__((aligned(128))) ws[32];if (sums16) {// |sum of 16 int8| <= 2032 fits a halfword- int32_t __attribute__((aligned(128))) ws[32];*(HVX_Vector *) ws = v_sums;- for (int b = 0; b < 4; b++) {- r_offset[b] = Q6_V_vsplat_R(((uint32_t) ws[8 * b + 4] << 16) | ((uint32_t) ws[8 * b] & 0xFFFF));- }}static const uint8_t __attribute__((aligned(128))) repl[128] = {@@ -114,8 +97,15 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_};HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl;+ #pragma unrollfor (int b = 0; b < 4; b++) {HVX_Vector v_act = Q6_V_vror_VR(vx_i8, b * 32);+ HVX_Vector vd_b = (b < 2) ? vd01_hf : vd23_hf;+ HVX_Vector voff_b = (b < 2) ? voff01_hf : voff23_hf;+ HVX_Vector r_scale = hvx_vec_repl_f16((b & 1) ? Q6_V_vror_VR(vd_b, 64) : vd_b);+ HVX_Vector r_offset = sums16+ ? Q6_V_vsplat_R(((uint32_t) ws[8 * b + 4] << 16) | ((uint32_t) ws[8 * b] & 0xFFFF))+ : hvx_vec_repl_f16((b & 1) ? Q6_V_vror_VR(voff_b, 64) : voff_b);HVX_Vector r0 = Q6_V_vdelta_VV(v_act, v_repl_ctrl);HVX_Vector r1 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 4), v_repl_ctrl);@@ -135,8 +125,8 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_dst[5] = r5;dst[6] = r6;dst[7] = r7;- dst[8] = r_scale[b];- dst[9] = r_offset[b];+ dst[8] = r_scale;+ dst[9] = r_offset;}}@@ -932,7 +922,7 @@ static void tiled_vec_dot_q1_0_32x1(const uint32_t n, float * restrict s, constconst uint8_t * restrict tile_ptr = vx;const uint8_t * restrict y_q = vy;- HVX_Vector v_sum_float = Q6_V_vzero();+ HVX_Vector v_sum_qf = Q6_V_vzero();HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1);HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1);@@ -951,9 +941,10 @@ static void tiled_vec_dot_q1_0_32x1(const uint32_t n, float * restrict s, const}HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4]));- v_sum_float = hvx_vec_add_f32_f32(v_sum_float, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf));+ v_sum_qf = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf));}+ HVX_Vector v_sum_float = Q6_Vsf_equals_Vqf32(v_sum_qf);if (sz) {hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz)));} else {@@ -966,16 +957,16 @@ static void tiled_vec_dot_q1_0_32x2(const uint32_t n, float * restrict s0, floatconst uint8_t * restrict y0_q = vy0;const uint8_t * restrict y1_q = vy1;- HVX_Vector v_sum_float_c0 = Q6_V_vzero();- HVX_Vector v_sum_float_c1 = Q6_V_vzero();- HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1);- HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1);+ HVX_Vector v_sum_qf_c0 = Q6_V_vzero();+ HVX_Vector v_sum_qf_c1 = Q6_V_vzero();+ HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1);+ HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1);const uint32_t n_k_tiles = n / HTP_TENSOR_TILE_K_Q1_0;for (uint32_t kt = 0; kt < n_k_tiles; kt++) {const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * htp_tensor_get_tile_padded_size(HTP_TYPE_Q1_0));- HVX_Vector v_tile_c0 = Q6_V_vzero();- HVX_Vector v_tile_c1 = Q6_V_vzero();+ HVX_Vector v_tile_qf_c0 = Q6_V_vzero();+ HVX_Vector v_tile_qf_c1 = Q6_V_vzero();for (int quarter = 0; quarter < 4; quarter++) {const uint32_t act_tile = 4 * kt + quarter;@@ -987,15 +978,17 @@ static void tiled_vec_dot_q1_0_32x2(const uint32_t n, float * restrict s0, floatHVX_Vector v_da_sf_c0 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act0[8]));HVX_Vector v_da_sf_c1 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act1[8]));- v_tile_c0 = hvx_vec_add_f32_f32(v_tile_c0, hvx_vec_mul_f32_f32(v_sum_sf_c0, v_da_sf_c0));- v_tile_c1 = hvx_vec_add_f32_f32(v_tile_c1, hvx_vec_mul_f32_f32(v_sum_sf_c1, v_da_sf_c1));+ v_tile_qf_c0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c0, Q6_Vqf32_vmpy_VsfVsf(v_sum_sf_c0, v_da_sf_c0));+ v_tile_qf_c1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c1, Q6_Vqf32_vmpy_VsfVsf(v_sum_sf_c1, v_da_sf_c1));}HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4]));- v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_mul_f32_f32(v_tile_c0, v_dw_sf));- v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_mul_f32_f32(v_tile_c1, v_dw_sf));+ v_sum_qf_c0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf_c0, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf_c0), v_dw_sf));+ v_sum_qf_c1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf_c1, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf_c1), v_dw_sf));}+ HVX_Vector v_sum_float_c0 = Q6_Vsf_equals_Vqf32(v_sum_qf_c0);+ HVX_Vector v_sum_float_c1 = Q6_Vsf_equals_Vqf32(v_sum_qf_c1);if (sz0) {hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0)));} else {diff --git a/ggml/src/ggml-hexagon/htp/hvx-quant.h b/ggml/src/ggml-hexagon/htp/hvx-quant.hindex b7837967a8..b0ca6ecf6a 100644--- a/ggml/src/ggml-hexagon/htp/hvx-quant.h+++ b/ggml/src/ggml-hexagon/htp/hvx-quant.h@@ -191,6 +191,64 @@ static inline void hvx_dequantize_row_flat_bf16_f32(float * restrict dst_ptr, co}}+static inline void hvx_dequantize_row_flat_q1_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {+ static const uint32_t __attribute__((aligned(128))) bit_mask[32] = {+ 1u << 0, 1u << 1, 1u << 2, 1u << 3, 1u << 4, 1u << 5, 1u << 6, 1u << 7,+ 1u << 8, 1u << 9, 1u << 10, 1u << 11, 1u << 12, 1u << 13, 1u << 14, 1u << 15,+ 1u << 16, 1u << 17, 1u << 18, 1u << 19, 1u << 20, 1u << 21, 1u << 22, 1u << 23,+ 1u << 24, 1u << 25, 1u << 26, 1u << 27, 1u << 28, 1u << 29, 1u << 30, 1u << 31,+ };+ const int nb = n / QK1_0;+ const block_q1_0 * src = (const block_q1_0 *) src_ptr;+ const HVX_Vector v_bit_mask = *(const HVX_Vector *) bit_mask;+ const HVX_Vector v_zero = Q6_V_vzero();+ const HVX_Vector v_sign_bit = Q6_V_vsplat_R(0x80000000);++ for (int i = 0; i < nb; i++) {+ HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);+ HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(vd_f16));+ HVX_Vector neg_vd = Q6_V_vxor_VV(vd, v_sign_bit);+ HVX_Vector v_qs = *(const HVX_UVector *) src[i].qs;++ float * block_dst = dst_ptr + i * QK1_0;+ #pragma unroll+ for (int q = 0; q < 4; q++) {+ HVX_Vector v_bits = hvx_vec_repl_u32(q == 0 ? v_qs : Q6_V_vror_VR(v_qs, 4 * q));+ HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(v_bits, v_bit_mask), v_zero);+ hvx_vmem(block_dst + q * 32) = Q6_V_vmux_QVV(q_zero, neg_vd, vd);+ }+ }+}++static inline void hvx_dequantize_row_flat_q2_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {+ const int nb = n / QK2_0;+ const block_q2_0 * src = (const block_q2_0 *) src_ptr;+ const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);+ const HVX_Vector v_one = Q6_Vb_vsplat_R(1);++ for (int i = 0; i < nb; i++) {+ HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);+ HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(vd_f16));++ HVX_Vector vq = *(const HVX_UVector *) src[i].qs;+ HVX_Vector v0 = Q6_V_vand_VV(vq, mask_03);+ HVX_Vector v1 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vq, 2), mask_03);+ HVX_Vector v2 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vq, 4), mask_03);+ HVX_Vector v3 = Q6_Vub_vlsr_VubR(vq, 6);++ HVX_Vector v01 = Q6_V_lo_W(Q6_W_vshuff_VVR(v1, v0, -1));+ HVX_Vector v23 = Q6_V_lo_W(Q6_W_vshuff_VVR(v3, v2, -1));+ HVX_Vector v64 = Q6_Vb_vsub_VbVb(Q6_V_lo_W(Q6_W_vshuff_VVR(v23, v01, -2)), v_one);++ HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(v64);+ HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));++ float * block_dst = dst_ptr + i * QK2_0;+ hvx_vmem(block_dst + 0) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), vd);+ hvx_vmem(block_dst + 32) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(p32)), vd);+ }+}+static inline void hvx_dequantize_row_flat_q4_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {const int nb = n / QK4_0;const block_q4_0 * src = (const block_q4_0 *) src_ptr;@@ -603,4 +661,66 @@ static inline void hvx_dequantize_row_tiled_f16_f32(float * dst_spad, const uint}}+static inline void hvx_dequantize_row_tiled_q1_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {+ static const uint32_t __attribute__((aligned(128))) bit_mask[32] = {+ 1u << 0, 1u << 8, 1u << 16, 1u << 24,+ 1u << 1, 1u << 9, 1u << 17, 1u << 25,+ 1u << 2, 1u << 10, 1u << 18, 1u << 26,+ 1u << 3, 1u << 11, 1u << 19, 1u << 27,+ 1u << 4, 1u << 12, 1u << 20, 1u << 28,+ 1u << 5, 1u << 13, 1u << 21, 1u << 29,+ 1u << 6, 1u << 14, 1u << 22, 1u << 30,+ 1u << 7, 1u << 15, 1u << 23, 1u << 31,+ };+ const HVX_Vector v_bit_mask = *(const HVX_Vector *) bit_mask;+ const HVX_Vector v_zero = Q6_V_vzero();+ const HVX_Vector v_sign_bit = Q6_V_vsplat_R(0x80000000);++ for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {+ const uint8_t * tile = src_spad + k_tile * tile_stride;+ float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_K_Q1_0;++ const HVX_Vector scales = hvx_vmem(tile + 512);+ const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));+ const HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));+ const HVX_Vector neg_vd = Q6_V_vxor_VV(vd, v_sign_bit);++ #pragma unroll+ for (int q = 0; q < 4; ++q) {+ const HVX_Vector v_bits = hvx_vec_repl_u32(Q6_V_vror_VR(hvx_vmem(tile + q * VLEN), 4 * row));+ const HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(v_bits, v_bit_mask), v_zero);+ hvx_vmem(dst + q * 32) = Q6_V_vmux_QVV(q_zero, neg_vd, vd);+ }+ }+}++static inline void hvx_dequantize_row_tiled_q2_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {+ const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);+ const HVX_Vector v_one = Q6_Vb_vsplat_R(1);++ for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {+ const uint8_t * tile = src_spad + k_tile * tile_stride;+ float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_K_Q2_0;++ const HVX_Vector v_packed = hvx_dequantize_tiled_f16_gather4(tile, 4 * row);+ const HVX_Vector p0 = Q6_V_vand_VV(v_packed, mask_03);+ const HVX_Vector p1 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_packed, 2), mask_03);+ const HVX_Vector p2 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_packed, 4), mask_03);+ const HVX_Vector p3 = Q6_Vub_vlsr_VubR(v_packed, 6);++ const HVX_Vector p01 = Q6_V_lo_W(Q6_W_vshuff_VVR(p1, p0, -4));+ const HVX_Vector p23 = Q6_V_lo_W(Q6_W_vshuff_VVR(p3, p2, -4));+ const HVX_Vector vq = Q6_Vb_vsub_VbVb(Q6_V_lo_W(Q6_W_vshuff_VVR(p23, p01, -8)), v_one);++ const HVX_Vector scales = hvx_vmem(tile + 512);+ const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));+ const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));++ const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);+ const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));+ hvx_vmem(dst + 0) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);+ hvx_vmem(dst + 32) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(p32)), scale);+ }+}+#endif // HVX_QUANT_Hdiff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.cindex 2374add3f5..b72ef2b759 100644--- a/ggml/src/ggml-hexagon/htp/main.c+++ b/ggml/src/ggml-hexagon/htp/main.c@@ -940,6 +940,7 @@ static int execute_op(struct htp_ops_context * octx) {case HTP_OP_UNARY_RELU:case HTP_OP_UNARY_STEP:case HTP_OP_L2_NORM:+ case HTP_OP_HADAMARD:return op_unary(octx);case HTP_OP_GLU_SWIGLU:diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.cindex 37df31ddba..ea3d92ad04 100644--- a/ggml/src/ggml-hexagon/htp/unary-ops.c+++ b/ggml/src/ggml-hexagon/htp/unary-ops.c@@ -751,6 +751,143 @@ static void l2_norm_f32(const void * restrict src,}}+static inline HVX_VectorPair hvx_fwht_stage_pair(HVX_Vector v0, HVX_Vector v1, int32_t b) {+ HVX_VectorPair p = Q6_W_vshuff_VVR(v1, v0, b);+ HVX_Vector s = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(p), Q6_V_hi_W(p));+ HVX_Vector d = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_V_lo_W(p), Q6_V_hi_W(p));+ return Q6_W_vshuff_VVR(d, s, b);+}++static inline HVX_VectorPair hvx_fwht32_pair(HVX_Vector v0_sf, HVX_Vector v1_sf, HVX_Vector v_scale) {+ HVX_Vector v0 = Q6_Vqf32_vmpy_VsfVsf(v0_sf, v_scale);+ HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(v1_sf, v_scale);+ HVX_VectorPair p;+ p = hvx_fwht_stage_pair(v0, v1, 4);+ p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 8);+ p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 16);+ p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 32);+ p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 64);+ return p;+}++static inline void hvx_fwht_store32(HVX_Vector * restrict v_dst, HVX_VectorPair p) {+ v_dst[0] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p));+}++static inline void hvx_fwht_store64(HVX_Vector * restrict v_dst, HVX_VectorPair p) {+ HVX_Vector u = Q6_V_lo_W(p);+ HVX_Vector v = Q6_V_hi_W(p);+ v_dst[0] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(u, v));+ v_dst[1] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(u, v));+}++static inline void hvx_fwht_tail_f32(HVX_Vector * restrict v_dst, uint32_t nvec) {+ uint32_t len_vec = 2;+ for (; len_vec * 2 < nvec; len_vec <<= 2) {+ const uint32_t step = len_vec * 4;+ for (uint32_t i = 0; i < nvec; i += step) {+ for (uint32_t j = 0; j < len_vec; j++) {+ HVX_Vector a0 = v_dst[i + j];+ HVX_Vector a1 = v_dst[i + j + len_vec];+ HVX_Vector a2 = v_dst[i + j + 2 * len_vec];+ HVX_Vector a3 = v_dst[i + j + 3 * len_vec];++ HVX_Vector t0 = Q6_Vqf32_vadd_VsfVsf(a0, a1);+ HVX_Vector t1 = Q6_Vqf32_vsub_VsfVsf(a0, a1);+ HVX_Vector t2 = Q6_Vqf32_vadd_VsfVsf(a2, a3);+ HVX_Vector t3 = Q6_Vqf32_vsub_VsfVsf(a2, a3);++ v_dst[i + j] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(t0, t2));+ v_dst[i + j + len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(t1, t3));+ v_dst[i + j + 2 * len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(t0, t2));+ v_dst[i + j + 3 * len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(t1, t3));+ }+ }+ }++ if (len_vec < nvec) {+ for (uint32_t i = 0; i < nvec; i += 2 * len_vec) {+ for (uint32_t j = 0; j < len_vec; j++) {+ HVX_Vector u = v_dst[i + j];+ HVX_Vector v = v_dst[i + len_vec + j];+ v_dst[i + j] = hvx_vec_add_f32_f32(u, v);+ v_dst[i + len_vec + j] = hvx_vec_sub_f32_f32(u, v);+ }+ }+ }+}++static inline void hvx_fwht_row_f32(const uint8_t * restrict src, uint8_t * restrict dst, uint32_t n, HVX_Vector v_scale) {+ const HVX_Vector * restrict v_src = (const HVX_Vector *) src;+ HVX_Vector * restrict v_dst = (HVX_Vector *) dst;+ const uint32_t nvec = n / VLEN_FP32;++ if (nvec == 1) {+ HVX_VectorPair p = hvx_fwht32_pair(v_src[0], v_src[0], v_scale);+ hvx_fwht_store32(v_dst, p);+ return;+ }++ for (uint32_t i = 0; i < nvec; i += 2) {+ HVX_VectorPair p = hvx_fwht32_pair(v_src[i], v_src[i + 1], v_scale);+ hvx_fwht_store64(&v_dst[i], p);+ }++ hvx_fwht_tail_f32(v_dst, nvec);+}++static inline void hvx_fwht_row_f16(const uint8_t * restrict src, uint8_t * restrict dst, uint32_t n, HVX_Vector v_scale) {+ const HVX_Vector * restrict v_src = (const HVX_Vector *) src;+ HVX_Vector * restrict v_dst = (HVX_Vector *) dst;+ const uint32_t nvec = n / VLEN_FP32;++ if (nvec == 1) {+ HVX_Vector v0 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_src[0]));+ HVX_VectorPair p = hvx_fwht32_pair(v0, v0, v_scale);+ hvx_fwht_store32(v_dst, p);+ return;+ }++ const uint32_t nvec_f16 = n / VLEN_FP16;+ for (uint32_t i = 0; i < nvec_f16; i++) {+ HVX_VectorPair f32p = hvx_vec_f16_to_f32(v_src[i]);+ HVX_VectorPair p = hvx_fwht32_pair(Q6_V_lo_W(f32p), Q6_V_hi_W(f32p), v_scale);+ hvx_fwht_store64(&v_dst[2 * i], p);+ }++ hvx_fwht_tail_f32(v_dst, nvec);+}++static void hadamard_f32(const void * restrict src,+ void * restrict dst,+ const uint32_t num_rows,+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;+ const HVX_Vector v_scale = hvx_vec_splat_f32(1.0f / sqrtf((float) ne0));++ for (uint32_t ir = 0; ir < num_rows; ir++) {+ const uint8_t * restrict src_row = (const uint8_t *) src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_row = (uint8_t *) dst + (ir * dst_row_size_aligned);++ hvx_fwht_row_f32(src_row, dst_row, ne0, v_scale);+ }+}++static void hadamard_f16(const void * restrict src,+ void * restrict dst,+ const uint32_t num_rows,+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;+ const HVX_Vector v_scale = hvx_vec_splat_f32(1.0f / sqrtf((float) ne0));++ for (uint32_t ir = 0; ir < num_rows; ir++) {+ const uint8_t * restrict src_row = (const uint8_t *) src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_row = (uint8_t *) dst + (ir * dst_row_size_aligned);++ hvx_fwht_row_f16(src_row, dst_row, ne0, v_scale);+ }+}+static void tanh_f32(const void * restrict src,void * restrict dst,const uint32_t num_rows,@@ -1644,6 +1781,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {case HTP_OP_UNARY_STEP: op_type = is_f16 ? "step-f16" : "step-f32"; break;case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break;case HTP_OP_TRI: op_type = "tri-f32"; break;+ case HTP_OP_HADAMARD: op_type = is_f16 ? "hadamard-f16" : "hadamard-f32"; break;default:FARF(ERROR, "Unsupported unary Op %u\n", octx->op);return HTP_STATUS_NO_SUPPORT;@@ -1663,6 +1801,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {case HTP_OP_UNARY_ABS:case HTP_OP_UNARY_LOG:case HTP_OP_UNARY_STEP:+ case HTP_OP_HADAMARD:break;default:FARF(ERROR, "unary-%s: not supported for F16\n", op_type);@@ -1677,16 +1816,17 @@ static int execute_op_unary(struct htp_ops_context * octx) {}const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];- const size_t elem_size = is_f16 ? sizeof(_Float16) : sizeof(float);- const size_t src0_data_row_size = src0->ne[0] * elem_size;- const size_t dst_data_row_size = dst->ne[0] * elem_size;+ const size_t src0_elem_size = is_f16 ? sizeof(_Float16) : sizeof(float);+ const size_t dst_elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float);+ const size_t src0_data_row_size = src0->ne[0] * src0_elem_size;+ const size_t dst_data_row_size = dst->ne[0] * dst_elem_size;uint32_t row_start = 0;uint32_t nrows = src0_nrows;if (octx->ctx->mdev.count > 1) {uint32_t rows_per_chunk = 0;- htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk);+ htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) dst_elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk);const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);row_start = range.start;nrows = range.count;@@ -1813,6 +1953,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {case HTP_OP_UNARY_ABS: compute_func = (void *) abs_f16; break;case HTP_OP_UNARY_LOG: compute_func = (void *) log_f16; break;case HTP_OP_UNARY_STEP: compute_func = (void *) step_f16; break;+ case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f16; break;default: break;}} else {@@ -1843,6 +1984,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {case HTP_OP_UNARY_RELU: compute_func = (void *) relu_f32; break;case HTP_OP_UNARY_STEP: compute_func = (void *) step_f32; break;case HTP_OP_L2_NORM: compute_func = (void *) l2_norm_f32; break;+ case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f32; break;case HTP_OP_TRI:task_func = unary_thread_tri_f32;compute_func = (void *) tri_f32;diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.hindex 7e5779e5af..902028635e 100644--- a/ggml/src/ggml-hexagon/htp/unary-ops.h+++ b/ggml/src/ggml-hexagon/htp/unary-ops.h@@ -64,6 +64,7 @@ static inline bool htp_op_is_unary(uint32_t opcode) {case HTP_OP_UNARY_STEP:case HTP_OP_L2_NORM:case HTP_OP_TRI:+ case HTP_OP_HADAMARD:return true;default:return false;@@ -94,8 +95,9 @@ static inline void htp_unary_vtcm_layout_build(uint32_t * out_col_tile,uint32_t * out_vtcm_row_per_thread) {+ const size_t dst_elem_size = (op == HTP_OP_HADAMARD) ? sizeof(float) : elem_size;const size_t src0_data_row_size = ne00 * elem_size;- const size_t dst_data_row_size = ne10 * elem_size;+ const size_t dst_data_row_size = ne10 * dst_elem_size;const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128);@@ -131,7 +133,8 @@ static inline void htp_unary_vtcm_layout_build(}const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM ||- op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM);+ op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM ||+ op == HTP_OP_HADAMARD);// The tiled fallback path below only has F32 task functions (unary_task_f32_tiled_*);// F16 has no tiled kernels, so it must stay on the row-block path like reduction ops.// NOTE: if F16 ends up with vtcm_row_per_thread == 0 here (row too large for the VTCM