LLM inference in C/C++
C++ ggml
ggml-hexagon: add HTP unary ops for ABS and LOG (#27786)
cqdereke70802a01f03
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex c1e9f919d9..53e8607559 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -4643,6 +4643,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {case GGML_OP_CLAMP: return HTP_OP_CLAMP;case GGML_OP_SQR: return HTP_OP_SQR;case GGML_OP_SQRT: return HTP_OP_SQRT;+ case GGML_OP_LOG: return HTP_OP_UNARY_LOG;case GGML_OP_SOFT_MAX: return HTP_OP_SOFTMAX;case GGML_OP_SSM_CONV: return HTP_OP_SSM_CONV;case GGML_OP_GATED_DELTA_NET: return HTP_OP_GATED_DELTA_NET;@@ -4666,6 +4667,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {case GGML_UNARY_OP_EXP: return HTP_OP_UNARY_EXP;case GGML_UNARY_OP_SOFTPLUS: return HTP_OP_UNARY_SOFTPLUS;case GGML_UNARY_OP_TANH: return HTP_OP_UNARY_TANH;+ case GGML_UNARY_OP_ABS: return HTP_OP_UNARY_ABS;default:break;}@@ -5463,6 +5465,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, conscase GGML_OP_SQR:case GGML_OP_SQRT:+ case GGML_OP_LOG:supp = ggml_hexagon_supported_unary(sess, op);break;@@ -5481,6 +5484,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, conscase GGML_UNARY_OP_SIGMOID:case GGML_UNARY_OP_SOFTPLUS:case GGML_UNARY_OP_TANH:+ case GGML_UNARY_OP_ABS:case GGML_UNARY_OP_SILU:case GGML_UNARY_OP_GELU:case GGML_UNARY_OP_GELU_QUICK:diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.hindex b4023b34d3..e804844d59 100644--- a/ggml/src/ggml-hexagon/htp/htp-ops.h+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h@@ -62,6 +62,8 @@ enum htp_op_code {HTP_OP_UNARY_NEG,HTP_OP_UNARY_SOFTPLUS,HTP_OP_UNARY_TANH,+ HTP_OP_UNARY_ABS,+ HTP_OP_UNARY_LOG,HTP_OP_GLU_SWIGLU,HTP_OP_GLU_SWIGLU_OAI,HTP_OP_GLU_GEGLU,diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.hindex c8d0003ab5..765c357766 100644--- a/ggml/src/ggml-hexagon/htp/hvx-arith.h+++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h@@ -358,6 +358,34 @@ static inline void hvx_clamp_scalar_f32(uint8_t * restrict dst, const uint8_t *}}+//+// Abs+//++static inline void hvx_abs_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {+ assert((unsigned long) dst % 128 == 0);+ assert((unsigned long) src % 128 == 0);++ HVX_Vector * restrict vdst = (HVX_Vector *) dst;+ HVX_Vector * restrict vsrc = (HVX_Vector *) src;++ const uint32_t elem_size = sizeof(float);+ const uint32_t epv = 128 / elem_size;+ const uint32_t nvec = n / epv;+ const uint32_t nloe = n % epv;++ uint32_t i = 0;++ _Pragma("unroll(4)")+ for (; i < nvec; i++) {+ vdst[i] = hvx_vec_abs_f32(vsrc[i]);+ }+ if (nloe) {+ HVX_Vector v = hvx_vec_abs_f32(vsrc[i]);+ hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);+ }+}+//// Square//diff --git a/ggml/src/ggml-hexagon/htp/hvx-log.h b/ggml/src/ggml-hexagon/htp/hvx-log.hindex 7013dae785..a209f88d55 100644--- a/ggml/src/ggml-hexagon/htp/hvx-log.h+++ b/ggml/src/ggml-hexagon/htp/hvx-log.h@@ -62,4 +62,28 @@ static inline HVX_Vector hvx_vec_log_f32(HVX_Vector x) {return hvx_vec_add_f32_f32(term_e, res);}+static inline void hvx_log_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {+ assert((unsigned long) dst % 128 == 0);+ assert((unsigned long) src % 128 == 0);++ HVX_Vector * restrict vdst = (HVX_Vector *) dst;+ HVX_Vector * restrict vsrc = (HVX_Vector *) src;++ const uint32_t elem_size = sizeof(float);+ const uint32_t epv = 128 / elem_size;+ const uint32_t nvec = n / epv;+ const uint32_t nloe = n % epv;++ uint32_t i = 0;++ _Pragma("unroll(4)")+ for (; i < nvec; i++) {+ vdst[i] = hvx_vec_log_f32(vsrc[i]);+ }+ if (nloe) {+ HVX_Vector v = hvx_vec_log_f32(vsrc[i]);+ hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);+ }+}+#endif /* HVX_LOG_H */diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.cindex 975ba0c7af..fe7d093a81 100644--- a/ggml/src/ggml-hexagon/htp/main.c+++ b/ggml/src/ggml-hexagon/htp/main.c@@ -777,6 +777,8 @@ static int execute_op(struct htp_ops_context * octx) {case HTP_OP_UNARY_NEG:case HTP_OP_UNARY_EXP:case HTP_OP_UNARY_TANH:+ case HTP_OP_UNARY_ABS:+ case HTP_OP_UNARY_LOG:case HTP_OP_L2_NORM:return op_unary(octx);diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.cindex 971f3b5377..1a632bf563 100644--- a/ggml/src/ggml-hexagon/htp/unary-ops.c+++ b/ggml/src/ggml-hexagon/htp/unary-ops.c@@ -443,6 +443,34 @@ static void tanh_f32(const float * restrict src,}}+static void abs_f32(const float * restrict src,+ float * restrict dst,+ const uint32_t num_rows,+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;++ for (uint32_t ir = 0; ir < num_rows; ir++) {+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);++ hvx_abs_f32_aa(dst_local, src_local, ne0);+ }+}++static void log_f32(const float * restrict src,+ float * restrict dst,+ const uint32_t num_rows,+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;++ for (uint32_t ir = 0; ir < num_rows; ir++) {+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);++ hvx_log_f32_aa(dst_local, src_local, ne0);+ }+}+#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * data) { \const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \@@ -611,6 +639,8 @@ DEFINE_UNARY_TASK(unary_silu, false, false, silu_f32(src0_vtcm, dst_vtcm, blDEFINE_UNARY_TASK(unary_gelu, false, false, gelu_f32(src0_vtcm, dst_vtcm, block_size, uctx))DEFINE_UNARY_TASK(unary_softplus, false, false, softplus_f32(src0_vtcm, dst_vtcm, block_size, uctx))DEFINE_UNARY_TASK(unary_tanh, false, false, tanh_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(unary_abs, false, false, abs_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(unary_log, false, false, log_f32(src0_vtcm, dst_vtcm, block_size, uctx))DEFINE_UNARY_TASK(l2_norm, false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))DEFINE_UNARY_TASK(tri, false, true, tri_f32(src0_vtcm, dst_vtcm, block_size, ir, uctx))@@ -858,6 +888,8 @@ DEFINE_UNARY_TILED_TASK(unary_silu, false, tile_silu_f32(dst_vtcm, src_vtcm,DEFINE_UNARY_TILED_TASK(unary_gelu, false, tile_gelu_f32(dst_vtcm, src_vtcm, tw))DEFINE_UNARY_TILED_TASK(unary_softplus, false, tile_unary_softplus_f32(dst_vtcm, src_vtcm, tw))DEFINE_UNARY_TILED_TASK(unary_tanh, false, hvx_tanh_f32_aa(dst_vtcm, src_vtcm, tw))+DEFINE_UNARY_TILED_TASK(unary_abs, false, hvx_abs_f32_aa(dst_vtcm, src_vtcm, tw))+DEFINE_UNARY_TILED_TASK(unary_log, false, hvx_log_f32_aa(dst_vtcm, src_vtcm, tw))DEFINE_UNARY_TILED_TASK(tri, true, tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype))static int execute_op_unary_f32(struct htp_ops_context * octx) {@@ -883,6 +915,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break;case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break;case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break;+ case HTP_OP_UNARY_ABS: op_type = "abs-f32"; break;+ case HTP_OP_UNARY_LOG: op_type = "log-f32"; break;case HTP_OP_L2_NORM: op_type = "l2norm-f32"; break;case HTP_OP_TRI: op_type = "tri-f32"; break;@@ -981,6 +1015,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {case HTP_OP_UNARY_GELU: task_func = unary_task_f32_tiled_unary_gelu; break;case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_tiled_unary_softplus; break;case HTP_OP_UNARY_TANH: task_func = unary_task_f32_tiled_unary_tanh; break;+ case HTP_OP_UNARY_ABS: task_func = unary_task_f32_tiled_unary_abs; break;+ case HTP_OP_UNARY_LOG: task_func = unary_task_f32_tiled_unary_log; break;case HTP_OP_TRI: task_func = unary_task_f32_tiled_tri; break;default: break;}@@ -1000,6 +1036,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {case HTP_OP_UNARY_GELU: task_func = unary_task_f32_unary_gelu; break;case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_unary_softplus; break;case HTP_OP_UNARY_TANH: task_func = unary_task_f32_unary_tanh; break;+ case HTP_OP_UNARY_ABS: task_func = unary_task_f32_unary_abs; break;+ case HTP_OP_UNARY_LOG: task_func = unary_task_f32_unary_log; break;case HTP_OP_L2_NORM: task_func = unary_task_f32_l2_norm; break;case HTP_OP_TRI: task_func = unary_task_f32_tri; break;default: break;diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.hindex 1f4c3a5c4d..458218ff44 100644--- a/ggml/src/ggml-hexagon/htp/unary-ops.h+++ b/ggml/src/ggml-hexagon/htp/unary-ops.h@@ -55,6 +55,8 @@ static inline bool htp_op_is_unary(uint32_t opcode) {case HTP_OP_UNARY_GELU:case HTP_OP_UNARY_SOFTPLUS:case HTP_OP_UNARY_TANH:+ case HTP_OP_UNARY_ABS:+ case HTP_OP_UNARY_LOG:case HTP_OP_L2_NORM:case HTP_OP_TRI:return true;