LLM inference in C/C++
C++ ggml
hexagon: activation ops update (#25974)
Todor Boinovski0278d8362d78
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex 54d313ade5..d19d2d5f03 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -3084,7 +3084,10 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_sessionreturn false;}- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {+ if (!ggml_is_contiguous_1(src0)) {+ return false;+ }+ if (!ggml_is_contiguous(dst)) {return false;}@@ -3095,7 +3098,7 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_sessionif (!ggml_are_same_shape(src0, src1)) {return false;}- if (!ggml_is_contiguous(src1)) {+ if (!ggml_is_contiguous_1(src1)) {return false;}}@@ -4152,12 +4155,10 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, conscase GGML_UNARY_OP_SIGMOID:case GGML_UNARY_OP_SOFTPLUS:case GGML_UNARY_OP_TANH:- supp = ggml_hexagon_supported_unary(sess, op);- break;case GGML_UNARY_OP_SILU:case GGML_UNARY_OP_GELU:case GGML_UNARY_OP_GELU_QUICK:- supp = ggml_hexagon_supported_activations(sess, op);+ supp = ggml_hexagon_supported_unary(sess, op);break;default:break;diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.cindex e56f2b6a52..9973c088dd 100644--- a/ggml/src/ggml-hexagon/htp/act-ops.c+++ b/ggml/src/ggml-hexagon/htp/act-ops.c@@ -17,6 +17,7 @@#include "htp-ops.h"#include "htp-ops.h"#include "htp-tensor.h"+#include "htp-vtcm.h"#define htp_act_preamble \const struct htp_tensor * src0 = actx->octx->src[0]; \@@ -54,581 +55,413 @@const uint32_t nb3 = dst->nb[3];struct htp_act_context {- struct htp_ops_context * octx;+ struct htp_ops_context * octx;// Precomputed values- const uint8_t * data_src0;- const uint8_t * data_src1;- uint8_t * data_dst;-- size_t src0_row_size;- size_t src1_row_size;- size_t dst_row_size;-- size_t src0_row_size_aligned;- size_t src1_row_size_aligned;- size_t dst_row_size_aligned;-- size_t src0_spad_half_size;- size_t src1_spad_half_size;- size_t dst_spad_half_size;-- uint32_t block;- uint32_t src0_nrows;- uint32_t src0_nrows_per_thread;- int nc;-};--static void glu_swiglu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {- struct htp_act_context * actx = (struct htp_act_context *) data;- htp_act_preamble;-- size_t src0_row_size = actx->src0_row_size;- size_t src1_row_size = actx->src1_row_size;- size_t dst_row_size = actx->dst_row_size;+ const uint8_t * data_src0;+ const uint8_t * data_src1;+ uint8_t * data_dst;- const uint32_t src0_nrows = actx->src0_nrows;- const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread;- const uint32_t src0_start_row = src0_nrows_per_thread * ith;- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);-- // no work for this thread- if (src0_start_row >= src0_end_row) {- return;- }+ size_t src0_row_size;+ size_t src1_row_size;+ size_t dst_row_size;- uint64_t t1, t2;- t1 = HAP_perf_get_qtimer_count();+ size_t src0_row_stride;+ size_t src1_row_stride;- const uint8_t * restrict data_src0 = actx->data_src0;- const uint8_t * restrict data_src1 = actx->data_src1;- uint8_t * restrict data_dst = actx->data_dst;+ size_t src0_row_size_aligned;+ size_t src1_row_size_aligned;+ size_t dst_row_size_aligned;- const int nc = actx->nc;+ size_t src0_spad_half_size;+ size_t src1_spad_half_size;+ size_t dst_spad_half_size;- const size_t src0_row_size_aligned = actx->src0_row_size_aligned;- const size_t src1_row_size_aligned = actx->src1_row_size_aligned;- const size_t dst_row_size_aligned = actx->dst_row_size_aligned;+ uint32_t block;+ uint32_t src0_nrows;+ uint32_t src0_nrows_per_thread;+ int nc;- uint8_t * restrict src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread);- uint8_t * restrict src1_spad_data = actx->octx->src1_spad.data + (ith * actx->octx->src1_spad.size_per_thread);- uint8_t * restrict dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread);+ uint8_t * vtcm_src0;+ uint8_t * vtcm_src1;+ uint8_t * vtcm_dst;- size_t src0_spad_half_size = actx->src0_spad_half_size;- size_t src1_spad_half_size = actx->src1_spad_half_size;- size_t dst_spad_half_size = actx->dst_spad_half_size;+ size_t vtcm_src0_size_per_thread;+ size_t vtcm_src1_size_per_thread;+ size_t vtcm_dst_size_per_thread;+};- const int BLOCK = actx->block;- if (BLOCK == 0) {- FARF(ERROR,- "swiglu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n",- actx->octx->src0_spad.size_per_thread, src0_row_size_aligned);- return;- }+struct htp_act_vtcm_layout {+ size_t total_bytes;+ size_t off_src0;+ size_t off_src1;+ size_t off_dst;- dma_queue * dma_queue = actx->octx->ctx->dma[ith];+ size_t src0_bytes_per_thread;+ size_t src1_bytes_per_thread;+ size_t dst_bytes_per_thread;- // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379- for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {- const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);+ uint32_t vtcm_row_per_thread;+};- // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)- dma_queue_push_vtcm_to_ddr(dma_queue,- dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),- dst_row_size, dst_row_size_aligned, 0);+static inline void htp_act_vtcm_layout_build(struct htp_act_vtcm_layout * L,+ size_t src0_row_size_aligned,+ size_t src1_row_size_aligned,+ size_t dst_row_size_aligned,+ uint32_t n_threads,+ size_t vtcm_size) {+ const size_t spad_size_per_row = src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned;+ const uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * spad_size_per_row));- dma_queue_push_ddr_to_vtcm(dma_queue,- dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),- src0_row_size_aligned, src0_row_size, block_size);- dma_queue_push_ddr_to_vtcm(dma_queue,- dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)),- src1_row_size_aligned, src1_row_size, block_size);- }+ L->vtcm_row_per_thread = vtcm_row_per_thread;- for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {- const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);-- float * dst_spad = (float *) dma_queue_pop(dma_queue).src;- float * src0_spad = (float *) dma_queue_pop(dma_queue).dst;- float * src1_spad = (float *) dma_queue_pop(dma_queue).dst;-- for (uint32_t ib = 0; ib < block_size; ib++) {- const float * src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float));- const float * src1_spad_ptr = src1_spad + ib * (src1_row_size_aligned / sizeof(float));- float * dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float));-- //swiglu(x) = x1 * sigmoid(x0)- hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, nc);- hvx_mul_mul_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr,- (const uint8_t *) src1_spad_ptr, nc);- }-- dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size,- dst_row_size_aligned, block_size);-- // prefetch N+2 loop iteration if any- const uint32_t pref_block = (ir + BLOCK * 2);- if (pref_block < src0_end_row) {- const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);- dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),- src0_row_size_aligned, src0_row_size, pref_block_size);- dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)),- src1_row_size_aligned, src1_row_size, pref_block_size);- }- }+ L->src0_bytes_per_thread = src0_row_size_aligned * vtcm_row_per_thread;+ L->src1_bytes_per_thread = src1_row_size_aligned * vtcm_row_per_thread;+ L->dst_bytes_per_thread = dst_row_size_aligned * vtcm_row_per_thread;- dma_queue_flush(dma_queue);+ L->off_src0 = 0;+ L->off_src1 = L->off_src0 + L->src0_bytes_per_thread * n_threads;+ L->off_dst = L->off_src1 + L->src1_bytes_per_thread * n_threads;- t2 = HAP_perf_get_qtimer_count();-- FARF(HIGH, "swiglu-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth,- ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3,- (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));+ L->total_bytes = L->off_dst + L->dst_bytes_per_thread * n_threads;}-static void glu_swiglu_oai_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {- struct htp_act_context * actx = (struct htp_act_context *) data;- htp_act_preamble;-- uint64_t t1, t2;- t1 = HAP_perf_get_qtimer_count();-- size_t src0_row_size = actx->src0_row_size;- size_t src1_row_size = actx->src1_row_size;- size_t dst_row_size = actx->dst_row_size;-- const uint32_t src0_nrows = actx->src0_nrows;- const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread;-- const uint32_t src0_start_row = src0_nrows_per_thread * ith;- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);-- // no work for this thread- if (src0_start_row >= src0_end_row) {- return;+#define htp_glu_op_preamble \+ const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \+ const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \+ const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \+ const int nc = actx->nc;++// swiglu(x) = x1 * sigmoid(x0)+static void swiglu_f32(const float * restrict src0,+ const float * restrict src1,+ float * restrict dst,+ const uint32_t num_rows,+ const struct htp_act_context * actx) {+ htp_glu_op_preamble;++ for (uint32_t ib = 0; ib < num_rows; ib++) {+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);++ hvx_sigmoid_f32_aa(dst_ptr, src0_ptr, nc);+ hvx_mul_mul_f32_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc);}+}- const uint8_t * restrict data_src0 = actx->data_src0;- const uint8_t * restrict data_src1 = actx->data_src1;- uint8_t * restrict data_dst = actx->data_dst;-- const int nc = actx->nc;-- const size_t src0_row_size_aligned = actx->src0_row_size_aligned;- const size_t src1_row_size_aligned = actx->src1_row_size_aligned;- const size_t dst_row_size_aligned = actx->dst_row_size_aligned;-- uint8_t * restrict src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread);- uint8_t * restrict src1_spad_data = actx->octx->src1_spad.data + (ith * actx->octx->src1_spad.size_per_thread);- uint8_t * restrict dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread);-- size_t src0_spad_half_size = actx->src0_spad_half_size;- size_t src1_spad_half_size = actx->src1_spad_half_size;- size_t dst_spad_half_size = actx->dst_spad_half_size;-- const int BLOCK = actx->block;- if (BLOCK == 0) {- FARF(ERROR,- "swiglu-oai-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least "- "%zu\n",- actx->octx->src0_spad.size_per_thread, src0_row_size_aligned);- return;- }+// out = x * sigmoid(alpha * x) * (clamp(y, -limit, limit) + 1.f)+static void swiglu_oai_f32(const float * restrict src0,+ const float * restrict src1,+ float * restrict dst,+ const uint32_t num_rows,+ const struct htp_act_context * actx) {+ htp_glu_op_preamble;const float alpha = ((const float *) (actx->octx->op_params))[2];const float limit = ((const float *) (actx->octx->op_params))[3];- dma_queue * dma_queue = actx->octx->ctx->dma[ith];-- // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379- for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {- const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);-- // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)- dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),- dst_row_size, dst_row_size_aligned, 0);-- dma_queue_push_ddr_to_vtcm(- dma_queue,- dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),- src0_row_size_aligned, src0_row_size, block_size);- dma_queue_push_ddr_to_vtcm(- dma_queue,- dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)),- src1_row_size_aligned, src1_row_size, block_size);+ for (uint32_t ib = 0; ib < num_rows; ib++) {+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);++ // x (src0_ptr) = std::min(src0_p[k], limit);+ hvx_min_scalar_f32((uint8_t *) src0_ptr, src0_ptr, limit, nc);+ // y1 (src1_ptr) = std::clamp(src1_p[k], -limit, limit);+ hvx_clamp_scalar_f32((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc);+ // y (src1_ptr) = y1 + 1.f+ hvx_add_scalar_f32((uint8_t *) src1_ptr, src1_ptr, 1.0, nc);+ // x1 (dst_ptr) = alpha * x+ hvx_mul_scalar_f32(dst_ptr, src0_ptr, alpha, nc);+ // x2 (dst_ptr) = sigmoid(x1) = 1/(1+exp(-x1))+ hvx_sigmoid_f32_aa(dst_ptr, dst_ptr, nc);+ // out = x * sigmoid(alpha * x) * (y + 1.f)+ hvx_mul_mul_f32_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc);}-- for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {- const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);-- float * dst_spad = (float *) dma_queue_pop(dma_queue).src;- float * src0_spad = (float *) dma_queue_pop(dma_queue).dst;- float * src1_spad = (float *) dma_queue_pop(dma_queue).dst;-- for (uint32_t ib = 0; ib < block_size; ib++) {- const float * src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float));- const float * src1_spad_ptr = src1_spad + ib * (src1_row_size_aligned / sizeof(float));- float * dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float));-- // x (src0_spad_data) = std::min(src0_p[k], limit);- hvx_min_scalar_f32((uint8_t *) src0_spad_ptr, (const uint8_t *) src0_spad_ptr, limit, nc);- // y1 (src1_spad_data) = std::clamp(src1_p[k], -limit, limit);- hvx_clamp_scalar_f32((uint8_t *) src1_spad_ptr, (const uint8_t *) src1_spad_ptr, -limit, limit, nc);- // y (src1_spad_data) = y1 + 1.f- hvx_add_scalar_f32((uint8_t *) src1_spad_ptr, (const uint8_t *) src1_spad_ptr, 1.0, nc);- // x1 (dst_spad_data) = alpha * (x)- hvx_mul_scalar_f32((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, alpha, nc);- // x2 (dst_spad_data) = sigmoid(x1) = 1/(1+exp(-x1))- hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, nc);- // out = x * sigmoid(alpha * x) * (y + 1.f)- hvx_mul_mul_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr,- (const uint8_t *) src1_spad_ptr, nc);- }-- dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size,- dst_row_size_aligned, block_size);-- // prefetch N+2 loop iteration if any- const uint32_t pref_block = (ir + BLOCK * 2);- if (pref_block < src0_end_row) {- const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);- dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),- src0_row_size_aligned, src0_row_size, pref_block_size);- dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)),- src1_row_size_aligned, src1_row_size, pref_block_size);- }- }-- dma_queue_flush(dma_queue);-- t2 = HAP_perf_get_qtimer_count();-- FARF(HIGH, "swiglu-oai-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, src0->ne[0],- src0->ne[1], src0->ne[2], src0->ne[3], src0_start_row, src0_end_row, src1->ne[0], src1->ne[1], src1->ne[2],- src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));}+static const float GELU_COEF_A = 0.044715f;+static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;-static void unary_gelu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {- struct htp_act_context * actx = (struct htp_act_context *) data;- htp_act_preamble;-- uint64_t t1, t2;- t1 = HAP_perf_get_qtimer_count();-- const size_t src0_row_size = actx->src0_row_size;- const size_t dst_row_size = actx->dst_row_size;- const size_t src0_row_size_aligned = actx->src0_row_size_aligned;- const size_t dst_row_size_aligned = actx->dst_row_size_aligned;-- const uint32_t src0_nrows = actx->src0_nrows;- const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread;-- const uint32_t src0_start_row = src0_nrows_per_thread * ith;- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);-- // no work for this thread- if (src0_start_row >= src0_end_row) {- return;- }-- const uint8_t * data_src0 = actx->data_src0;- uint8_t * data_dst = actx->data_dst;-- // nc/ne0 matches.- const int ne0_val = actx->nc; // == dst->ne[0]-- uint8_t * src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread);- uint8_t * dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread);-- size_t src0_spad_half_size = actx->src0_spad_half_size;- size_t dst_spad_half_size = actx->dst_spad_half_size;-- // In gelu = x*sigmoid(x*1.702)- const int BLOCK = actx->block;-- if (BLOCK == 0) {- FARF(ERROR, "gelu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n",- actx->octx->src0_spad.size_per_thread, src0_row_size_aligned);- return;- }-- dma_queue * dma_queue = actx->octx->ctx->dma[ith];-- // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379- for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {- const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);-- // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)- dma_queue_push_vtcm_to_ddr(dma_queue,- dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),- dst_row_size, dst_row_size_aligned, 0);-- dma_queue_push_ddr_to_vtcm(dma_queue,- dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),- src0_row_size_aligned, src0_row_size, block_size);- }+static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {+ assert((unsigned long) dst % 128 == 0);+ assert((unsigned long) src0 % 128 == 0);+ assert((unsigned long) src1 % 128 == 0);- for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {- const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);-- float* dst_spad = (float *) dma_queue_pop(dma_queue).src;- float* src0_spad = (float *) dma_queue_pop(dma_queue).dst;-- for (uint32_t ib = 0; ib < block_size; ib++) {- const float* src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float));- float* dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float));-- // gelu = x * sigmoid(1.702 * x) // current implementation- hvx_mul_scalar_f32((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (float) 1.702, ne0_val);- hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0_val);- hvx_mul_f32_aaa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0_val);- }-- dma_queue_push_vtcm_to_ddr(dma_queue,- dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad),- dst_row_size, dst_row_size_aligned, block_size);-- // prefetch N+2 loop iteration if any- const uint32_t pref_block = (ir + BLOCK * 2);- if (pref_block < src0_end_row) {- const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);- dma_queue_push_ddr_to_vtcm(dma_queue,- dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),- src0_row_size_aligned, src0_row_size, pref_block_size);- }- }+ HVX_Vector * restrict vdst = (HVX_Vector *) dst;+ const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0;+ const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1;- dma_queue_flush(dma_queue);+ const uint32_t epv = 128 / sizeof(float);+ const uint32_t nvec = n / epv;+ const uint32_t nloe = n % epv;- t2 = HAP_perf_get_qtimer_count();+ const float GELU_COEF_A_TIMES_SQRT = GELU_COEF_A * SQRT_2_OVER_PI;- FARF(HIGH, "gelu-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", ith, nth, ne00, ne01, ne02,- ne03, src0_start_row, src0_end_row, ne0, ne1, ne2, ne3, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));-}+ const HVX_Vector v_coef_a_times_sqrt = hvx_vec_splat_f32(GELU_COEF_A_TIMES_SQRT);+ const HVX_Vector v_sqrt_2_pi = hvx_vec_splat_f32(SQRT_2_OVER_PI);+ const HVX_Vector v_half = hvx_vec_splat_f32(0.5f);+ const HVX_Vector v_one = hvx_vec_splat_f32(1.0f);+ const HVX_Vector v_two = hvx_vec_splat_f32(2.0f);+ // Hoisted fast sigmoid / inverse constants to avoid loop-internal overhead+ const HVX_Vector v_log2f = Q6_V_vsplat_R(FAST_SIGMOID_LOG2F);+ const HVX_Vector v_c1 = Q6_V_vsplat_R(FAST_SIGMOID_C1);+ const HVX_Vector v_c2 = Q6_V_vsplat_R(FAST_SIGMOID_C2);+ const HVX_Vector v_inv_aprox = Q6_V_vsplat_R(0x7EEEEBB3);+ const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f);+ const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f);-static void unary_silu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {- struct htp_act_context * actx = (struct htp_act_context *) data;- htp_act_preamble;+ uint32_t i = 0;- uint64_t t1, t2;- t1 = HAP_perf_get_qtimer_count();+ for (; i < nvec; i++) {+ HVX_Vector x = vsrc0[i];+ HVX_Vector g = vsrc1[i];- const size_t src0_row_size = actx->src0_row_size;- const size_t dst_row_size = actx->dst_row_size;- const size_t src0_row_size_aligned = actx->src0_row_size_aligned;- const size_t dst_row_size_aligned = actx->dst_row_size_aligned;+ HVX_Vector x2 = hvx_vec_mul_f32_f32(x, x);+ HVX_Vector coef = hvx_vec_mul_f32_f32(x2, v_coef_a_times_sqrt);+ coef = hvx_vec_add_f32_f32(coef, v_sqrt_2_pi);+ HVX_Vector inner = hvx_vec_mul_f32_f32(x, coef);- const uint32_t src0_nrows = actx->src0_nrows;- const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread;+ // y2 = 2 * inner+ HVX_Vector y2 = hvx_vec_mul_f32_f32(inner, v_two);- const uint32_t src0_start_row = src0_nrows_per_thread * ith;- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);+ // Sigmoid guard check predicates+ HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(v_max_exp, y2);+ HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(y2, v_min_exp);- // no work for this thread- if (src0_start_row >= src0_end_row) {- return;- }+ // Fast sigmoid approximation+ HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(y2, v_log2f);+ v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), v_half);- const uint8_t * data_src0 = actx->data_src0;- uint8_t * data_dst = actx->data_dst;+ HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v));+ HVX_Vector x_sig = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int));+ HVX_Vector xx_sig = Q6_Vqf32_vmpy_Vqf32Vqf32(x_sig, x_sig);- const int ne0_val = actx->nc; // == dst->ne[0]+ HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx_sig), v_c2);+ v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, v_log2f);- uint8_t * src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread);- uint8_t * dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread);+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x_sig), v_c1);+ v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx_sig);+ v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x_sig);- size_t src0_spad_half_size = actx->src0_spad_half_size;- size_t dst_spad_half_size = actx->dst_spad_half_size;+ HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1));+ v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24);- const int BLOCK = actx->block;+ HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1));+ HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4));- if (BLOCK == 0) {- FARF(ERROR, "silu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n",- actx->octx->src0_spad.size_per_thread, src0_row_size_aligned);- return;- }+ // Fast division (Newton-Raphson with 2 iterations)+ HVX_Vector i_sf = Q6_Vw_vsub_VwVw(v_inv_aprox, v5);+ HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf(+ i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5)))));+ r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32(+ r_qf, Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5))));+ HVX_Vector res_inv = Q6_Vsf_equals_Vqf32(r_qf);- dma_queue * dma_queue = actx->octx->ctx->dma[ith];+ HVX_Vector sig2y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, res_inv));- // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379- for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {- const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);+ // Sigmoid guards+ sig2y = Q6_V_vmux_QVV(pred_max, sig2y, v_one);+ sig2y = Q6_V_vmux_QVV(pred_min, sig2y, Q6_V_vzero());- // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)- dma_queue_push_vtcm_to_ddr(dma_queue,- dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),- dst_row_size, dst_row_size_aligned, 0);+ // tanh(inner) = 2 * sigmoid(2 * inner) - 1+ HVX_Vector tanh_val = hvx_vec_mul_f32_f32(sig2y, v_two);+ tanh_val = hvx_vec_sub_f32_f32(tanh_val, v_one);- dma_queue_push_ddr_to_vtcm(dma_queue,- dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),- src0_row_size_aligned, src0_row_size, block_size);- }+ HVX_Vector tanh_plus_one = hvx_vec_add_f32_f32(tanh_val, v_one);+ HVX_Vector half_x = hvx_vec_mul_f32_f32(x, v_half);+ HVX_Vector gelu_x = hvx_vec_mul_f32_f32(half_x, tanh_plus_one);- for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {- const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);-- float* dst_spad = (float *) dma_queue_pop(dma_queue).src;- float* src0_spad = (float *) dma_queue_pop(dma_queue).dst;-- for (uint32_t ib = 0; ib < block_size; ib++) {- const float* src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float));- float* dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float));-- // silu = x * sigmoid(x)- hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, ne0_val);- hvx_mul_f32_aaa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0_val);- }-- dma_queue_push_vtcm_to_ddr(dma_queue,- dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad),- dst_row_size, dst_row_size_aligned, block_size);-- // prefetch N+2 loop iteration if any- const uint32_t pref_block = (ir + BLOCK * 2);- if (pref_block < src0_end_row) {- const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);- dma_queue_push_ddr_to_vtcm(dma_queue,- dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),- src0_row_size_aligned, src0_row_size, pref_block_size);- }+ vdst[i] = hvx_vec_mul_f32_f32(gelu_x, g);}- dma_queue_flush(dma_queue);+ if (nloe) {+ HVX_Vector x = vsrc0[i];+ HVX_Vector g = vsrc1[i];- t2 = HAP_perf_get_qtimer_count();+ HVX_Vector x2 = hvx_vec_mul_f32_f32(x, x);+ HVX_Vector coef = hvx_vec_mul_f32_f32(x2, v_coef_a_times_sqrt);+ coef = hvx_vec_add_f32_f32(coef, v_sqrt_2_pi);+ HVX_Vector inner = hvx_vec_mul_f32_f32(x, coef);- FARF(HIGH, "silu-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", ith, nth, ne00, ne01, ne02,- ne03, src0_start_row, src0_end_row, ne0, ne1, ne2, ne3, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));-}+ HVX_Vector y2 = hvx_vec_mul_f32_f32(inner, v_two);-static const float GELU_COEF_A = 0.044715f;-static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;+ HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(v_max_exp, y2);+ HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(y2, v_min_exp);-static void glu_geglu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {- struct htp_act_context * actx = (struct htp_act_context *) data;- htp_act_preamble;+ HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(y2, v_log2f);+ v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), v_half);- size_t src0_row_size = actx->src0_row_size;- size_t src1_row_size = actx->src1_row_size;- size_t dst_row_size = actx->dst_row_size;+ HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v));+ HVX_Vector x_sig = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int));+ HVX_Vector xx_sig = Q6_Vqf32_vmpy_Vqf32Vqf32(x_sig, x_sig);- uint64_t t1, t2;- t1 = HAP_perf_get_qtimer_count();+ HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx_sig), v_c2);+ v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, v_log2f);- const uint32_t src0_nrows = actx->src0_nrows;- const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread;+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x_sig), v_c1);+ v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx_sig);+ v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x_sig);- const uint32_t src0_start_row = src0_nrows_per_thread * ith;- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);+ HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1));+ v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24);- // no work for this thread- if (src0_start_row >= src0_end_row) {- return;- }+ HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1));+ HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4));- const uint8_t * restrict data_src0 = actx->data_src0;- const uint8_t * restrict data_src1 = actx->data_src1;- uint8_t * restrict data_dst = actx->data_dst;+ HVX_Vector i_sf = Q6_Vw_vsub_VwVw(v_inv_aprox, v5);+ HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf(+ i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5)))));+ r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32(+ r_qf, Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5))));+ HVX_Vector res_inv = Q6_Vsf_equals_Vqf32(r_qf);- const int nc = actx->nc;+ HVX_Vector sig2y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, res_inv));- const size_t src0_row_size_aligned = actx->src0_row_size_aligned;- const size_t src1_row_size_aligned = actx->src1_row_size_aligned;- const size_t dst_row_size_aligned = actx->dst_row_size_aligned;+ sig2y = Q6_V_vmux_QVV(pred_max, sig2y, v_one);+ sig2y = Q6_V_vmux_QVV(pred_min, sig2y, Q6_V_vzero());- uint8_t * restrict src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread);- uint8_t * restrict src1_spad_data = actx->octx->src1_spad.data + (ith * actx->octx->src1_spad.size_per_thread);- uint8_t * restrict dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread);+ HVX_Vector tanh_val = hvx_vec_mul_f32_f32(sig2y, v_two);+ tanh_val = hvx_vec_sub_f32_f32(tanh_val, v_one);- size_t src0_spad_half_size = actx->src0_spad_half_size;- size_t src1_spad_half_size = actx->src1_spad_half_size;- size_t dst_spad_half_size = actx->dst_spad_half_size;+ HVX_Vector tanh_plus_one = hvx_vec_add_f32_f32(tanh_val, v_one);+ HVX_Vector half_x = hvx_vec_mul_f32_f32(x, v_half);+ HVX_Vector gelu_x = hvx_vec_mul_f32_f32(half_x, tanh_plus_one);- const int BLOCK = actx->block;- if (BLOCK == 0) {- FARF(ERROR,- "geglu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n",- actx->octx->src0_spad.size_per_thread, src0_row_size_aligned);- return;+ HVX_Vector res = hvx_vec_mul_f32_f32(gelu_x, g);+ hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(float), res);}+}- dma_queue * dma_queue = actx->octx->ctx->dma[ith];-- // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379- for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {- const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);+// geglu(x, g) = gelu(x) * g+static void geglu_f32(const float * restrict src0,+ const float * restrict src1,+ float * restrict dst,+ const uint32_t num_rows,+ const struct htp_act_context * actx) {+ htp_glu_op_preamble;- // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)- dma_queue_push_vtcm_to_ddr(dma_queue,- dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),- dst_row_size, dst_row_size_aligned, 0);+ for (uint32_t ib = 0; ib < num_rows; ib++) {+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);- dma_queue_push_ddr_to_vtcm(dma_queue,- dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),- src0_row_size_aligned, src0_row_size, block_size);- dma_queue_push_ddr_to_vtcm(dma_queue,- dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)),- src1_row_size_aligned, src1_row_size, block_size);+ hvx_geglu_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc);}+}- for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {- const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);-- float * dst_spad = (float *) dma_queue_pop(dma_queue).src;- float * src0_spad = (float *) dma_queue_pop(dma_queue).dst;- float * src1_spad = (float *) dma_queue_pop(dma_queue).dst;-- for (uint32_t ib = 0; ib < block_size; ib++) {- const uint8_t * src0_spad_ptr = (const uint8_t *)(src0_spad + ib * (src0_row_size_aligned / sizeof(float)));- const uint8_t * src1_spad_ptr = (const uint8_t *)(src1_spad + ib * (src1_row_size_aligned / sizeof(float)));- uint8_t * dst_spad_ptr = (uint8_t *)(dst_spad + ib * (dst_row_size_aligned / sizeof(float)));-- // geglu tanh implementation- // geglu(x, g) = gelu(x) * g- // gelu(x) = 0.5f*x*(1.0f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)))- hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, src0_spad_ptr, nc); // res = x*x- hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, GELU_COEF_A, nc); // res = res * GELU_COEF_A- hvx_add_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, 1.0f, nc); // res = res + 1.0f- hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, (const uint8_t *)dst_spad_ptr, nc); // res = res * x- hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t*)dst_spad_ptr, SQRT_2_OVER_PI, nc); // res = result * SQRT_2_OVER_PI- hvx_tanh_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, nc); // res = tanh(res)- hvx_add_scalar_f32_aa(dst_spad_ptr, (const uint8_t*)dst_spad_ptr, 1.0f, nc); // res = res + 1.0f- hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, (const uint8_t *)dst_spad_ptr, nc); // res = res * x- hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, 0.5f, nc); // res = res + 0.5f- hvx_mul_f32_aaa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, src1_spad_ptr, nc); // res = res * g- }-- dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size,- dst_row_size_aligned, block_size);-- // prefetch N+2 loop iteration if any- const uint32_t pref_block = (ir + BLOCK * 2);- if (pref_block < src0_end_row) {- const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);- dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),- src0_row_size_aligned, src0_row_size, pref_block_size);- dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)),- src1_row_size_aligned, src1_row_size, pref_block_size);- }+#define DEFINE_GLU_PER_THREAD(NAME, OP_STR, CORE_EXPR) \+ static void glu_##NAME##_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { \+ struct htp_act_context * actx = (struct htp_act_context *) data; \+ htp_act_preamble; \+ \+ struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \+ \+ size_t src0_row_size = actx->src0_row_size; \+ size_t src1_row_size = actx->src1_row_size; \+ size_t dst_row_size = actx->dst_row_size; \+ \+ size_t src0_row_stride = actx->src0_row_stride; \+ size_t src1_row_stride = actx->src1_row_stride; \+ \+ const uint32_t src0_nrows = actx->src0_nrows; \+ const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \+ \+ const uint32_t src0_start_row = src0_nrows_per_thread * ith; \+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \+ \+ /* no work for this thread */ \+ if (src0_start_row >= src0_end_row) { \+ return; \+ } \+ \+ const uint8_t * restrict data_src0 = actx->data_src0; \+ const uint8_t * restrict data_src1 = actx->data_src1; \+ uint8_t * restrict data_dst = actx->data_dst; \+ \+ const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \+ const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \+ const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \+ \+ uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \+ uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \+ uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \+ \+ size_t src0_spad_half_size = actx->src0_spad_half_size; \+ size_t src1_spad_half_size = actx->src1_spad_half_size; \+ size_t dst_spad_half_size = actx->dst_spad_half_size; \+ \+ const int BLOCK = actx->block; \+ if (BLOCK == 0) { \+ FARF(ERROR, \+ OP_STR \+ " : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", \+ actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \+ return; \+ } \+ \+ dma_queue * dma_queue = actx->octx->ctx->dma[ith]; \+ \+ /* See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 */ \+ for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \+ const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \+ \+ /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ \+ dma_queue_push_vtcm_to_ddr(dma_queue, \+ dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), \+ dst_row_size, dst_row_size_aligned, 0); \+ \+ dma_queue_push( \+ dma_queue, \+ dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), \+ src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); \+ dma_queue_push( \+ dma_queue, \+ dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), \+ src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); \+ } \+ \+ for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { \+ const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \+ \+ float * dst_spad = (float *) dma_queue_pop(dma_queue).src; \+ float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; \+ float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; \+ \+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \+ CORE_EXPR; \+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \+ \+ dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), \+ dst_row_size, dst_row_size_aligned, block_size); \+ \+ /* prefetch N+2 loop iteration if any */ \+ const uint32_t pref_block = (ir + BLOCK * 2); \+ if (pref_block < src0_end_row) { \+ const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); \+ dma_queue_push(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_stride)), \+ src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); \+ dma_queue_push(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_stride)), \+ src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); \+ } \+ } \+ \+ dma_queue_flush(dma_queue); \+ \}- dma_queue_flush(dma_queue);-- t2 = HAP_perf_get_qtimer_count();-- FARF(HIGH, "geglu-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth,- ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3,- (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));-}+DEFINE_GLU_PER_THREAD(swiglu, "swiglu-f32", swiglu_f32(src0_spad, src1_spad, dst_spad, block_size, actx))+DEFINE_GLU_PER_THREAD(swiglu_oai, "swiglu-oai-f32", swiglu_oai_f32(src0_spad, src1_spad, dst_spad, block_size, actx))+DEFINE_GLU_PER_THREAD(geglu, "geglu-f32", geglu_f32(src0_spad, src1_spad, dst_spad, block_size, actx))static int execute_op_activations_f32(struct htp_ops_context * octx) {const struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * src1 = octx->src[1];const struct htp_tensor * dst = octx->dst;- if (((src0->ne[0] * SIZEOF_FP32) != src0->nb[1]) || ((dst->ne[0] * SIZEOF_FP32) != dst->nb[1])) {- FARF(ERROR, "Non-contiguous tensors are not supported at this time \n");+ if ((dst->ne[0] * SIZEOF_FP32) != dst->nb[1]) {+ FARF(ERROR, "Non-contiguous dst is not supported at this time \n");return HTP_STATUS_NO_SUPPORT;}@@ -636,11 +469,6 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {const char * op_type = NULL;switch (octx->op) {- case HTP_OP_UNARY_SILU:- act_op_func = (worker_callback_t)unary_silu_f32_per_thread;- op_type = "silu-f32";- break;-case HTP_OP_GLU_SWIGLU:act_op_func = (worker_callback_t)glu_swiglu_f32_per_thread;op_type = "swiglu-f32";@@ -650,10 +478,6 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {act_op_func = (worker_callback_t)glu_swiglu_oai_f32_per_thread;op_type = "swiglu-oai-f32";break;- case HTP_OP_UNARY_GELU:- act_op_func = (worker_callback_t)unary_gelu_f32_per_thread;- op_type = "gelu-f32";- break;case HTP_OP_GLU_GEGLU:act_op_func = (worker_callback_t)glu_geglu_f32_per_thread;@@ -667,51 +491,39 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);- size_t src0_row_size = src0->nb[1];- size_t src1_row_size = src1 ? src1->nb[1] : src0->nb[1];- size_t dst_row_size = dst->nb[1];+ // row_size = bytes of useful data per row (what the kernel touches / what DMA copies).+ // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src).+ const size_t nc_bytes = dst->ne[0] * SIZEOF_FP32;+ const size_t src0_row_size = nc_bytes;+ const size_t src1_row_size = nc_bytes;+ const size_t dst_row_size = nc_bytes;+ const size_t src0_row_stride = src0->nb[1];+ const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1];const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);- // VTCM scratchpads for all tensors- // N rows per thread, padded to HVX vector size- size_t spad_size_per_row = (src0_row_size_aligned + src1_row_size_aligned) + dst_row_size_aligned;- size_t vtcm_row_per_thread = (octx->ctx->vtcm_size)/ (n_threads* spad_size_per_row);+ struct htp_act_vtcm_layout L;+ htp_act_vtcm_layout_build(&L, src0_row_size_aligned, src1_row_size_aligned, dst_row_size_aligned, n_threads,+ octx->ctx->vtcm_size);// Make sure the reserved vtcm size is sufficient- if (vtcm_row_per_thread == 0) {+ if (L.vtcm_row_per_thread == 0) {FARF(ERROR, "act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,- spad_size_per_row * n_threads);+ (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads);return HTP_STATUS_VTCM_TOO_SMALL;}- octx->src0_spad.size_per_thread = src0_row_size_aligned * vtcm_row_per_thread;- octx->src1_spad.size_per_thread = src1_row_size_aligned * vtcm_row_per_thread;- octx->dst_spad.size_per_thread = dst_row_size_aligned * vtcm_row_per_thread;-- octx->dst_spad.size = n_threads* octx->dst_spad.size_per_thread;- octx->src0_spad.size = n_threads* octx->src0_spad.size_per_thread;- octx->src1_spad.size = n_threads* octx->src1_spad.size_per_thread;-- octx->src0_spad.data = octx->ctx->vtcm_base;- octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;- octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size;-- octx->src0_spad.src = NULL;- octx->src1_spad.src = NULL;- octx->dst_spad.src = NULL;-if (src1) {- FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\n",+ FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n",op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2],- src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], octx->src0_spad.size, octx->src1_spad.size,- octx->dst_spad.size);+ src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads,+ L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads);} else {- FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\n", op_type,+ FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type,src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],- octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size);+ L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads);}if ((octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {@@ -732,9 +544,21 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {actx.src1_row_size_aligned = src1_row_size_aligned;actx.dst_row_size_aligned = dst_row_size_aligned;- actx.src0_spad_half_size = octx->src0_spad.size_per_thread / 2;- actx.src1_spad_half_size = octx->src1_spad.size_per_thread / 2;- actx.dst_spad_half_size = octx->dst_spad.size_per_thread / 2;+ actx.src0_row_stride = src0_row_stride;+ actx.src1_row_stride = src1_row_stride;++ uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;+ actx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);+ actx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);+ actx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);++ actx.vtcm_src0_size_per_thread = L.src0_bytes_per_thread;+ actx.vtcm_src1_size_per_thread = L.src1_bytes_per_thread;+ actx.vtcm_dst_size_per_thread = L.dst_bytes_per_thread;++ actx.src0_spad_half_size = L.src0_bytes_per_thread / 2;+ actx.src1_spad_half_size = L.src1_bytes_per_thread / 2;+ actx.dst_spad_half_size = L.dst_bytes_per_thread / 2;actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned;actx.src0_nrows = src0_nrows;diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.cindex e76ac7f488..321b832e14 100644--- a/ggml/src/ggml-hexagon/htp/main.c+++ b/ggml/src/ggml-hexagon/htp/main.c@@ -723,14 +723,14 @@ static int execute_op(struct htp_ops_context * octx) {case HTP_OP_SQRT:case HTP_OP_UNARY_SOFTPLUS:case HTP_OP_UNARY_SIGMOID:+ case HTP_OP_UNARY_SILU:+ case HTP_OP_UNARY_GELU:case HTP_OP_UNARY_NEG:case HTP_OP_UNARY_EXP:case HTP_OP_UNARY_TANH:case HTP_OP_L2_NORM:return op_unary(octx);- case HTP_OP_UNARY_SILU:- case HTP_OP_UNARY_GELU:case HTP_OP_GLU_SWIGLU:case HTP_OP_GLU_SWIGLU_OAI:case HTP_OP_GLU_GEGLU:diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.cindex c6806efc37..b21415a67d 100644--- a/ggml/src/ggml-hexagon/htp/unary-ops.c+++ b/ggml/src/ggml-hexagon/htp/unary-ops.c@@ -276,6 +276,39 @@ static void sigmoid_f32(const float * restrict src,}}+// silu(x) = x * sigmoid(x)+static void silu_f32(const float * restrict src,+ float * restrict dst,+ const uint32_t num_rows,+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;++ for (uint32_t ir = 0; ir < num_rows; ir++) {+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);++ hvx_sigmoid_f32_aa(dst_local, src_local, ne0);+ hvx_mul_f32_aaa(dst_local, src_local, dst_local, ne0);+ }+}++// gelu(x) = x * sigmoid(1.702 * x) (quick/sigmoid approximation, matches CPU GELU_QUICK reference)+static void gelu_f32(const float * restrict src,+ float * restrict dst,+ const uint32_t num_rows,+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;++ for (uint32_t ir = 0; ir < num_rows; ir++) {+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);++ hvx_mul_scalar_f32(dst_local, src_local, 1.702f, ne0);+ hvx_sigmoid_f32_aa(dst_local, dst_local, ne0);+ hvx_mul_f32_aaa(dst_local, src_local, dst_local, ne0);+ }+}+static void tri_f32(const float * restrict src,float * restrict dst,const uint32_t num_rows,@@ -566,6 +599,8 @@ DEFINE_UNARY_TASK(sqrt, false, false, sqrt_f32(src0_vtcm, dst_vtcm, blDEFINE_UNARY_TASK(unary_neg, false, false, neg_f32(src0_vtcm, dst_vtcm, block_size, uctx))DEFINE_UNARY_TASK(unary_exp, false, false, exp_f32(src0_vtcm, dst_vtcm, block_size, uctx))DEFINE_UNARY_TASK(unary_sigmoid, false, false, sigmoid_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(unary_silu, false, false, silu_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(unary_gelu, false, false, gelu_f32(src0_vtcm, dst_vtcm, block_size, uctx))DEFINE_UNARY_TASK(unary_softplus, false, false, softplus_f32(src0_vtcm, dst_vtcm, block_size, uctx))DEFINE_UNARY_TASK(unary_tanh, false, false, tanh_f32(src0_vtcm, dst_vtcm, block_size, uctx))DEFINE_UNARY_TASK(l2_norm, false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))@@ -717,6 +752,19 @@ static inline void tile_unary_softplus_f32(uint8_t * dst_vtcm, const uint8_t * s}}+// silu(x) = x * sigmoid(x)+static inline void tile_silu_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw) {+ hvx_sigmoid_f32_aa(dst_vtcm, src_vtcm, tw);+ hvx_mul_f32_aaa(dst_vtcm, src_vtcm, dst_vtcm, tw);+}++// gelu(x) = x * sigmoid(1.702 * x) (quick/sigmoid approximation, matches CPU GELU_QUICK reference)+static inline void tile_gelu_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw) {+ hvx_mul_scalar_f32(dst_vtcm, src_vtcm, 1.702f, tw);+ hvx_sigmoid_f32_aa(dst_vtcm, dst_vtcm, tw);+ hvx_mul_f32_aaa(dst_vtcm, src_vtcm, dst_vtcm, tw);+}+// Triangular mask applied to one column tile. Boundary is an absolute column index, so// each vector compares against its absolute column position (col_start + i*VLEN_FP32).static inline void tri_apply_tile_f32(const uint8_t * restrict src, uint8_t * restrict dst,@@ -798,6 +846,8 @@ DEFINE_UNARY_TILED_TASK(sqrt, false, hvx_sqrt_f32_aa(dst_vtcm, src_vtcDEFINE_UNARY_TILED_TASK(unary_neg, false, hvx_scale_f32_aa(dst_vtcm, src_vtcm, tw, -1.0f))DEFINE_UNARY_TILED_TASK(unary_exp, false, hvx_exp_f32(dst_vtcm, src_vtcm, tw, false))DEFINE_UNARY_TILED_TASK(unary_sigmoid, false, hvx_sigmoid_f32_aa(dst_vtcm, src_vtcm, tw))+DEFINE_UNARY_TILED_TASK(unary_silu, false, tile_silu_f32(dst_vtcm, src_vtcm, tw))+DEFINE_UNARY_TILED_TASK(unary_gelu, false, tile_gelu_f32(dst_vtcm, src_vtcm, tw))DEFINE_UNARY_TILED_TASK(unary_softplus, false, tile_unary_softplus_f32(dst_vtcm, src_vtcm, tw))DEFINE_UNARY_TILED_TASK(unary_tanh, false, hvx_tanh_f32_aa(dst_vtcm, src_vtcm, tw))DEFINE_UNARY_TILED_TASK(tri, true, tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype))@@ -821,6 +871,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break;case HTP_OP_UNARY_EXP: op_type = "exp-f32"; break;case HTP_OP_UNARY_SIGMOID: op_type = "sigmoid-f32"; break;+ case HTP_OP_UNARY_SILU: op_type = "silu-f32"; break;+ case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break;case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break;case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break;case HTP_OP_L2_NORM: op_type = "l2norm-f32"; break;@@ -917,6 +969,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {case HTP_OP_UNARY_NEG: task_func = unary_task_f32_tiled_unary_neg; break;case HTP_OP_UNARY_EXP: task_func = unary_task_f32_tiled_unary_exp; break;case HTP_OP_UNARY_SIGMOID: task_func = unary_task_f32_tiled_unary_sigmoid; break;+ case HTP_OP_UNARY_SILU: task_func = unary_task_f32_tiled_unary_silu; break;+ case HTP_OP_UNARY_GELU: task_func = unary_task_f32_tiled_unary_gelu; break;case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_tiled_unary_softplus; break;case HTP_OP_UNARY_TANH: task_func = unary_task_f32_tiled_unary_tanh; break;case HTP_OP_TRI: task_func = unary_task_f32_tiled_tri; break;@@ -934,6 +988,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {case HTP_OP_UNARY_NEG: task_func = unary_task_f32_unary_neg; break;case HTP_OP_UNARY_EXP: task_func = unary_task_f32_unary_exp; break;case HTP_OP_UNARY_SIGMOID: task_func = unary_task_f32_unary_sigmoid; break;+ case HTP_OP_UNARY_SILU: task_func = unary_task_f32_unary_silu; break;+ case HTP_OP_UNARY_GELU: task_func = unary_task_f32_unary_gelu; break;case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_unary_softplus; break;case HTP_OP_UNARY_TANH: task_func = unary_task_f32_unary_tanh; break;case HTP_OP_L2_NORM: task_func = unary_task_f32_l2_norm; break;diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.hindex 4005e65151..1f4c3a5c4d 100644--- a/ggml/src/ggml-hexagon/htp/unary-ops.h+++ b/ggml/src/ggml-hexagon/htp/unary-ops.h@@ -51,6 +51,8 @@ static inline bool htp_op_is_unary(uint32_t opcode) {case HTP_OP_UNARY_NEG:case HTP_OP_UNARY_EXP:case HTP_OP_UNARY_SIGMOID:+ case HTP_OP_UNARY_SILU:+ case HTP_OP_UNARY_GELU:case HTP_OP_UNARY_SOFTPLUS:case HTP_OP_UNARY_TANH:case HTP_OP_L2_NORM: