LLM inference in C/C++
C++ ggml
Hexagon f16 activation ops (#29209)
cqderek19e28a277021
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex 161eee248e..4b913947d8 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -6037,10 +6037,10 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_sessionconst struct ggml_tensor * src1 = op->src[1];const struct ggml_tensor * dst = op;- if (src0->type != GGML_TYPE_F32) {+ if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) {return false;}- if (dst->type != GGML_TYPE_F32) {+ if (dst->type != src0->type) {return false;}@@ -6052,7 +6052,7 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session}if (src1) {- if (src1->type != GGML_TYPE_F32) {+ if (src1->type != src0->type) {return false;}if (!ggml_are_same_shape(src0, src1)) {@@ -6637,12 +6637,12 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {case GGML_OP_GLU:switch (ggml_get_glu_op(t)) {- case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU;- case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI;+ case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU;+ case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI;case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP;- case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU;- case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK;- case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF;+ case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU;+ case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK;+ case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF;default: break;}break;diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.cindex 28c581df41..59a0575c4b 100644--- a/ggml/src/ggml-hexagon/htp/act-ops.c+++ b/ggml/src/ggml-hexagon/htp/act-ops.c@@ -55,16 +55,22 @@struct htp_act_context;-typedef void (*glu_compute_fn_t)(const float * restrict src0,- const float * restrict src1,- float * restrict dst,- const uint32_t num_rows,- const struct htp_act_context * actx);+typedef void (*glu_compute_f32_fn_t)(const float * restrict src0,+ const float * restrict src1,+ float * restrict dst,+ const uint32_t num_rows,+ const struct htp_act_context * actx);++typedef void (*glu_compute_f16_fn_t)(const _Float16 * restrict src0,+ const _Float16 * restrict src1,+ _Float16 * restrict dst,+ const uint32_t num_rows,+ const struct htp_act_context * actx);struct htp_act_context {struct htp_ops_context * octx;- glu_compute_fn_t compute;+ void * compute;const char * op_str;// Precomputed values@@ -392,6 +398,58 @@ static inline void hvx_geglu_erf_f32_aa(uint8_t * restrict dst, const uint8_t *}}+static inline void hvx_geglu_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {+ assert((unsigned long) dst % 128 == 0);+ assert((unsigned long) src0 % 128 == 0);+ assert((unsigned long) src1 % 128 == 0);++ HVX_Vector * restrict vdst = (HVX_Vector *) dst;+ const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0;+ const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1;++ const uint32_t epv = 128 / sizeof(_Float16);+ const uint32_t nvec = n / epv;+ const uint32_t nloe = n % epv;++ const HVX_Vector v_coef_a = hvx_vec_splat_f16((_Float16) GELU_COEF_A);+ const HVX_Vector v_sqrt_2_pi = hvx_vec_splat_f16((_Float16) SQRT_2_OVER_PI);+ const HVX_Vector v_one = hvx_vec_splat_f16((_Float16) 1.0f);+ const HVX_Vector v_half = hvx_vec_splat_f16((_Float16) 0.5f);++ uint32_t i = 0;++ _Pragma("unroll(4)")+ for (; i < nvec; i++) {+ HVX_Vector x = vsrc0[i];+ HVX_Vector g = vsrc1[i];++ // gelu(x) = 0.5*x*(1 + tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x)))+ HVX_Vector x2 = hvx_vec_mul_f16_f16(x, x);+ HVX_Vector coef = hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(x2, v_coef_a), v_one);+ HVX_Vector inner = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, coef), v_sqrt_2_pi);++ HVX_Vector t = hvx_vec_add_f16_f16(hvx_vec_tanh_f16(inner), v_one);++ HVX_Vector gelu_x = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, t), v_half);+ vdst[i] = hvx_vec_mul_f16_f16(gelu_x, g);+ }++ if (nloe) {+ HVX_Vector x = vsrc0[i];+ HVX_Vector g = vsrc1[i];++ HVX_Vector x2 = hvx_vec_mul_f16_f16(x, x);+ HVX_Vector coef = hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(x2, v_coef_a), v_one);+ HVX_Vector inner = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, coef), v_sqrt_2_pi);++ HVX_Vector t = hvx_vec_add_f16_f16(hvx_vec_tanh_f16(inner), v_one);++ HVX_Vector gelu_x = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, t), v_half);+ HVX_Vector res = hvx_vec_mul_f16_f16(gelu_x, g);+ hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(_Float16), res);+ }+}+// geglu(x, g) = gelu(x) * gstatic void geglu_f32(const float * restrict src0,const float * restrict src1,@@ -443,100 +501,189 @@ static void geglu_erf_f32(const float * restrict src0,}}-static void glu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {- struct htp_act_context * actx = (struct htp_act_context *) data;- htp_act_preamble;-- struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL;-- size_t src0_row_size = actx->src0_row_size;- size_t src1_row_size = actx->src1_row_size;- size_t dst_row_size = actx->dst_row_size;-- size_t src0_row_stride = actx->src0_row_stride;- size_t src1_row_stride = actx->src1_row_stride;-- const uint32_t src0_nrows = actx->src0_nrows;- const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread;+// swiglu(x) = x1 * sigmoid(x0)+static void swiglu_f16(const _Float16 * restrict src0,+ const _Float16 * restrict src1,+ _Float16 * restrict dst,+ const uint32_t num_rows,+ const struct htp_act_context * actx) {+ htp_glu_op_preamble;- const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith;- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows);+ for (uint32_t ib = 0; ib < num_rows; ib++) {+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);- /* no work for this thread */- if (src0_start_row >= src0_end_row) {- return;+ hvx_sigmoid_f16_aa(dst_ptr, src0_ptr, nc);+ hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc);}+}- const dma_addr_t data_src0 = actx->data_src0;- const dma_addr_t data_src1 = actx->data_src1;- const dma_addr_t data_dst = actx->data_dst;-- const size_t src0_row_size_aligned = actx->src0_row_size_aligned;- const size_t src1_row_size_aligned = actx->src1_row_size_aligned;- const size_t dst_row_size_aligned = actx->dst_row_size_aligned;-- uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread);- uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread);- uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread);+// out = x * sigmoid(alpha * x) * (clamp(y, -limit, limit) + 1.f)+static void swiglu_oai_f16(const _Float16 * restrict src0,+ const _Float16 * restrict src1,+ _Float16 * restrict dst,+ const uint32_t num_rows,+ const struct htp_act_context * actx) {+ htp_glu_op_preamble;+ const _Float16 alpha = (_Float16) ((const float *) (actx->octx->op_params))[2];+ const _Float16 limit = (_Float16) ((const float *) (actx->octx->op_params))[3];- size_t src0_spad_half_size = actx->src0_spad_half_size;- size_t src1_spad_half_size = actx->src1_spad_half_size;- size_t dst_spad_half_size = actx->dst_spad_half_size;+ for (uint32_t ib = 0; ib < num_rows; ib++) {+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);- const int BLOCK = actx->block;- if (BLOCK == 0) {- FARF(ERROR, "%s : VTCM reservation %zu is too small, needed %zu\n",- actx->op_str, actx->vtcm_src0_size_per_thread, src0_row_size_aligned);- return;+ // x (src0_ptr) = std::min(src0_p[k], limit);+ hvx_min_scalar_f16((uint8_t *) src0_ptr, src0_ptr, limit, nc);+ // y1 (src1_ptr) = std::clamp(src1_p[k], -limit, limit);+ hvx_clamp_scalar_f16((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc);+ // y (src1_ptr) = y1 + 1.f+ hvx_add_scalar_f16((uint8_t *) src1_ptr, src1_ptr, (_Float16) 1.0f, nc);+ // x1 (dst_ptr) = alpha * x+ hvx_mul_scalar_f16(dst_ptr, src0_ptr, alpha, nc);+ // x2 (dst_ptr) = sigmoid(x1) = 1/(1+exp(-x1))+ hvx_sigmoid_f16_aa(dst_ptr, dst_ptr, nc);+ // out = x * sigmoid(alpha * x) * (y + 1.f)+ hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc);}+}- dma_queue * dma_q = actx->octx->ctx->dma[ith];- glu_compute_fn_t compute = actx->compute;-- for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {- const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);-- /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */- dma_queue_push(dma_q,- dma_make_data(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),- dst_row_size, dst_row_size_aligned, dst_row_size, 0);+static void swiglu_clamp_f16(const _Float16 * restrict src0,+ const _Float16 * restrict src1,+ _Float16 * restrict dst,+ const uint32_t num_rows,+ const struct htp_act_context * actx) {+ htp_glu_op_preamble;+ const _Float16 limit = (_Float16) ((const float *) (actx->octx->op_params))[3];- dma_queue_push(dma_q,- dma_make_data(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)),- src0_row_size_aligned, src0_row_stride, src0_row_size, block_size);+ for (uint32_t ib = 0; ib < num_rows; ib++) {+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);- dma_queue_push(dma_q,- dma_make_data(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)),- src1_row_size_aligned, src1_row_stride, src1_row_size, block_size);+ hvx_min_scalar_f16((uint8_t *) src0_ptr, src0_ptr, limit, nc);+ hvx_clamp_scalar_f16((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc);+ hvx_sigmoid_f16_aa(dst_ptr, src0_ptr, nc);+ hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc);}+}- for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {- const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);-- float * dst_spad = (float *) dma_queue_pop(dma_q).src;- float * src0_spad = (float *) dma_queue_pop(dma_q).dst;- float * src1_spad = (float *) dma_queue_pop(dma_q).dst;+// geglu(x, g) = gelu(x) * g, gelu via tanh approximation+static void geglu_f16(const _Float16 * restrict src0,+ const _Float16 * restrict src1,+ _Float16 * restrict dst,+ const uint32_t num_rows,+ const struct htp_act_context * actx) {+ htp_glu_op_preamble;- htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir);- compute(src0_spad, src1_spad, dst_spad, block_size, actx);- htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir);+ for (uint32_t ib = 0; ib < num_rows; ib++) {+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);- dma_queue_push(dma_q, dma_make_data(data_dst + (ir * dst_row_size), dst_spad),- dst_row_size, dst_row_size_aligned, dst_row_size, block_size);+ hvx_geglu_f16_aa(dst_ptr, src0_ptr, src1_ptr, nc);+ }+}- /* prefetch N+2 loop iteration if any */- const uint32_t pref_block = (ir + BLOCK * 2);- if (pref_block < src0_end_row) {- const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);- dma_queue_push(dma_q, dma_make_data(src0_spad, data_src0 + (pref_block * src0_row_stride)),- src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size);- dma_queue_push(dma_q, dma_make_data(src1_spad, data_src1 + (pref_block * src1_row_stride)),- src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size);- }+#define DEFINE_GLU_PER_THREAD(TYPE, SUFFIX) \+ static void glu_##SUFFIX##_per_thread(unsigned int nth, unsigned int ith, void * data) { \+ struct htp_act_context * actx = (struct htp_act_context *) data; \+ htp_act_preamble; \+ \+ struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \+ \+ size_t src0_row_size = actx->src0_row_size; \+ size_t src1_row_size = actx->src1_row_size; \+ size_t dst_row_size = actx->dst_row_size; \+ \+ size_t src0_row_stride = actx->src0_row_stride; \+ size_t src1_row_stride = actx->src1_row_stride; \+ \+ const uint32_t src0_nrows = actx->src0_nrows; \+ const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \+ \+ const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith; \+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows); \+ \+ /* no work for this thread */ \+ if (src0_start_row >= src0_end_row) { \+ return; \+ } \+ \+ const dma_addr_t data_src0 = actx->data_src0; \+ const dma_addr_t data_src1 = actx->data_src1; \+ const dma_addr_t data_dst = actx->data_dst; \+ \+ const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \+ const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \+ const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \+ \+ uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \+ uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \+ uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \+ \+ size_t src0_spad_half_size = actx->src0_spad_half_size; \+ size_t src1_spad_half_size = actx->src1_spad_half_size; \+ size_t dst_spad_half_size = actx->dst_spad_half_size; \+ \+ const int BLOCK = actx->block; \+ if (BLOCK == 0) { \+ FARF(ERROR, "%s : VTCM reservation %zu is too small, needed %zu\n", \+ actx->op_str, actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \+ return; \+ } \+ \+ dma_queue * dma_q = actx->octx->ctx->dma[ith]; \+ glu_compute_##SUFFIX##_fn_t compute = (glu_compute_##SUFFIX##_fn_t) actx->compute; \+ \+ for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \+ const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \+ \+ /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ \+ dma_queue_push(dma_q, \+ dma_make_data(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), \+ dst_row_size, dst_row_size_aligned, dst_row_size, 0); \+ \+ dma_queue_push(dma_q, \+ dma_make_data(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), \+ src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); \+ \+ dma_queue_push(dma_q, \+ dma_make_data(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), \+ src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); \+ } \+ \+ for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { \+ const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \+ \+ TYPE * dst_spad = (TYPE *) dma_queue_pop(dma_q).src; \+ TYPE * src0_spad = (TYPE *) dma_queue_pop(dma_q).dst; \+ TYPE * src1_spad = (TYPE *) dma_queue_pop(dma_q).dst; \+ \+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \+ compute(src0_spad, src1_spad, dst_spad, block_size, actx); \+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \+ \+ dma_queue_push(dma_q, dma_make_data(data_dst + (ir * dst_row_size), dst_spad), \+ dst_row_size, dst_row_size_aligned, dst_row_size, block_size); \+ \+ /* prefetch N+2 loop iteration if any */ \+ const uint32_t pref_block = (ir + BLOCK * 2); \+ if (pref_block < src0_end_row) { \+ const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); \+ dma_queue_push(dma_q, dma_make_data(src0_spad, data_src0 + (pref_block * src0_row_stride)), \+ src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); \+ dma_queue_push(dma_q, dma_make_data(src1_spad, data_src1 + (pref_block * src1_row_stride)), \+ src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); \+ } \+ } \+ \+ dma_queue_flush(dma_q); \}- dma_queue_flush(dma_q);-}+DEFINE_GLU_PER_THREAD(float, f32)+DEFINE_GLU_PER_THREAD(_Float16, f16)static int execute_op_activations_f32(struct htp_ops_context * octx) {const struct htp_tensor * src0 = octx->src[0];@@ -548,8 +695,8 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {return HTP_STATUS_NO_SUPPORT;}- glu_compute_fn_t compute_fn = NULL;- const char * op_type = NULL;+ glu_compute_f32_fn_t compute_fn = NULL;+ const char * op_type = NULL;switch (octx->op) {case HTP_OP_GLU_SWIGLU:@@ -581,6 +728,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {compute_fn = geglu_erf_f32;op_type = "geglu-erf-f32";break;+default:FARF(ERROR, "Unsupported activations Op %u\n", octx->op);return HTP_STATUS_NO_SUPPORT;@@ -643,7 +791,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {// Prepare contextstruct htp_act_context actx;actx.octx = octx;- actx.compute = compute_fn;+ actx.compute = (void *) compute_fn;actx.op_str = op_type;actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);@@ -708,11 +856,174 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {return HTP_STATUS_OK;}+static int execute_op_activations_f16(struct htp_ops_context * octx) {+ const struct htp_tensor * src0 = octx->src[0];+ const struct htp_tensor * src1 = octx->src[1];+ const struct htp_tensor * dst = octx->dst;++ if ((dst->ne[0] * SIZEOF_FP16) != dst->nb[1]) {+ FARF(ERROR, "Non-contiguous dst is not supported at this time \n");+ return HTP_STATUS_NO_SUPPORT;+ }++ glu_compute_f16_fn_t compute_fn = NULL;+ const char * op_type = NULL;++ switch (octx->op) {+ case HTP_OP_GLU_SWIGLU:+ compute_fn = swiglu_f16;+ op_type = "swiglu-f16";+ break;++ case HTP_OP_GLU_SWIGLU_OAI:+ compute_fn = swiglu_oai_f16;+ op_type = "swiglu-oai-f16";+ break;++ case HTP_OP_GLU_SWIGLU_CLAMP:+ compute_fn = swiglu_clamp_f16;+ op_type = "swiglu-clamp-f16";+ break;++ case HTP_OP_GLU_GEGLU:+ compute_fn = geglu_f16;+ op_type = "geglu-f16";+ break;++ default:+ FARF(ERROR, "Unsupported activations Op %u\n", octx->op);+ return HTP_STATUS_NO_SUPPORT;+ }++ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];+ const size_t dst_row_size = dst->ne[0] * SIZEOF_FP16;++ uint32_t row_start = 0;+ uint32_t nrows = src0_nrows;++ if (octx->ctx->mdev.count > 1) {+ uint32_t rows_per_chunk = 0;+ htp_tensor_mdev_rows_per_chunk(dst, sizeof(_Float16), (uint32_t) dst_row_size, &rows_per_chunk);+ const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);+ row_start = range.start;+ nrows = range.count;+ }++ if (nrows == 0) {+ return HTP_STATUS_OK;+ }++ const uint32_t n_threads = octx->n_threads;++ // row_size = bytes of useful data per row (what the kernel touches / what DMA copies).+ // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src).+ const size_t nc_bytes = dst_row_size;+ const size_t src0_row_size = nc_bytes;+ const size_t src1_row_size = nc_bytes;+ const size_t src0_row_stride = src0->nb[1];+ const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1];++ const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);+ const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);+ const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);++ struct htp_act_vtcm_layout L;+ htp_act_vtcm_layout_build(&L, src0_row_size_aligned, src1_row_size_aligned, dst_row_size_aligned, n_threads,+ octx->ctx->vtcm_size);++ // Make sure the reserved vtcm size is sufficient+ if (L.vtcm_row_per_thread == 0) {+ FARF(ERROR, "act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,+ (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads);+ return HTP_STATUS_VTCM_TOO_SMALL;+ }++ if (src1) {+ FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n",+ op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2],+ src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads,+ L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads);+ } else {+ FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type,+ src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],+ L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads);+ }++ // Prepare context+ struct htp_act_context actx;+ actx.octx = octx;+ actx.compute = (void *) compute_fn;+ actx.op_str = op_type;++ actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);++ actx.src0_row_size = src0_row_size;+ actx.src1_row_size = src1_row_size;+ actx.dst_row_size = dst_row_size;++ actx.src0_row_size_aligned = src0_row_size_aligned;+ actx.src1_row_size_aligned = src1_row_size_aligned;+ actx.dst_row_size_aligned = dst_row_size_aligned;++ actx.src0_row_stride = src0_row_stride;+ actx.src1_row_stride = src1_row_stride;++ uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;+ actx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);+ actx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);+ actx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);++ actx.vtcm_src0_size_per_thread = L.src0_bytes_per_thread;+ actx.vtcm_src1_size_per_thread = L.src1_bytes_per_thread;+ actx.vtcm_dst_size_per_thread = L.dst_bytes_per_thread;++ actx.src0_spad_half_size = L.src0_bytes_per_thread / 2;+ actx.src1_spad_half_size = L.src1_bytes_per_thread / 2;+ actx.dst_spad_half_size = L.dst_bytes_per_thread / 2;++ actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned;+ actx.src0_nrows = nrows;+ actx.row_start = row_start;++ actx.nc = dst->ne[0];++ // Addresses and GLU logic+ dma_addr_t data_src0 = src0->data;+ dma_addr_t data_src1 = src1 ? src1->data : 0;++ if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU ||+ octx->op == HTP_OP_GLU_SWIGLU_OAI ||+ octx->op == HTP_OP_GLU_SWIGLU_CLAMP ||+ octx->op == HTP_OP_GLU_GEGLU)) {+ const int32_t swapped = octx->op_params[1];+ data_src1 = data_src0;+ actx.src1_row_size = actx.src0_row_size;++ size_t nc_in_bytes = actx.nc * SIZEOF_FP16;+ if (swapped) {+ data_src0 += nc_in_bytes;+ } else {+ data_src1 += nc_in_bytes;+ }+ }++ actx.data_src0 = data_src0;+ actx.data_src1 = data_src1;+ actx.data_dst = dst->data;++ work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f16_per_thread, &actx, n_threads);+ return HTP_STATUS_OK;+}++int op_activations(struct htp_ops_context * octx) {switch (octx->src[0]->type) {case HTP_TYPE_F32:return execute_op_activations_f32(octx);+ case HTP_TYPE_F16:+ return execute_op_activations_f16(octx);+default:return HTP_STATUS_NO_SUPPORT;}diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.hindex c9a776f625..ab2064af3b 100644--- a/ggml/src/ggml-hexagon/htp/hvx-arith.h+++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h@@ -164,6 +164,37 @@ static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * re}}+static inline void hvx_mul_mul_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint8_t * restrict src2, const uint32_t num_elems) {+ assert((unsigned long) dst % 128 == 0);+ assert((unsigned long) src0 % 128 == 0);+ assert((unsigned long) src1 % 128 == 0);+ assert((unsigned long) src2 % 128 == 0);++ HVX_Vector * restrict vdst = (HVX_Vector *) dst;+ HVX_Vector * restrict vsrc0 = (HVX_Vector *) src0;+ HVX_Vector * restrict vsrc1 = (HVX_Vector *) src1;+ HVX_Vector * restrict vsrc2 = (HVX_Vector *) src2;++ const uint32_t elem_size = sizeof(_Float16);+ const uint32_t epv = 128 / elem_size;+ const uint32_t nvec = num_elems / epv;+ const uint32_t nloe = num_elems % epv;++ uint32_t i = 0;++ _Pragma("unroll(4)")+ for (; i < nvec; i++) {+ HVX_Vector v1 = HVX_OP_MUL_F16(vsrc0[i], vsrc1[i]);+ vdst[i] = HVX_OP_MUL_F16(v1, vsrc2[i]);+ }++ if (nloe) {+ HVX_Vector v1 = HVX_OP_MUL_F16(vsrc0[i], vsrc1[i]);+ HVX_Vector v2 = HVX_OP_MUL_F16(v1, vsrc2[i]);+ hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v2);+ }+}+// Scalar Operations#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \@@ -348,6 +379,44 @@ static inline void hvx_max_scalar_f32(uint8_t * restrict dst, const uint8_t * re}}+#define HVX_OP_MIN_SCALAR_F16(v) Q6_Vhf_vmin_VhfVhf(val_vec, v)++static inline void hvx_min_scalar_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {+ const HVX_Vector val_vec = hvx_vec_splat_f16(val);+ assert((unsigned long) dst % 128 == 0);+ assert((unsigned long) src % 128 == 0);+ hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_MIN_SCALAR_F16);+}++static inline void hvx_min_scalar_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {+ const HVX_Vector val_vec = hvx_vec_splat_f16(val);+ assert((unsigned long) dst % 128 == 0);+ hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_MIN_SCALAR_F16);+}++static inline void hvx_min_scalar_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {+ const HVX_Vector val_vec = hvx_vec_splat_f16(val);+ assert((unsigned long) src % 128 == 0);+ hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_MIN_SCALAR_F16);+}++static inline void hvx_min_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {+ const HVX_Vector val_vec = hvx_vec_splat_f16(val);+ hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_MIN_SCALAR_F16);+}++static inline void hvx_min_scalar_f16(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, const int num_elems) {+ if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) {+ hvx_min_scalar_f16_aa(dst, src, val, num_elems);+ } else if (hex_is_aligned((void *) dst, 128)) {+ hvx_min_scalar_f16_au(dst, src, val, num_elems);+ } else if (hex_is_aligned((void *) src, 128)) {+ hvx_min_scalar_f16_ua(dst, src, val, num_elems);+ } else {+ hvx_min_scalar_f16_uu(dst, src, val, num_elems);+ }+}+// CLAMP Scalar variants#define HVX_OP_CLAMP_SCALAR(v) \@@ -767,10 +836,12 @@ static inline void hvx_sqr_f16(uint8_t * restrict dst, const uint8_t * restrict#undef hvx_scalar_loop_body#undef HVX_OP_MIN_SCALAR#undef HVX_OP_MAX_SCALAR+#undef HVX_OP_MIN_SCALAR_F16#undef HVX_OP_CLAMP_SCALAR#undef HVX_OP_CLAMP_SCALAR_F16#undef HVX_OP_LEAKY_RELU_SCALAR#undef DEFINE_HVX_BINARY_OP_VARIANTS+#undef DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS#undef HVX_BINARY_DISPATCHER#undef UNUSEDdiff --git a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.hindex 552017309d..b7c2d74091 100644--- a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h+++ b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h@@ -178,4 +178,58 @@ static inline HVX_Vector hvx_vec_tanh_f16(HVX_Vector x) {return hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(sig2x, v_two), v_neg_one);}+#define hvx_sigmoid_f16_loop_body(dst_type, src_type, vec_store) \+ do { \+ dst_type * restrict vdst = (dst_type *) dst; \+ src_type * restrict vsrc = (src_type *) src; \+ \+ const uint32_t epv = 128 / sizeof(_Float16); \+ const uint32_t nvec = n / epv; \+ const uint32_t nloe = n % epv; \+ \+ uint32_t i = 0; \+ \+ _Pragma("unroll(4)") \+ for (; i < nvec; i++) { \+ vdst[i] = hvx_vec_fast_sigmoid_f16(vsrc[i]); \+ } \+ if (nloe) { \+ HVX_Vector tmp = hvx_vec_fast_sigmoid_f16(vsrc[i]); \+ vec_store((void *) &vdst[i], nloe * sizeof(_Float16), tmp); \+ } \+ } while(0)++#define hvx_tanh_f16_loop_body(dst_type, src_type, vec_store) \+ do { \+ dst_type * restrict vdst = (dst_type *) dst; \+ src_type * restrict vsrc = (src_type *) src; \+ \+ const uint32_t epv = 128 / sizeof(_Float16); \+ const uint32_t nvec = n / epv; \+ const uint32_t nloe = n % epv; \+ \+ uint32_t i = 0; \+ \+ _Pragma("unroll(4)") \+ for (; i < nvec; i++) { \+ vdst[i] = hvx_vec_tanh_f16(vsrc[i]); \+ } \+ if (nloe) { \+ HVX_Vector tmp = hvx_vec_tanh_f16(vsrc[i]); \+ vec_store((void *) &vdst[i], nloe * sizeof(_Float16), tmp); \+ } \+ } while(0)++static inline void hvx_sigmoid_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {+ assert((unsigned long) dst % 128 == 0);+ assert((unsigned long) src % 128 == 0);+ hvx_sigmoid_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);+}++static inline void hvx_tanh_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {+ assert((unsigned long) dst % 128 == 0);+ assert((unsigned long) src % 128 == 0);+ hvx_tanh_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);+}+#endif /* HVX_SIGMOID_H */