LLM inference in C/C++
C++ ggml
hexagon: tiling, tracing and optimizations for unary ops (#25474)
Aparna M Pfb30ba9a6c5b
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex 0c78879cbb..76c71d7ee7 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -44,6 +44,7 @@#include "htp-ops.h"#include "htp/matmul-ops.h"#include "htp/flash-attn-ops.h"+#include "htp/unary-ops.h"#include "htp_iface.h"#include "htp-drv.h"@@ -170,8 +171,8 @@ static inline bool ggml_hexagon_is_hmx_weight_type(enum ggml_type type) {return type == GGML_TYPE_F16 || type == GGML_TYPE_F32 || ggml_hexagon_is_repack_type(type);}-struct htp_mm_kernel_params;struct ggml_hexagon_session;+static void ggml_hexagon_precompute_matmul_params(const struct ggml_hexagon_session * sess,const struct ggml_tensor * src0,@@ -180,6 +181,15 @@ static void ggml_hexagon_precompute_matmul_params(struct htp_mm_kernel_params * kparams);+static void ggml_hexagon_precompute_unary_params(+ const struct ggml_hexagon_session * sess,+ uint32_t op,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * dst,+ struct htp_unary_kernel_params * kparams+);+static void ggml_hexagon_precompute_fused_qkv_params(const struct ggml_hexagon_session * sess,const struct ggml_tensor * src0,@@ -2591,6 +2601,74 @@ finalize:kparams->div_ne11 = init_fastdiv_values(ne11);}+static void ggml_hexagon_precompute_unary_params(+ const struct ggml_hexagon_session * sess,+ uint32_t op,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * dst,+ struct htp_unary_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];+ const uint32_t n_threads = (std::min)((uint32_t)sess->n_threads, src0_nrows);++ kparams->n_threads = n_threads;++ const size_t src0_data_row_size = src0->ne[0] * sizeof(float);+ const size_t dst_data_row_size = dst->ne[0] * sizeof(float);++ const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);+ const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128);++ kparams->src0_row_size_aligned = src0_row_size_aligned;+ kparams->dst_row_size_aligned = dst_row_size_aligned;++ size_t src1_data_row_size = 0;+ size_t src1_row_size_aligned = 0;+ bool broadcast_weight = false;++ if (op == HTP_OP_RMS_NORM_MUL) {+ GGML_ASSERT(src1 != nullptr);+ src1_data_row_size = src1->ne[0] * sizeof(float);+ src1_row_size_aligned = hex_round_up(src1_data_row_size, 128);+ broadcast_weight = (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1);+ }++ kparams->src1_row_size_aligned = src1_row_size_aligned;+ kparams->broadcast_weight = broadcast_weight;++ struct htp_unary_vtcm_layout L;+ uint32_t col_tile = 0;+ uint32_t vtcm_row_per_thread = 0;++ htp_unary_vtcm_layout_build(&L, op, src0->ne[0], dst->ne[0],+ op == HTP_OP_RMS_NORM_MUL ? src1->ne[0] : 0,+ broadcast_weight, n_threads, sess->vtcm_size,+ &col_tile, &vtcm_row_per_thread);++ kparams->col_tile = col_tile;+ kparams->vtcm_row_per_thread = vtcm_row_per_thread;+ kparams->vtcm_size = L.total_bytes;++ kparams->vtcm_src0_size_per_thread = L.src0_bytes;+ kparams->vtcm_src1_size_per_thread = L.src1_bytes;+ kparams->vtcm_dst_size_per_thread = L.dst_bytes;++ kparams->vtcm_src0_size = L.src0_bytes * n_threads;+ kparams->vtcm_src1_size = L.src1_bytes * n_threads;+ kparams->vtcm_dst_size = L.dst_bytes * n_threads;++ kparams->block = col_tile ? 0 : ((L.src0_bytes / 2) / src0_row_size_aligned);++ const uint32_t tiles_per_row = col_tile > 0 ? (src0->ne[0] + col_tile - 1) / col_tile : 1;+ kparams->div_ne01 = init_fastdiv_values(src0->ne[1]);+ kparams->div_ne02 = init_fastdiv_values(src0->ne[2]);+ kparams->div_ne012 = init_fastdiv_values(src0->ne[1] * src0->ne[2]);+ kparams->div_tpr = init_fastdiv_values(tiles_per_row);+}+static void ggml_hexagon_precompute_fused_qkv_params(const struct ggml_hexagon_session * sess,const struct ggml_tensor * src0, // Wk@@ -2866,6 +2944,9 @@ static bool ggml_hexagon_supported_binary(const struct ggml_hexagon_session * sereturn false;}+ if (ggml_is_permuted(src0) || ggml_is_permuted(dst)) {+ return false;+ }if (!ggml_are_same_shape(src0, dst)) {return false;}@@ -2912,6 +2993,9 @@ static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * sesif (dst->type != GGML_TYPE_F32) {return false;}+ if (ggml_is_permuted(src0)) {+ return false;+ }if (!ggml_are_same_shape(src0, dst)) {return false;}@@ -3451,6 +3535,15 @@ static bool try_fuse_node(const ggml_hexagon_session * sess, const ggml_cgraph *if (next_node->op == GGML_OP_MUL && op_is_compute(next_node) && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {htp_opnode node(n, {}, HTP_OP_RMS_NORM_MUL);node.add_fused(next_node);++ auto inputs = node.get_inputs();+ const struct ggml_tensor * src0 = inputs[0];+ const struct ggml_tensor * src1 = inputs.size() > 1 ? inputs[1] : nullptr;+ ggml_hexagon_precompute_unary_params(sess,+ node.opcode, src0, src1, node.dst(),+ (struct htp_unary_kernel_params *)node.kernel_params+ );+nodes.push_back(std::move(node));i++; // skip the fused MUL nodereturn true;@@ -3555,6 +3648,14 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggnode.node,(struct htp_fa_kernel_params *)node.kernel_params);+ } else if (htp_op_is_unary(node.opcode)) {+ auto inputs = node.get_inputs();+ const struct ggml_tensor * src0 = inputs[0];+ const struct ggml_tensor * src1 = inputs.size() > 1 ? inputs[1] : nullptr;+ ggml_hexagon_precompute_unary_params(sess,+ node.opcode, src0, src1, node.dst(),+ (struct htp_unary_kernel_params *)node.kernel_params+ );}computed_nodes.push_back(std::move(node));}diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.hindex 19a2504c78..b0c859dacf 100644--- a/ggml/src/ggml-hexagon/htp-opnode.h+++ b/ggml/src/ggml-hexagon/htp-opnode.h@@ -12,6 +12,7 @@#include "htp-ops.h"#include "htp/matmul-ops.h"#include "htp/flash-attn-ops.h"+#include "htp/unary-ops.h"struct htp_opnode {ggml_tensor * node = nullptr;@@ -362,6 +363,9 @@ struct htp_opformat {path = "hvx";}snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);+ } else if (htp_op_is_unary(node.opcode)) {+ const auto * kparams = (const struct htp_unary_kernel_params *) node.kernel_params;+ snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size);} else {snprintf(str, max_size, "----");}diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txtindex be575796d6..2389be9883 100644--- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt+++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt@@ -39,8 +39,8 @@ add_library(${HTP_LIB} SHAREDdiag-ops.csolve-tri-ops.cpad-ops.c- flash-attn-ops.cmatmul-ops.c+ flash-attn-ops.c)target_compile_definitions(${HTP_LIB} PRIVATEdiff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.hindex 6ad77d3daa..e13103fb18 100644--- a/ggml/src/ggml-hexagon/htp/htp-ctx.h+++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h@@ -120,7 +120,6 @@ int op_concat(struct htp_ops_context * octx);int op_diag(struct htp_ops_context * octx);int op_solve_tri(struct htp_ops_context * octx);int op_gated_delta_net(struct htp_ops_context * octx);-int op_tri(struct htp_ops_context * octx);int op_pad(struct htp_ops_context * octx);#endif /* HTP_CTX_H */diff --git a/ggml/src/ggml-hexagon/htp/hvx-norm.h b/ggml/src/ggml-hexagon/htp/hvx-norm.hnew file mode 100644index 0000000000..a8645e412d--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/hvx-norm.h@@ -0,0 +1,257 @@+#ifndef HVX_NORM_H+#define HVX_NORM_H++#include <stdint.h>+#include "hvx-base.h"+#include "hvx-reduce.h"+#include "hvx-inverse.h"+#include "hvx-sqrt.h"+#include "hvx-repl.h"++static inline void hvx_fast_rms_norm_f32(const uint8_t * restrict src,+ uint8_t * restrict dst,+ const int num_elems,+ float epsilon) {++ const HVX_Vector * restrict v_src = (HVX_Vector *) src;+ HVX_Vector * restrict v_dst = (HVX_Vector *) dst;++ const int nvec = num_elems / VLEN_FP32; // number of full vectors+ const int nloe = num_elems % VLEN_FP32; // leftover elements++ // Compute sum of squares for full vectors+ HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000);+ HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);++ #pragma unroll(4)+ for (int i = 0; i < nvec; i++) {+ HVX_Vector v1 = v_src[i];+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);+ sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2);+ }++ // Handle tail elements using vectorized ops with masking+ if (nloe > 0) {+ HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);+ HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);+ sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2);+ }++ // Reduce HVX sum+ sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));++ HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);+ HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);+ HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v);+ HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v);++ // Scale full vectors+ HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v));++ #pragma unroll(4)+ for (int i = 0; i < nvec; i++) {+ HVX_Vector v1 = v_src[i];+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v);+ v_dst[i] = Q6_Vsf_equals_Vqf32(v2);+ }++ // Handle tail elements using vectorized ops with masking+ if (nloe > 0) {+ HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);+ HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v);+ HVX_Vector result = Q6_Vsf_equals_Vqf32(v2);++ // Store with masking to avoid overwriting memory beyond the tensor+ hvx_vec_store_a(&v_dst[nvec], nloe * 4, result);+ }+}++static inline void hvx_fast_rms_norm_mul_f32(const uint8_t * restrict src,+ const uint8_t * restrict weight,+ uint8_t * restrict dst,+ const int num_elems,+ float epsilon) {+ const HVX_Vector * restrict v_src = (const HVX_Vector *) src;+ const HVX_Vector * restrict v_weight = (const HVX_Vector *) weight;+ HVX_Vector * restrict v_dst = (HVX_Vector *) dst;++ const int nvec = num_elems / VLEN_FP32; // number of full vectors+ const int nloe = num_elems % VLEN_FP32; // leftover elements++ // Compute sum of squares for full vectors+ HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000);+ HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);++ #pragma unroll(4)+ for (int i = 0; i < nvec; i++) {+ HVX_Vector v1 = v_src[i];+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);+ sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2);+ }++ // Handle tail elements using vectorized ops with masking+ if (nloe > 0) {+ HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);+ HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);+ sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2);+ }++ // Reduce HVX sum+ sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));++ HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);+ HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);+ HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v);+ HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v);++ // Scale and multiply+ HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v));++ #pragma unroll(4)+ for (int i = 0; i < nvec; i++) {+ HVX_Vector v1 = v_src[i];+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v);+ HVX_Vector v3 = Q6_Vsf_equals_Vqf32(v2);+ HVX_Vector result = Q6_Vqf32_vmpy_VsfVsf(v3, v_weight[i]);+ v_dst[i] = Q6_Vsf_equals_Vqf32(result);+ }++ // Handle tail elements using vectorized ops with masking+ if (nloe > 0) {+ HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);+ HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v);+ HVX_Vector v3 = Q6_Vsf_equals_Vqf32(v2);+ HVX_Vector result = Q6_Vqf32_vmpy_VsfVsf(v3, v_weight[nvec]);+ HVX_Vector res_v = Q6_Vsf_equals_Vqf32(result);++ // Store with masking to avoid overwriting memory beyond the tensor+ hvx_vec_store_a(&v_dst[nvec], nloe * 4, res_v);+ }+}++static inline void hvx_fast_norm_f32(const uint8_t * restrict src,+ uint8_t * restrict dst,+ const int num_elems,+ float epsilon) {++ const HVX_Vector * restrict v_src = (HVX_Vector *) src;+ HVX_Vector * restrict v_dst = (HVX_Vector *) dst;++ const int nvec = num_elems / VLEN_FP32; // number of full vectors+ const int nloe = num_elems % VLEN_FP32; // leftover elements++ // Compute sum of squares and sum of values for full vectors+ HVX_Vector sum_sq_v = Q6_V_vsplat_R(0x00000000);+ HVX_Vector sum_x_v = Q6_V_vsplat_R(0x00000000);+ HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);++ #pragma unroll(4)+ for (int i = 0; i < nvec; i++) {+ HVX_Vector v1 = v_src[i];+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);+ sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, v2);+ sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(v1, Q6_V_vzero()));+ }++ // Handle tail elements using vectorized ops with masking+ if (nloe > 0) {+ HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);+ HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);+ HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);+ sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, v2);+ sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(v1, Q6_V_vzero()));+ }++ // Reduce HVX sums+ sum_sq_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_sq_v));+ sum_x_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_x_v));++ HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);+ HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);+ HVX_Vector mean_sq_v = Q6_Vqf32_vmpy_VsfVsf(sum_sq_v, denom_v);+ HVX_Vector mean_x_v = Q6_Vqf32_vmpy_VsfVsf(sum_x_v, denom_v);+ HVX_Vector mean_x_sq_v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(mean_x_v), Q6_Vsf_equals_Vqf32(mean_x_v));+ HVX_Vector var_v = Q6_Vqf32_vsub_Vqf32Vqf32(mean_sq_v, mean_x_sq_v);+ HVX_Vector var_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(var_v, epsilon_v);++ // scale = rsqrt(variance + epsilon), mean_x broadcast for subtraction+ HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(var_epsilon_v));+ HVX_Vector mean_x_b = hvx_vec_repl_f32(Q6_Vsf_equals_Vqf32(mean_x_v));++ #pragma unroll(4)+ for (int i = 0; i < nvec; i++) {+ HVX_Vector v1 = v_src[i];+ HVX_Vector v2 = Q6_Vqf32_vsub_VsfVsf(v1, mean_x_b);+ HVX_Vector v3 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v2), scale_v);+ v_dst[i] = Q6_Vsf_equals_Vqf32(v3);+ }++ // Handle tail elements using vectorized ops with masking+ if (nloe > 0) {+ HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);+ HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);+ HVX_Vector v2 = Q6_Vqf32_vsub_VsfVsf(v1, mean_x_b);+ HVX_Vector v3 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v2), scale_v);+ HVX_Vector result = Q6_Vsf_equals_Vqf32(v3);++ // Store with masking to avoid overwriting memory beyond the tensor+ hvx_vec_store_a(&v_dst[nvec], nloe * 4, result);+ }+}++static inline void hvx_fast_l2_norm_f32(const uint8_t * restrict src,+ uint8_t * restrict dst,+ const int num_elems,+ float epsilon) {++ const HVX_Vector * restrict v_src = (HVX_Vector *) src;+ HVX_Vector * restrict v_dst = (HVX_Vector *) dst;++ HVX_Vector sum_v = hvx_vec_splat_f32(0.0f);++ const int nvec = num_elems / VLEN_FP32;+ const int nloe = num_elems % VLEN_FP32;++ #pragma unroll(4)+ for (int i = 0; i < nvec; i++) {+ HVX_Vector v1 = v_src[i];+ HVX_Vector sq = Q6_Vqf32_vmpy_VsfVsf(v1, v1);+ sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, sq);+ }++ // Include tail elements in the sum-of-squares using a predicate mask+ if (nloe > 0) {+ HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);+ HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);+ HVX_Vector sq = Q6_Vqf32_vmpy_VsfVsf(v1, v1);+ sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, sq);+ }++ // Compute scale = 1/fmax(sqrt(sum), epsilon) entirely in HVX registers.+ // hvx_vec_rsqrt_f32 + hvx_vec_inverse_f32 avoids scalar extraction.+ HVX_Vector sum_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));+ HVX_Vector rsqrt_v = hvx_vec_rsqrt_f32(sum_sf); // 1/sqrt(sum)+ HVX_Vector sqrt_v = hvx_vec_inverse_f32(rsqrt_v); // sqrt(sum)+ HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);+ HVX_Vector denom_v = Q6_Vsf_vmax_VsfVsf(sqrt_v, epsilon_v); // fmax(sqrt(sum), epsilon)+ HVX_Vector scale_v = hvx_vec_inverse_f32(denom_v); // 1/fmax(sqrt(sum), epsilon)++ #pragma unroll(4)+ for (int i = 0; i < nvec; i++) {+ HVX_Vector v1 = v_src[i];+ v_dst[i] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v1, scale_v));+ }++ if (nloe > 0) {+ HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);+ HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);+ HVX_Vector result = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v1, scale_v));+ hvx_vec_store_a(&v_dst[nvec], nloe * 4, result);+ }+}++#endif // HVX_NORM_Hdiff --git a/ggml/src/ggml-hexagon/htp/hvx-utils.h b/ggml/src/ggml-hexagon/htp/hvx-utils.hindex 23373f73ae..706a64f3ab 100644--- a/ggml/src/ggml-hexagon/htp/hvx-utils.h+++ b/ggml/src/ggml-hexagon/htp/hvx-utils.h@@ -19,5 +19,6 @@#include "hvx-base.h"#include "hvx-pow.h"#include "hvx-log.h"+#include "hvx-norm.h"#endif /* HVX_UTILS_H */diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.cindex bfb3e3138e..d971b60f3a 100644--- a/ggml/src/ggml-hexagon/htp/main.c+++ b/ggml/src/ggml-hexagon/htp/main.c@@ -667,7 +667,7 @@ static int execute_op(struct htp_ops_context * octx) {return op_gated_delta_net(octx);case HTP_OP_TRI:- return op_tri(octx);+ return op_unary(octx);case HTP_OP_INVALID:break;diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.cindex 71fab2cdbc..a71107f104 100644--- a/ggml/src/ggml-hexagon/htp/unary-ops.c+++ b/ggml/src/ggml-hexagon/htp/unary-ops.c@@ -9,19 +9,23 @@#include <string.h>#include "hex-dma.h"+#include "hex-fastdiv.h"#include "hvx-exp.h"#include "hvx-sigmoid.h"#include "hvx-utils.h"+#include "unary-ops.h"#define GGML_COMMON_DECL_C#include "ggml-common.h"#include "htp-ctx.h"#include "htp-ops.h"+#include "htp-vtcm.h"+#include "hex-profile.h"struct htp_unary_context {struct htp_ops_context * octx;+ const struct htp_unary_kernel_params * kparams;- // Precomputed valuesconst uint8_t * data_src0;const uint8_t * data_src1; // weight/scale tensor for RMS_NORM_MULuint8_t * data_dst;@@ -34,27 +38,41 @@ struct htp_unary_context {size_t src1_row_size_aligned;size_t dst_row_size_aligned;- size_t src0_spad_half_size;- size_t src1_spad_half_size;- size_t dst_spad_half_size;+ size_t src0_vtcm_half_size;+ size_t src1_vtcm_half_size;+ size_t dst_vtcm_half_size;uint32_t block;uint32_t src0_nrows;uint32_t src0_nrows_per_thread;uint32_t nc;+ uint32_t col_tile; // tiled modebool broadcast_weight;++ uint8_t * vtcm_src0;+ uint8_t * vtcm_src1;+ uint8_t * vtcm_dst;++ size_t vtcm_src0_size_per_thread;+ size_t vtcm_src1_size_per_thread;+ size_t vtcm_dst_size_per_thread;};// Convert flat row index to DDR byte offset using the tensor's actual strides.// ir = i1 + ne1*(i2 + ne2*i3) => offset = i1*nb1 + i2*nb2 + i3*nb3static inline size_t unary_row_offset(uint32_t ir,uint32_t ne1, uint32_t ne2,+ const struct fastdiv_values * div_ne1,+ const struct fastdiv_values * div_ne2,+ const struct fastdiv_values * div_ne12,size_t nb1, size_t nb2, size_t nb3) {- const uint32_t i1 = ir % ne1;- const uint32_t i2 = (ir / ne1) % ne2;- const uint32_t i3 = ir / (ne1 * ne2);+ const uint32_t i1 = fastmodulo(ir, ne1, div_ne1);+ const uint32_t ir_div_ne1 = fastdiv(ir, div_ne1);+ const uint32_t i2 = fastmodulo(ir_div_ne1, ne2, div_ne2);+ const uint32_t i3 = fastdiv(ir, div_ne12);return i1 * nb1 + i2 * nb2 + i3 * nb3;}+// Safe DMA block size from row `ir`: clamp to the tighter dim-1 slice// boundary of src and dst so the nb1 stride stays valid for all rows.static inline uint32_t unary_block_size(uint32_t ir,@@ -62,18 +80,13 @@ static inline uint32_t unary_block_size(uint32_t ir,uint32_t block,bool src_contig,bool dst_contig,- uint32_t src_ne1,- uint32_t dst_ne1) {+ uint32_t ne1,+ const struct fastdiv_values * div_ne1) {uint32_t limit = MIN(block, end_row - ir);- if (!src_contig) {- const uint32_t src_slice_end = (ir / src_ne1 + 1) * src_ne1;- limit = MIN(limit, src_slice_end - ir);- }-- if (!dst_contig) {- const uint32_t dst_slice_end = (ir / dst_ne1 + 1) * dst_ne1;- limit = MIN(limit, dst_slice_end - ir);+ if (!src_contig || !dst_contig) {+ const uint32_t slice_end = (fastdiv(ir, div_ne1) + 1) * ne1;+ limit = MIN(limit, slice_end - ir);}return limit;@@ -100,242 +113,43 @@ static inline uint32_t unary_block_size(uint32_t ir,const uint32_t nb2 = dst->nb[2]; \const uint32_t nb3 = dst->nb[3];-static void hvx_fast_rms_norm_f32(const uint8_t * restrict src,- uint8_t * restrict dst,- uint8_t * restrict pad,- const int num_elems,- float epsilon) {- (void)pad;-- const HVX_Vector * restrict v_src = (HVX_Vector *) src;- HVX_Vector * restrict v_dst = (HVX_Vector *) dst;-- const int nvec = num_elems / VLEN_FP32; // number of full vectors- const int nloe = num_elems % VLEN_FP32; // leftover elements-- // Compute sum of squares for full vectors- HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000);- HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);-- #pragma unroll(4)- for (int i = 0; i < nvec; i++) {- HVX_Vector v1 = v_src[i];- HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);- sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2);- }-- // Handle tail elements using vectorized ops with masking- if (nloe > 0) {- HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);- HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);- HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);- sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2);- }-- // Reduce HVX sum- sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));-- HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);- HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);- HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v);- HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v);-- // Scale full vectors- HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v));-- #pragma unroll(4)- for (int i = 0; i < nvec; i++) {- HVX_Vector v1 = v_src[i];- HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v);- v_dst[i] = Q6_Vsf_equals_Vqf32(v2);- }-- // Handle tail elements using vectorized ops with masking- if (nloe > 0) {-- HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);- HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);- HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v);- HVX_Vector result = Q6_Vsf_equals_Vqf32(v2);-- // Store with masking to avoid overwriting memory beyond the tensor- hvx_vec_store_a(&v_dst[nvec], nloe * 4, result);- }-}--static void hvx_fast_rms_norm_mul_f32(const uint8_t * restrict src,- const uint8_t * restrict weight,- uint8_t * restrict dst,- const int num_elems,- float epsilon) {- const HVX_Vector * restrict v_src = (const HVX_Vector *) src;- const HVX_Vector * restrict v_weight = (const HVX_Vector *) weight;- HVX_Vector * restrict v_dst = (HVX_Vector *) dst;-- const int nvec = num_elems / VLEN_FP32; // number of full vectors- const int nloe = num_elems % VLEN_FP32; // leftover elements-- // Compute sum of squares for full vectors- HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000);- HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);-- #pragma unroll(4)- for (int i = 0; i < nvec; i++) {- HVX_Vector v1 = v_src[i];- HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);- sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2);- }-- // Handle tail elements using vectorized ops with masking- if (nloe > 0) {- HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);- HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);- HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);- sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2);- }-- // Reduce HVX sum- sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));-- HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);- HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);- HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v);- HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v);-- // Scale and multiply- HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v));-- #pragma unroll(4)- for (int i = 0; i < nvec; i++) {- HVX_Vector v1 = v_src[i];- HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v);- HVX_Vector v3 = Q6_Vsf_equals_Vqf32(v2);- HVX_Vector result = Q6_Vqf32_vmpy_VsfVsf(v3, v_weight[i]);- v_dst[i] = Q6_Vsf_equals_Vqf32(result);- }-- // Handle tail elements using vectorized ops with masking- if (nloe > 0) {- HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);- HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);- HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v);- HVX_Vector v3 = Q6_Vsf_equals_Vqf32(v2);- HVX_Vector result = Q6_Vqf32_vmpy_VsfVsf(v3, v_weight[nvec]);- HVX_Vector res_v = Q6_Vsf_equals_Vqf32(result);-- // Store with masking to avoid overwriting memory beyond the tensor- hvx_vec_store_a(&v_dst[nvec], nloe * 4, res_v);- }-}--static void hvx_fast_norm_f32(const uint8_t * restrict src,- uint8_t * restrict dst,- uint8_t * restrict pad,- const int num_elems,- float epsilon) {- (void)pad;-- const HVX_Vector * restrict v_src = (HVX_Vector *) src;- HVX_Vector * restrict v_dst = (HVX_Vector *) dst;-- const int nvec = num_elems / VLEN_FP32; // number of full vectors- const int nloe = num_elems % VLEN_FP32; // leftover elements-- // Compute sum of squares and sum of values for full vectors- HVX_Vector sum_sq_v = Q6_V_vsplat_R(0x00000000);- HVX_Vector sum_x_v = Q6_V_vsplat_R(0x00000000);- HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);-- #pragma unroll(4)- for (int i = 0; i < nvec; i++) {- HVX_Vector v1 = v_src[i];- HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);- sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, v2);- sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(v1, Q6_V_vzero()));- }-- // Handle tail elements using vectorized ops with masking- if (nloe > 0) {- HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);- HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);- HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);- sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, v2);- sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(v1, Q6_V_vzero()));- }-- // Reduce HVX sums- sum_sq_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_sq_v));- sum_x_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_x_v));-- HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);- HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);- HVX_Vector mean_sq_v = Q6_Vqf32_vmpy_VsfVsf(sum_sq_v, denom_v);- HVX_Vector mean_x_v = Q6_Vqf32_vmpy_VsfVsf(sum_x_v, denom_v);- HVX_Vector mean_x_sq_v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(mean_x_v), Q6_Vsf_equals_Vqf32(mean_x_v));- HVX_Vector var_v = Q6_Vqf32_vsub_Vqf32Vqf32(mean_sq_v, mean_x_sq_v);- HVX_Vector var_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(var_v, epsilon_v);-- // scale = rsqrt(variance + epsilon), mean_x broadcast for subtraction- HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(var_epsilon_v));- HVX_Vector mean_x_b = hvx_vec_repl_f32(Q6_Vsf_equals_Vqf32(mean_x_v));-- #pragma unroll(4)- for (int i = 0; i < nvec; i++) {- HVX_Vector v1 = v_src[i];- HVX_Vector v2 = Q6_Vqf32_vsub_VsfVsf(v1, mean_x_b);- HVX_Vector v3 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v2), scale_v);- v_dst[i] = Q6_Vsf_equals_Vqf32(v3);- }-- // Handle tail elements using vectorized ops with masking- if (nloe > 0) {-- HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);- HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);- HVX_Vector v2 = Q6_Vqf32_vsub_VsfVsf(v1, mean_x_b);- HVX_Vector v3 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v2), scale_v);- HVX_Vector result = Q6_Vsf_equals_Vqf32(v3);-- // Store with masking to avoid overwriting memory beyond the tensor- hvx_vec_store_a(&v_dst[nvec], nloe * 4, result);- }-}+#define htp_unary_op_preamble \+ int32_t * op_params = uctx->octx->op_params; \+ const uint32_t ne0 = uctx->nc; \+ const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; \+ const size_t dst_row_size_aligned = uctx->dst_row_size_aligned;static void scale_f32(const float * restrict src,float * restrict dst,- uint8_t * restrict spad,const uint32_t num_rows,- const uint32_t row_elems,- const size_t row_size,- int32_t * op_params) {+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;float scale = 0.f;float bias = 0.f;memcpy(&scale, &op_params[0], sizeof(float));memcpy(&bias, &op_params[1], sizeof(float));for (uint32_t ir = 0; ir < num_rows; ir++) {- const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size);- uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size);+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);- hvx_scale_offset_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, row_elems, scale, bias);+ hvx_scale_offset_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0, scale, bias);}}static void rms_norm_f32(const float * restrict src,float * restrict dst,- uint8_t * restrict spad,const uint32_t num_rows,- const uint32_t row_elems,- const size_t row_size,- int32_t * op_params) {+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;float epsilon = 0.f;memcpy(&epsilon, op_params, sizeof(float));for (uint32_t ir = 0; ir < num_rows; ir++) {- const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size);- uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size);+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);- hvx_fast_rms_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, spad, row_elems, epsilon);+ hvx_fast_rms_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon);}}@@ -343,135 +157,116 @@ static void rms_norm_mul_f32(const float * restrict src,const float * restrict weight,float * restrict dst,const uint32_t num_rows,- const uint32_t row_elems,- const size_t row_size,- const size_t weight_row_size,- int32_t * op_params,- bool broadcast_weight) {+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;float epsilon = 0.f;memcpy(&epsilon, op_params, sizeof(float));for (uint32_t ir = 0; ir < num_rows; ir++) {- const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size);- const uint8_t * restrict w_local = (const uint8_t *)weight + (broadcast_weight ? 0 : ir * weight_row_size);- uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size);+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);+ const uint8_t * restrict w_local = (const uint8_t *)weight + (uctx->broadcast_weight ? 0 : ir * uctx->src1_row_size_aligned);+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);- hvx_fast_rms_norm_mul_f32(src_local, w_local, dst_local, row_elems, epsilon);+ hvx_fast_rms_norm_mul_f32(src_local, w_local, dst_local, ne0, epsilon);}}static void norm_f32(const float * restrict src,- float * restrict dst,- uint8_t * restrict spad,- const uint32_t num_rows,- const uint32_t row_elems,- const size_t row_size,- int32_t * op_params) {+ float * restrict dst,+ const uint32_t num_rows,+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;float epsilon = 0.f;memcpy(&epsilon, op_params, sizeof(float));for (uint32_t ir = 0; ir < num_rows; ir++) {- const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size);- uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size);+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);- hvx_fast_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, spad, row_elems, epsilon);+ hvx_fast_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon);}}static void sqr_f32(const float * restrict src,float * restrict dst,- uint8_t * restrict spad,const uint32_t num_rows,- const uint32_t row_elems,- const size_t row_size,- int32_t * op_params) {+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;for (uint32_t ir = 0; ir < num_rows; ir++) {- const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size);- uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size);+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);- hvx_sqr_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, row_elems);+ hvx_sqr_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);}}static void sqrt_f32(const float * restrict src,float * restrict dst,- uint8_t * restrict spad,const uint32_t num_rows,- const uint32_t row_elems,- const size_t row_size,- int32_t * op_params) {+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;for (uint32_t ir = 0; ir < num_rows; ir++) {- const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size);- uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size);+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);- hvx_sqrt_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, row_elems);+ hvx_sqrt_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);}}static void neg_f32(const float * restrict src,float * restrict dst,- uint8_t * restrict spad,const uint32_t num_rows,- const uint32_t row_elems,- const size_t row_size,- int32_t * op_params) {+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;for (uint32_t ir = 0; ir < num_rows; ir++) {- const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size);- uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size);+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);- hvx_scale_f32_aa(dst_local, src_local, row_elems, -1.0f);+ hvx_scale_f32_aa(dst_local, src_local, ne0, -1.0f);}}static void exp_f32(const float * restrict src,float * restrict dst,- uint8_t * restrict spad,const uint32_t num_rows,- const uint32_t row_elems,- const size_t row_size,- int32_t * op_params) {+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;for (uint32_t ir = 0; ir < num_rows; ir++) {- const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size);- uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size);+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);- hvx_exp_f32(dst_local, src_local, row_elems, false);+ hvx_exp_f32(dst_local, src_local, ne0, false);}}static void sigmoid_f32(const float * restrict src,float * restrict dst,- uint8_t * restrict spad,const uint32_t num_rows,- const uint32_t row_elems,- const size_t row_size,- int32_t * op_params) {+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;for (uint32_t ir = 0; ir < num_rows; ir++) {- const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size);- uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size);+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);- hvx_sigmoid_f32_aa(dst_local, src_local, row_elems);+ hvx_sigmoid_f32_aa(dst_local, src_local, ne0);}}static void tri_f32(const float * restrict src,float * restrict dst,- uint8_t * restrict spad,const uint32_t num_rows,- const uint32_t row_elems,- const size_t row_size,- int32_t * op_params,const uint32_t ir,const struct htp_unary_context * uctx) {-+ htp_unary_op_preamble;const int32_t ttype = op_params[0];const HVX_Vector zero = hvx_vec_splat_f32(0.0f);- const uint32_t nvec = row_elems / VLEN_FP32;- const uint32_t nloe = row_elems % VLEN_FP32;+ const uint32_t nvec = ne0 / VLEN_FP32;+ const uint32_t nloe = ne0 % VLEN_FP32;const uint32_t ne01 = uctx->octx->src[0]->ne[1];@@ -479,8 +274,8 @@ static void tri_f32(const float * restrict src,const uint32_t abs_row = ir + b;const uint32_t i01 = abs_row % ne01;- const HVX_Vector * restrict v_src = (const HVX_Vector *) ((const uint8_t *) src + b * row_size);- HVX_Vector * restrict v_dst = (HVX_Vector *) ((uint8_t *) dst + b * row_size);+ const HVX_Vector * restrict v_src = (const HVX_Vector *) ((const uint8_t *) src + b * src0_row_size_aligned);+ HVX_Vector * restrict v_dst = (HVX_Vector *) ((uint8_t *) dst + b * dst_row_size_aligned);uint32_t boundary;int keep_left;@@ -491,7 +286,7 @@ static void tri_f32(const float * restrict src,case 3: boundary = i01; keep_left = 1; break; // keep col < rowdefault: boundary = 0; keep_left = 0; break;}- if (boundary > row_elems) boundary = row_elems;+ if (boundary > ne0) boundary = ne0;// Full HVX vectors — each starts at a 128-byte aligned offsetfor (uint32_t i = 0; i < nvec; i++) {@@ -520,25 +315,25 @@ static void tri_f32(const float * restrict src,// Tail elements (row_elems not a multiple of VLEN_FP32)if (nloe > 0) {- const uint32_t vec_start = nvec * VLEN_FP32;- const uint32_t vec_end = vec_start + nloe;+ const uint32_t abs_start = nvec * VLEN_FP32;+ const uint32_t abs_end = abs_start + nloe;HVX_Vector tail_val;if (keep_left) {- if (vec_end <= boundary) {+ if (abs_end <= boundary) {tail_val = v_src[nvec];- } else if (vec_start >= boundary) {+ } else if (abs_start >= boundary) {tail_val = zero;} else {- HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - vec_start) * sizeof(float));+ HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float));tail_val = Q6_V_vmux_QVV(mask, v_src[nvec], zero);}} else {- if (vec_end <= boundary) {+ if (abs_end <= boundary) {tail_val = zero;- } else if (vec_start >= boundary) {+ } else if (abs_start >= boundary) {tail_val = v_src[nvec];} else {- HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - vec_start) * sizeof(float));+ HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float));tail_val = Q6_V_vmux_QVV(mask, zero, v_src[nvec]);}}@@ -549,18 +344,16 @@ static void tri_f32(const float * restrict src,static void softplus_f32(const float * restrict src,float * restrict dst,- uint8_t * restrict spad,const uint32_t num_rows,- const uint32_t row_elems,- const size_t row_size,- int32_t * op_params) {+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;// softplus(x) = log(1 + exp(x))// Match CPU reference: ggml_compute_softplus_f32() in ggml-impl.hfor (uint32_t ir = 0; ir < num_rows; ir++) {- const float * restrict src_f = (const float *)((const uint8_t *)src + (ir * row_size));- float * restrict dst_f = (float *)((uint8_t *)dst + (ir * row_size));+ const float * restrict src_f = (const float *)((const uint8_t *)src + (ir * src0_row_size_aligned));+ float * restrict dst_f = (float *)((uint8_t *)dst + (ir * dst_row_size_aligned));- for (uint32_t i = 0; i < row_elems; i++) {+ for (uint32_t i = 0; i < ne0; i++) {float x = src_f[i];// For x > 20: softplus(x) ≈ x (avoids exp overflow)dst_f[i] = (x > 20.0f) ? x : logf(1.0f + expf(x));@@ -568,285 +361,418 @@ static void softplus_f32(const float * restrict src,}}-// --- L2_NORM HVX kernel ----// Computes y[i] = x[i] / fmax(sqrt(sum(x[j]^2)), epsilon) for each row.-// scale = 1/fmax(sqrt(sum), epsilon) is computed entirely in HVX registers-// using rsqrt + inverse to avoid scalar extraction.-static void hvx_fast_l2_norm_f32(const uint8_t * restrict src,- uint8_t * restrict dst,- uint8_t * restrict pad,- const int num_elems,- float epsilon) {- (void)pad;-- const HVX_Vector * restrict v_src = (HVX_Vector *) src;- HVX_Vector * restrict v_dst = (HVX_Vector *) dst;-- HVX_Vector sum_v = hvx_vec_splat_f32(0.0f);-- const int nvec = num_elems / VLEN_FP32;- const int nloe = num_elems % VLEN_FP32;-- #pragma unroll(4)- for (int i = 0; i < nvec; i++) {- HVX_Vector v1 = v_src[i];- HVX_Vector sq = Q6_Vqf32_vmpy_VsfVsf(v1, v1);- sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, sq);- }-- // Include tail elements in the sum-of-squares using a predicate mask- if (nloe > 0) {- HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);- HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);- HVX_Vector sq = Q6_Vqf32_vmpy_VsfVsf(v1, v1);- sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, sq);- }-- // Compute scale = 1/fmax(sqrt(sum), epsilon) entirely in HVX registers.- // hvx_vec_rsqrt_f32 + hvx_vec_inverse_f32 avoids scalar extraction.- HVX_Vector sum_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));- HVX_Vector rsqrt_v = hvx_vec_rsqrt_f32(sum_sf); // 1/sqrt(sum)- HVX_Vector sqrt_v = hvx_vec_inverse_f32(rsqrt_v); // sqrt(sum)- HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);- HVX_Vector denom_v = Q6_Vsf_vmax_VsfVsf(sqrt_v, epsilon_v); // fmax(sqrt(sum), epsilon)- HVX_Vector scale_v = hvx_vec_inverse_f32(denom_v); // 1/fmax(sqrt(sum), epsilon)-- #pragma unroll(4)- for (int i = 0; i < nvec; i++) {- HVX_Vector v1 = v_src[i];- v_dst[i] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v1, scale_v));- }-- if (nloe > 0) {- HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);- HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);- HVX_Vector result = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v1, scale_v));- hvx_vec_store_a(&v_dst[nvec], nloe * 4, result);- }-}-static void l2_norm_f32(const float * restrict src,float * restrict dst,- uint8_t * restrict spad,const uint32_t num_rows,- const uint32_t row_elems,- const size_t row_size,- int32_t * op_params) {+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;float epsilon = 0.f;memcpy(&epsilon, op_params, sizeof(float));for (uint32_t ir = 0; ir < num_rows; ir++) {- const float * restrict src_f = (const float *)((const uint8_t *)src + (ir * row_size));- float * restrict dst_f = (float *)((uint8_t *)dst + (ir * row_size));+ const float * restrict src_f = (const float *)((const uint8_t *)src + (ir * src0_row_size_aligned));+ float * restrict dst_f = (float *)((uint8_t *)dst + (ir * dst_row_size_aligned));- hvx_fast_l2_norm_f32((const uint8_t *)src_f, (uint8_t *)dst_f, spad, row_elems, epsilon);+ hvx_fast_l2_norm_f32((const uint8_t *)src_f, (uint8_t *)dst_f, ne0, epsilon);}}static void tanh_f32(const float * restrict src,float * restrict dst,- uint8_t * restrict spad,const uint32_t num_rows,- const uint32_t row_elems,- const size_t row_size,- int32_t * op_params) {+ const struct htp_unary_context * uctx) {+ htp_unary_op_preamble;+for (uint32_t ir = 0; ir < num_rows; ir++) {- const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size);- uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size);+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);- hvx_tanh_f32_aa(dst_local, src_local, row_elems);+ hvx_tanh_f32_aa(dst_local, src_local, ne0);}}-static void unary_job_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {- const struct htp_unary_context * uctx = (const struct htp_unary_context *) data;- struct htp_ops_context * octx = uctx->octx;- const struct htp_tensor * src = octx->src[0];- const struct htp_tensor * dst = octx->dst;-- htp_unary_preamble;-- int htp_op = octx->op;- int32_t * op_params = octx->op_params;- uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread;-- const size_t src0_data_row_size = uctx->src0_data_row_size;- const size_t dst_data_row_size = uctx->dst_data_row_size;-- const size_t src0_row_size_aligned = uctx->src0_row_size_aligned;- const size_t dst_row_size_aligned = uctx->dst_row_size_aligned;-- const uint32_t src0_nrows = uctx->src0_nrows;- const uint32_t src0_start_row = src0_nrows_per_thread * ith;- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);-- // no work for this thread- if (src0_start_row >= src0_end_row) {- return;- }+#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \+static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * data) { \+ const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \+ struct htp_ops_context * octx = uctx->octx; \+ const struct htp_tensor * src = octx->src[0]; \+ const struct htp_tensor * dst = octx->dst; \+ struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \+ \+ htp_unary_preamble; \+ \+ int32_t * op_params = octx->op_params; \+ uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; \+ \+ const size_t src0_data_row_size = uctx->src0_data_row_size; \+ const size_t dst_data_row_size = uctx->dst_data_row_size; \+ \+ const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; \+ const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; \+ \+ const uint32_t src0_nrows = uctx->src0_nrows; \+ const uint32_t src0_start_row = src0_nrows_per_thread * ith; \+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \+ \+ if (src0_start_row >= src0_end_row) { \+ return; \+ } \+ \+ const uint8_t * restrict data_src = uctx->data_src0; \+ const uint8_t * restrict data_src1 = uctx->data_src1; \+ uint8_t * restrict data_dst = uctx->data_dst; \+ \+ const struct htp_tensor * src1 = (IS_RMS_NORM_MUL) ? octx->src[1] : NULL; \+ const uint32_t nb11 = src1 ? src1->nb[1] : 0; \+ const uint32_t nb12 = src1 ? src1->nb[2] : 0; \+ const uint32_t nb13 = src1 ? src1->nb[3] : 0; \+ const bool src1_contig = src1 ? ((nb12 == (size_t)ne01 * nb11) && (nb13 == (size_t)ne02 * nb12)) : false; \+ \+ uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); \+ uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_thread)) : NULL;\+ uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); \+ \+ size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; \+ size_t src1_vtcm_half_size = uctx->src1_vtcm_half_size; \+ size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; \+ \+ const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && \+ (nb03 == (size_t)ne02 * nb02); \+ const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && \+ (nb3 == (size_t)ne2 * nb2); \+ \+ const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; \+ const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; \+ const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; \+ \+ const uint32_t src0_max_block = src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01); \+ const uint32_t dst_max_block = dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); \+ const uint32_t BLOCK = MIN(src0_max_block, dst_max_block); \+ if (BLOCK == 0) { \+ FARF(ERROR, "unary-f32 : current VTCM reservation %zu is too small, needed at least %zu\n", \+ uctx->vtcm_src0_size_per_thread, src0_row_size_aligned); \+ return; \+ } \+ \+ dma_queue * dma_queue = octx->ctx->dma[ith]; \+ \+ if ((IS_RMS_NORM_MUL) && uctx->broadcast_weight) { \+ dma_queue_push(dma_queue, dma_make_ptr(src1_vtcm_data, data_src1), \+ uctx->src1_row_size_aligned, 0, uctx->src1_data_row_size, 1); \+ dma_queue_flush(dma_queue); \+ } \+ \+ for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { \+ const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, ne01, \+ div_ne01); \+ \+ dma_queue_push(dma_queue, \+ dma_make_ptr(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), \+ nb1, dst_row_size_aligned, dst_data_row_size, 0); \+ \+ const size_t src0_off = src0_contig ? (ir * nb01) : \+ unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); \+ dma_queue_push(dma_queue, \+ dma_make_ptr(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off), \+ src0_row_size_aligned, nb01, src0_data_row_size, block_size); \+ \+ if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) { \+ const size_t src1_off = src1_contig ? (ir * nb11) : \+ unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11, nb12, nb13); \+ dma_queue_push(dma_queue, \+ dma_make_ptr(src1_vtcm_data + (vtcm_idx * src1_vtcm_half_size), data_src1 + src1_off), \+ uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, block_size); \+ } \+ \+ ir += block_size; \+ } \+ \+ for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { \+ const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, ne01, \+ div_ne01); \+ \+ float * dst_vtcm = (float *) dma_queue_pop(dma_queue).src; \+ float * src0_vtcm = (float *) dma_queue_pop(dma_queue).dst; \+ float * src1_vtcm = NULL; \+ if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) { \+ src1_vtcm = (float *) dma_queue_pop(dma_queue).dst; \+ } \+ \+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \+ CORE_EXPR; \+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \+ \+ const size_t dst_off = dst_contig ? (ir * nb1) : \+ unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3); \+ dma_queue_push(dma_queue, \+ dma_make_ptr(data_dst + dst_off, dst_vtcm), \+ nb1, dst_row_size_aligned, dst_data_row_size, block_size); \+ \+ const uint32_t next_ir = ir + block_size; \+ if (next_ir < src0_end_row) { \+ const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, src0_contig, dst_contig,\+ ne01, div_ne01); \+ const uint32_t pref_ir = next_ir + next_block_size; \+ if (pref_ir < src0_end_row) { \+ const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, src0_contig, \+ dst_contig, ne01, div_ne01); \+ const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) : \+ unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); \+ dma_queue_push(dma_queue, \+ dma_make_ptr(src0_vtcm, data_src + src0_pref_off), \+ src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size); \+ \+ if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) { \+ const size_t src1_pref_off = src1_contig ? (pref_ir * nb11) : \+ unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11, nb12, nb13); \+ dma_queue_push(dma_queue, \+ dma_make_ptr(src1_vtcm, data_src1 + src1_pref_off), \+ uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, pref_block_size); \+ } \+ } \+ } \+ ir += block_size; \+ } \+ \+ dma_queue_flush(dma_queue); \+}- uint64_t t1, t2;- t1 = HAP_perf_get_qtimer_count();-- const uint8_t * restrict data_src = uctx->data_src0;- const uint8_t * restrict data_src1 = uctx->data_src1;- uint8_t * restrict data_dst = uctx->data_dst;-- const struct htp_tensor * src1 = (htp_op == HTP_OP_RMS_NORM_MUL) ? octx->src[1] : NULL;- const uint32_t nb11 = src1 ? src1->nb[1] : 0;- const uint32_t nb12 = src1 ? src1->nb[2] : 0;- const uint32_t nb13 = src1 ? src1->nb[3] : 0;-- uint8_t * src0_spad_data = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);- uint8_t * src1_spad_data = octx->src1_spad.data + (ith * octx->src1_spad.size_per_thread);- uint8_t * dst_spad_data = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread);-- size_t src0_spad_half_size = uctx->src0_spad_half_size;- size_t src1_spad_half_size = uctx->src1_spad_half_size;- size_t dst_spad_half_size = uctx->dst_spad_half_size;-- // Non-contiguous tensors have gaps at dim-2/3 boundaries that a single-stride- // 2D DMA descriptor cannot span. Clamp BLOCK to ne1 (one dim-1 slice) so every- // transfer stays within a nb1-uniform region. Skipped for contiguous tensors.- const bool src0_contig = (nb02 == (size_t)ne01 * nb01) &&- (nb03 == (size_t)ne02 * nb02);- const bool dst_contig = (nb2 == (size_t)ne1 * nb1) &&- (nb3 == (size_t)ne2 * nb2);- const uint32_t src0_max_block = src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01);- const uint32_t dst_max_block = dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1);- const uint32_t BLOCK = MIN(src0_max_block, dst_max_block);- if (BLOCK == 0) {- FARF(ERROR, "unary-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n",- octx->src0_spad.size_per_thread, src0_row_size_aligned);- return;- }+DEFINE_UNARY_TASK(norm, false, false, norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(rms_norm, false, false, rms_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(rms_norm_mul, true, false, rms_norm_mul_f32(src0_vtcm, uctx->broadcast_weight ? (const float *) src1_vtcm_data : src1_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(scale, false, false, scale_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(sqr, false, false, sqr_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(sqrt, false, false, sqrt_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(unary_neg, false, false, neg_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(unary_exp, false, false, exp_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(unary_sigmoid, false, false, sigmoid_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(unary_softplus, false, false, softplus_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(unary_tanh, false, false, tanh_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(l2_norm, false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))+DEFINE_UNARY_TASK(tri, false, true, tri_f32(src0_vtcm, dst_vtcm, block_size, ir, uctx))++// Apply a pointwise unary op to one column tile that is already in VTCM.+#define DEFINE_UNARY_TILED_TASK(NAME, IS_TRI, CORE_TILE_EXPR) \+static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void * data) { \+ const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \+ struct htp_ops_context * octx = uctx->octx; \+ const struct htp_tensor * src = octx->src[0]; \+ const struct htp_tensor * dst = octx->dst; \+ struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \+ \+ htp_unary_preamble; \+ \+ int32_t * op_params = octx->op_params; \+ const uint32_t col_tile = uctx->col_tile; \+ \+ const uint32_t src0_nrows = uctx->src0_nrows; \+ const uint32_t src0_start_row = uctx->src0_nrows_per_thread * ith; \+ const uint32_t src0_end_row = MIN(src0_start_row + uctx->src0_nrows_per_thread, src0_nrows); \+ \+ if (src0_start_row >= src0_end_row) { \+ return; \+ } \+ \+ const uint8_t * restrict data_src = uctx->data_src0; \+ uint8_t * restrict data_dst = uctx->data_dst; \+ \+ uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); \+ uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); \+ \+ const size_t src0_half = uctx->src0_vtcm_half_size; \+ const size_t dst_half = uctx->dst_vtcm_half_size; \+ \+ dma_queue * dmaq = octx->ctx->dma[ith]; \+ \+ const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; \+ const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; \+ const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; \+ const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; \+ \+ const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile; \+ const int32_t tri_ttype = (IS_TRI) ? op_params[0] : 0; \+ \+ const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && \+ (nb03 == (size_t)ne02 * nb02); \+ const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && \+ (nb3 == (size_t)ne2 * nb2); \+ \+ const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row; \+ \+ for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) { \+ const uint32_t row = src0_start_row + t / tiles_per_row; \+ const uint32_t col = (t % tiles_per_row) * col_tile; \+ const uint32_t tw = MIN(col_tile, ne0 - col); \+ const size_t tb = (size_t) tw * sizeof(float); \+ const size_t soff = (src0_contig ? (row * nb01) : \+ unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) +\+ (size_t) col * sizeof(float); \+ \+ dma_queue_push(dmaq, dma_make_ptr(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0); \+ dma_queue_push(dmaq, dma_make_ptr(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1);\+ } \+ \+ uint32_t row = src0_start_row; \+ uint32_t col = 0; \+ uint32_t tile_in_row = 0; \+ uint32_t i01 = fastmodulo(row, ne01, div_ne01); \+ \+ uint32_t prow = src0_start_row + fastdiv(2, div_tpr); \+ uint32_t pcol = fastmodulo(2, tiles_per_row, div_tpr) * col_tile; \+ uint32_t ptile_in_row = fastmodulo(2, tiles_per_row, div_tpr); \+ \+ for (uint32_t t = 0; t < total_tiles; t++) { \+ uint8_t * dst_vtcm = (uint8_t *) dma_queue_pop(dmaq).src; \+ uint8_t * src_vtcm = (uint8_t *) dma_queue_pop(dmaq).dst; \+ \+ const uint32_t tw = MIN(col_tile, ne0 - col); \+ \+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, t); \+ CORE_TILE_EXPR; \+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t); \+ \+ const size_t doff = (dst_contig ? (row * nb1) : \+ unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) + \+ (size_t) col * sizeof(float); \+ const size_t tb = (size_t) tw * sizeof(float); \+ dma_queue_push(dmaq, dma_make_ptr(data_dst + doff, dst_vtcm), tb, tb, tb, 1); \+ \+ const uint32_t pt = t + 2; \+ if (pt < total_tiles) { \+ const uint32_t ptw = MIN(col_tile, ne0 - pcol); \+ const size_t ptb = (size_t) ptw * sizeof(float); \+ const size_t psoff = (src0_contig ? (prow * nb01) : \+ unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, \+ nb03)) + \+ (size_t) pcol * sizeof(float); \+ dma_queue_push(dmaq, dma_make_ptr(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1); \+ } \+ \+ tile_in_row++; \+ col += col_tile; \+ if (tile_in_row == tiles_per_row) { \+ tile_in_row = 0; \+ col = 0; \+ row++; \+ i01++; \+ if (i01 == ne01) { \+ i01 = 0; \+ } \+ } \+ \+ ptile_in_row++; \+ pcol += col_tile; \+ if (ptile_in_row == tiles_per_row) { \+ ptile_in_row = 0; \+ pcol = 0; \+ prow++; \+ } \+ } \+ \+ dma_queue_flush(dmaq); \+}- dma_queue * dma_queue = octx->ctx->dma[ith];+static inline void tile_scale_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw, const int32_t * op_params) {+ float scale = 0.f;+ float bias = 0.f;+ memcpy(&scale, &op_params[0], sizeof(float));+ memcpy(&bias, &op_params[1], sizeof(float));+ hvx_scale_offset_f32_aa(dst_vtcm, src_vtcm, tw, scale, bias);+}- // If weight is broadcasted, load it once per thread at the beginning of execution- if (htp_op == HTP_OP_RMS_NORM_MUL && uctx->broadcast_weight) {- dma_queue_push(dma_queue, dma_make_ptr(src1_spad_data, data_src1), uctx->src1_row_size_aligned, 0, uctx->src1_data_row_size, 1);- dma_queue_flush(dma_queue);+static inline void tile_unary_softplus_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw) {+ const float * restrict sf = (const float *) src_vtcm;+ float * restrict df = (float *) dst_vtcm;+ for (uint32_t i = 0; i < tw; i++) {+ float x = sf[i];+ df[i] = (x > 20.0f) ? x : logf(1.0f + expf(x));}+}- for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; spad_idx++) {- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, ne01, ne1);-- // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)- dma_queue_push(dma_queue,- dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),- nb1, dst_row_size_aligned, dst_data_row_size, 0);-- const size_t src0_off = unary_row_offset(ir, ne01, ne02, nb01, nb02, nb03);- dma_queue_push(dma_queue,- dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src + src0_off),- src0_row_size_aligned, nb01, src0_data_row_size, block_size);+// Triangular mask applied to one column tile. Boundary is an absolute column index, so+// each vector compares against its absolute column position (col_start + i*VLEN_FP32).+static inline void tri_apply_tile_f32(const uint8_t * restrict src, uint8_t * restrict dst,+ uint32_t tile_elems, uint32_t col_start, uint32_t i01,+ uint32_t ne0, int32_t ttype) {+ const HVX_Vector * restrict v_src = (const HVX_Vector *) src;+ HVX_Vector * restrict v_dst = (HVX_Vector *) dst;+ const HVX_Vector zero = hvx_vec_splat_f32(0.0f);- if (htp_op == HTP_OP_RMS_NORM_MUL && !uctx->broadcast_weight) {- const size_t src1_off = unary_row_offset(ir, ne01, ne02, nb11, nb12, nb13);- dma_queue_push(dma_queue,- dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + src1_off),- uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, block_size);+ uint32_t boundary;+ int keep_left;+ switch (ttype) {+ case 0: boundary = i01; keep_left = 0; break;+ case 1: boundary = i01 + 1; keep_left = 0; break;+ case 2: boundary = i01 + 1; keep_left = 1; break;+ case 3: boundary = i01; keep_left = 1; break;+ default: boundary = 0; keep_left = 0; break;+ }+ if (boundary > ne0) boundary = ne0;++ const uint32_t nvec = tile_elems / VLEN_FP32;+ const uint32_t nloe = tile_elems % VLEN_FP32;++ for (uint32_t i = 0; i < nvec; i++) {+ const uint32_t abs_start = col_start + i * VLEN_FP32;+ const uint32_t abs_end = abs_start + VLEN_FP32;+ if (keep_left) {+ if (abs_end <= boundary) {+ v_dst[i] = v_src[i];+ } else if (abs_start >= boundary) {+ v_dst[i] = zero;+ } else {+ HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float));+ v_dst[i] = Q6_V_vmux_QVV(mask, v_src[i], zero);+ }+ } else {+ if (abs_end <= boundary) {+ v_dst[i] = zero;+ } else if (abs_start >= boundary) {+ v_dst[i] = v_src[i];+ } else {+ HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float));+ v_dst[i] = Q6_V_vmux_QVV(mask, zero, v_src[i]);+ }}-- ir += block_size;}- for (uint32_t ir = src0_start_row; ir < src0_end_row; ) {- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig, ne01, ne1);-- float * dst_spad = (float *) dma_queue_pop(dma_queue).src;- float * src0_spad = (float *) dma_queue_pop(dma_queue).dst;- float * src1_spad = NULL;- if (htp_op == HTP_OP_RMS_NORM_MUL && !uctx->broadcast_weight) {- src1_spad = (float *) dma_queue_pop(dma_queue).dst;- }-- // Process block in VTCM- switch (htp_op) {- case HTP_OP_NORM:- norm_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);- break;- case HTP_OP_RMS_NORM:- rms_norm_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);- break;- case HTP_OP_RMS_NORM_MUL:- {- const float * w_ptr = uctx->broadcast_weight ? (const float *) src1_spad_data : src1_spad;- rms_norm_mul_f32(src0_spad, w_ptr, dst_spad, block_size, ne0, src0_row_size_aligned, uctx->src1_row_size_aligned, op_params, uctx->broadcast_weight);- }- break;- case HTP_OP_SCALE:- scale_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);- break;- case HTP_OP_SQR:- sqr_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);- break;- case HTP_OP_SQRT:- sqrt_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);- break;- case HTP_OP_UNARY_NEG:- neg_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);- break;- case HTP_OP_UNARY_EXP:- exp_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);- break;- case HTP_OP_UNARY_SIGMOID:- sigmoid_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);- break;- case HTP_OP_UNARY_SOFTPLUS:- softplus_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);- break;- case HTP_OP_UNARY_TANH:- tanh_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);- break;- case HTP_OP_L2_NORM:- l2_norm_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params);- break;- case HTP_OP_TRI:- tri_f32(src0_spad, dst_spad, NULL, block_size, ne00, src0_row_size_aligned, op_params, ir, uctx);- break;- default:- break;- }-- const size_t dst_off = unary_row_offset(ir, ne1, ne2, nb1, nb2, nb3);- dma_queue_push(dma_queue,- dma_make_ptr(data_dst + dst_off, dst_spad),- nb1, dst_row_size_aligned, dst_data_row_size, block_size);-- // prefetch N+2 loop iteration if any- const uint32_t next_ir = ir + block_size;- if (next_ir < src0_end_row) {- const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, src0_contig, dst_contig, ne01, ne1);- const uint32_t pref_ir = next_ir + next_block_size;- if (pref_ir < src0_end_row) {- const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, src0_contig, dst_contig, ne01, ne1);- const size_t src0_pref_off = unary_row_offset(pref_ir, ne01, ne02, nb01, nb02, nb03);- dma_queue_push(dma_queue,- dma_make_ptr(src0_spad, data_src + src0_pref_off),- src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size);-- if (htp_op == HTP_OP_RMS_NORM_MUL && !uctx->broadcast_weight) {- const size_t src1_pref_off = unary_row_offset(pref_ir, ne01, ne02, nb11, nb12, nb13);- dma_queue_push(dma_queue,- dma_make_ptr(src1_spad, data_src1 + src1_pref_off),- uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, pref_block_size);- }+ if (nloe > 0) {+ const uint32_t abs_start = col_start + nvec * VLEN_FP32;+ const uint32_t abs_end = abs_start + nloe;+ HVX_Vector tail_val;+ if (keep_left) {+ if (abs_end <= boundary) {+ tail_val = v_src[nvec];+ } else if (abs_start >= boundary) {+ tail_val = zero;+ } else {+ HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float));+ tail_val = Q6_V_vmux_QVV(mask, v_src[nvec], zero);+ }+ } else {+ if (abs_end <= boundary) {+ tail_val = zero;+ } else if (abs_start >= boundary) {+ tail_val = v_src[nvec];+ } else {+ HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float));+ tail_val = Q6_V_vmux_QVV(mask, zero, v_src[nvec]);}}- ir += block_size;+ hvx_vec_store_a(&v_dst[nvec], nloe * sizeof(float), tail_val);}-- dma_queue_flush(dma_queue);-- t2 = HAP_perf_get_qtimer_count();-- FARF(HIGH, "unary-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", ith, nth, src->ne[0],- src->ne[1], src->ne[2], src->ne[3], src0_start_row, src0_end_row, dst->ne[0], dst->ne[1], dst->ne[2],- dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));}+DEFINE_UNARY_TILED_TASK(scale, false, tile_scale_f32(dst_vtcm, src_vtcm, tw, op_params))+DEFINE_UNARY_TILED_TASK(sqr, false, hvx_sqr_f32_aa(dst_vtcm, src_vtcm, tw))+DEFINE_UNARY_TILED_TASK(sqrt, false, hvx_sqrt_f32_aa(dst_vtcm, src_vtcm, tw))+DEFINE_UNARY_TILED_TASK(unary_neg, false, hvx_scale_f32_aa(dst_vtcm, src_vtcm, tw, -1.0f))+DEFINE_UNARY_TILED_TASK(unary_exp, false, hvx_exp_f32(dst_vtcm, src_vtcm, tw, false))+DEFINE_UNARY_TILED_TASK(unary_sigmoid, false, hvx_sigmoid_f32_aa(dst_vtcm, src_vtcm, tw))+DEFINE_UNARY_TILED_TASK(unary_softplus, false, tile_unary_softplus_f32(dst_vtcm, src_vtcm, tw))+DEFINE_UNARY_TILED_TASK(unary_tanh, false, hvx_tanh_f32_aa(dst_vtcm, src_vtcm, tw))+DEFINE_UNARY_TILED_TASK(tri, true, tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype))+static int execute_op_unary_f32(struct htp_ops_context * octx) {int err = HTP_STATUS_OK;@@ -856,143 +782,66 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {const char * op_type = NULL;switch (octx->op) {- case HTP_OP_NORM:- op_type = "norm-f32";- break;- case HTP_OP_RMS_NORM:- op_type = "rmsnorm-f32";- break;- case HTP_OP_RMS_NORM_MUL:- op_type = "rmsnorm-mul-f32";- break;- case HTP_OP_SCALE:- op_type = "scale-f32";- break;- case HTP_OP_SQR:- op_type = "sqr-f32";- break;- case HTP_OP_SQRT:- op_type = "sqrt-f32";- break;- case HTP_OP_UNARY_NEG:- op_type = "neg-f32";- break;- case HTP_OP_UNARY_EXP:- op_type = "exp-f32";- break;- case HTP_OP_UNARY_SIGMOID:- op_type = "sigmoid-f32";- break;- case HTP_OP_UNARY_SOFTPLUS:- op_type = "softplus-f32";- break;- case HTP_OP_UNARY_TANH:- op_type = "tanh-f32";- break;- case HTP_OP_L2_NORM:- op_type = "l2norm-f32";- break;- case HTP_OP_TRI:- op_type = "tri-f32";- break;+ case HTP_OP_NORM: op_type = "norm-f32"; break;+ case HTP_OP_RMS_NORM: op_type = "rmsnorm-f32"; break;+ case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break;+ case HTP_OP_SCALE: op_type = "scale-f32"; break;+ case HTP_OP_SQR: op_type = "sqr-f32"; break;+ case HTP_OP_SQRT: op_type = "sqrt-f32"; break;+ case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break;+ case HTP_OP_UNARY_EXP: op_type = "exp-f32"; break;+ case HTP_OP_UNARY_SIGMOID: op_type = "sigmoid-f32"; break;+ case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break;+ case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break;+ case HTP_OP_L2_NORM: op_type = "l2norm-f32"; break;+ case HTP_OP_TRI: op_type = "tri-f32"; break;default:FARF(ERROR, "Unsupported unary Op %u\n", octx->op);return HTP_STATUS_NO_SUPPORT;}+ const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params;+const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];- const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);+ const uint32_t n_threads = kparams->n_threads;const size_t src0_data_row_size = src0->ne[0] * sizeof(float);const size_t dst_data_row_size = dst->ne[0] * sizeof(float);- const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, VLEN);- const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, VLEN);+ const size_t src0_row_size_aligned = kparams->src0_row_size_aligned;+ const size_t dst_row_size_aligned = kparams->dst_row_size_aligned;++ const uint32_t col_tile = kparams->col_tile;size_t src1_data_row_size = 0;- size_t src1_row_size_aligned = 0;- bool broadcast_weight = false;+ size_t src1_row_size_aligned = kparams->src1_row_size_aligned;+ bool broadcast_weight = kparams->broadcast_weight;const struct htp_tensor * src1 = NULL;if (octx->op == HTP_OP_RMS_NORM_MUL) {src1 = octx->src[1];src1_data_row_size = src1->ne[0] * sizeof(float);- src1_row_size_aligned = hex_round_up(src1_data_row_size, VLEN);- broadcast_weight = (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1);}- // VTCM scratchpads for all tensors- // N rows per thread, padded to HVX vector size- // Double buffering requires 2x size per buffer-- size_t spad_size_per_row = 0;- size_t vtcm_row_per_thread = 0;-- if (octx->op == HTP_OP_RMS_NORM_MUL) {- if (broadcast_weight) {- size_t available_vtcm = octx->ctx->vtcm_size;- size_t src1_spad_total = n_threads * src1_row_size_aligned;- if (available_vtcm > src1_spad_total) {- available_vtcm -= src1_spad_total;- } else {- available_vtcm = 0;- }- spad_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned);- vtcm_row_per_thread = available_vtcm / (n_threads * spad_size_per_row);- } else {- spad_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned + src1_row_size_aligned);- vtcm_row_per_thread = (octx->ctx->vtcm_size) / (n_threads * spad_size_per_row);- }- } else {- spad_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned);- vtcm_row_per_thread = (octx->ctx->vtcm_size)/ (n_threads * spad_size_per_row);- }-- // Make sure the reserved vtcm size is sufficient- if (vtcm_row_per_thread == 0) {- FARF(ERROR, "unary-%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size,- spad_size_per_row * n_threads);+ if (octx->ctx->vtcm_size < (size_t)kparams->vtcm_size) {+ FARF(ERROR, "unary-%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, (size_t)kparams->vtcm_size);return HTP_STATUS_VTCM_TOO_SMALL;}- octx->src0_spad.size_per_thread = src0_row_size_aligned * vtcm_row_per_thread * 2;- octx->dst_spad.size_per_thread = dst_row_size_aligned * vtcm_row_per_thread * 2;-- octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread;- octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread;-- if (octx->op == HTP_OP_RMS_NORM_MUL) {- if (broadcast_weight) {- octx->src1_spad.size_per_thread = src1_row_size_aligned;- } else {- octx->src1_spad.size_per_thread = src1_row_size_aligned * vtcm_row_per_thread * 2;- }- octx->src1_spad.size = n_threads * octx->src1_spad.size_per_thread;- } else {- octx->src1_spad.size = 0;- octx->src1_spad.size_per_thread = 0;- }-- octx->src0_spad.data = octx->ctx->vtcm_base;- if (octx->op == HTP_OP_RMS_NORM_MUL) {- octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;- octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size;- } else {- octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size;- }-octx->src0_spad.src = NULL;octx->src1_spad.src = NULL;octx->dst_spad.src = NULL;- FARF(HIGH, "%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-spad-size %u src1-spad-size %u dst-spad-size %u\n", op_type,+ FARF(HIGH, "%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %u src1-vtcm-size %u dst-vtcm-size %u\n", op_type,src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],- octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size);+ kparams->vtcm_src0_size, kparams->vtcm_src1_size, kparams->vtcm_dst_size);if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {+ uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;struct htp_unary_context uctx = {.octx = octx,+ .kparams = kparams,.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads,.src0_nrows = src0_nrows,@@ -1008,32 +857,65 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {.src1_row_size_aligned = src1_row_size_aligned,.dst_row_size_aligned = dst_row_size_aligned,- .src0_spad_half_size = octx->src0_spad.size_per_thread / 2,- .src1_spad_half_size = (octx->op == HTP_OP_RMS_NORM_MUL) ? (octx->src1_spad.size_per_thread / (broadcast_weight ? 1 : 2)) : 0,- .dst_spad_half_size = octx->dst_spad.size_per_thread / 2,+ .src0_vtcm_half_size = kparams->vtcm_src0_size_per_thread / 2,+ .src1_vtcm_half_size = (octx->op == HTP_OP_RMS_NORM_MUL) ? (kparams->vtcm_src1_size_per_thread / (broadcast_weight ? 1 : 2)) : 0,+ .dst_vtcm_half_size = kparams->vtcm_dst_size_per_thread / 2,- .block = (octx->src0_spad.size_per_thread / 2) / src0_row_size_aligned,+ .block = kparams->block,.nc = src0->ne[0],+ .col_tile = (uint32_t) kparams->col_tile,.broadcast_weight = broadcast_weight,- };- worker_pool_run_func(octx->ctx->worker_pool, unary_job_f32_per_thread, &uctx, n_threads);- }+ .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, 0),+ .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, kparams->vtcm_src0_size, kparams->vtcm_src1_size > 0),+ .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, kparams->vtcm_src0_size + kparams->vtcm_src1_size),- return err;-}--int op_tri(struct htp_ops_context * octx) {- int err = HTP_STATUS_OK;+ .vtcm_src0_size_per_thread = kparams->vtcm_src0_size_per_thread,+ .vtcm_src1_size_per_thread = kparams->vtcm_src1_size_per_thread,+ .vtcm_dst_size_per_thread = kparams->vtcm_dst_size_per_thread,+ };- switch (octx->src[0]->type) {- case HTP_TYPE_F32:- err = execute_op_unary_f32(octx);- break;+ FARF(HIGH, "%s: %s mode (col_tile %u)\n", op_type, col_tile ? "tiled" : "row-block", col_tile);++ worker_callback_t task_func = NULL;+ if (col_tile) {+ switch (octx->op) {+ case HTP_OP_SCALE: task_func = unary_task_f32_tiled_scale; break;+ case HTP_OP_SQR: task_func = unary_task_f32_tiled_sqr; break;+ case HTP_OP_SQRT: task_func = unary_task_f32_tiled_sqrt; break;+ case HTP_OP_UNARY_NEG: task_func = unary_task_f32_tiled_unary_neg; break;+ case HTP_OP_UNARY_EXP: task_func = unary_task_f32_tiled_unary_exp; break;+ case HTP_OP_UNARY_SIGMOID: task_func = unary_task_f32_tiled_unary_sigmoid; break;+ case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_tiled_unary_softplus; break;+ case HTP_OP_UNARY_TANH: task_func = unary_task_f32_tiled_unary_tanh; break;+ case HTP_OP_TRI: task_func = unary_task_f32_tiled_tri; break;+ default: break;+ }+ } else {+ switch (octx->op) {+ case HTP_OP_NORM: task_func = unary_task_f32_norm; break;+ case HTP_OP_RMS_NORM: task_func = unary_task_f32_rms_norm; break;+ case HTP_OP_RMS_NORM_MUL: task_func = unary_task_f32_rms_norm_mul; break;+ case HTP_OP_SCALE: task_func = unary_task_f32_scale; break;+ case HTP_OP_SQR: task_func = unary_task_f32_sqr; break;+ case HTP_OP_SQRT: task_func = unary_task_f32_sqrt; break;+ case HTP_OP_UNARY_NEG: task_func = unary_task_f32_unary_neg; break;+ case HTP_OP_UNARY_EXP: task_func = unary_task_f32_unary_exp; break;+ case HTP_OP_UNARY_SIGMOID: task_func = unary_task_f32_unary_sigmoid; break;+ case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_unary_softplus; break;+ case HTP_OP_UNARY_TANH: task_func = unary_task_f32_unary_tanh; break;+ case HTP_OP_L2_NORM: task_func = unary_task_f32_l2_norm; break;+ case HTP_OP_TRI: task_func = unary_task_f32_tri; break;+ default: break;+ }+ }- default:+ if (task_func) {+ worker_pool_run_func(octx->ctx->worker_pool, task_func, &uctx, n_threads);+ } else {+ FARF(ERROR, "execute_op_unary_f32: task function is NULL for op %d\n", octx->op);err = HTP_STATUS_NO_SUPPORT;- break;+ }}return err;diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.hnew file mode 100644index 0000000000..b90b095ac9--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/unary-ops.h@@ -0,0 +1,162 @@+#ifndef HTP_UNARY_OPS_H+#define HTP_UNARY_OPS_H++#include "hex-common.h"+#include "htp-ops.h"++// Op-specific struct for precomputed unary params+struct htp_unary_kernel_params {+ uint32_t n_threads;+ uint32_t col_tile;+ uint32_t vtcm_row_per_thread;+ uint32_t block;+ uint32_t broadcast_weight;++ uint32_t vtcm_src0_size_per_thread;+ uint32_t vtcm_src1_size_per_thread;+ uint32_t vtcm_dst_size_per_thread;++ uint32_t vtcm_src0_size;+ uint32_t vtcm_src1_size;+ uint32_t vtcm_dst_size;++ uint32_t src0_row_size_aligned;+ uint32_t src1_row_size_aligned;+ uint32_t dst_row_size_aligned;++ uint32_t vtcm_size;++ // Fastdiv helpers+ struct fastdiv_values div_ne01;+ struct fastdiv_values div_ne02;+ struct fastdiv_values div_ne012;+ struct fastdiv_values div_tpr;+};++#if defined(__cplusplus)+static_assert(sizeof(struct htp_unary_kernel_params) <= 128, "htp_unary_kernel_params is too large for kernel_params blob");+#else+_Static_assert(sizeof(struct htp_unary_kernel_params) <= 128, "htp_unary_kernel_params is too large for kernel_params blob");+#endif++static inline bool htp_op_is_unary(uint32_t opcode) {+ switch (opcode) {+ case HTP_OP_NORM:+ case HTP_OP_RMS_NORM:+ case HTP_OP_RMS_NORM_MUL:+ case HTP_OP_SCALE:+ case HTP_OP_SQR:+ case HTP_OP_SQRT:+ case HTP_OP_UNARY_NEG:+ case HTP_OP_UNARY_EXP:+ case HTP_OP_UNARY_SIGMOID:+ case HTP_OP_UNARY_SOFTPLUS:+ case HTP_OP_UNARY_TANH:+ case HTP_OP_L2_NORM:+ case HTP_OP_TRI:+ return true;+ default:+ return false;+ }+}++struct htp_unary_vtcm_layout {+ size_t total_bytes;+ size_t off_src0;+ size_t off_src1;+ size_t off_dst;++ size_t src0_bytes;+ size_t src1_bytes;+ size_t dst_bytes;+};++static inline void htp_unary_vtcm_layout_build(+ struct htp_unary_vtcm_layout * L,+ uint32_t op,+ uint32_t ne00,+ uint32_t ne10,+ uint32_t ne11,+ bool broadcast_weight,+ uint32_t n_threads,+ size_t vtcm_size,+ uint32_t * out_col_tile,+ uint32_t * out_vtcm_row_per_thread+) {+ const size_t src0_data_row_size = ne00 * sizeof(float);+ const size_t dst_data_row_size = ne10 * sizeof(float);++ const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);+ const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128);++ size_t src1_row_size_aligned = 0;+ if (op == HTP_OP_RMS_NORM_MUL) {+ const size_t src1_data_row_size = ne11 * sizeof(float);+ src1_row_size_aligned = hex_round_up(src1_data_row_size, 128);+ }++ size_t vtcm_size_per_row = 0;+ size_t vtcm_row_per_thread = 0;++ if (op == HTP_OP_RMS_NORM_MUL) {+ if (broadcast_weight) {+ size_t available_vtcm = vtcm_size;+ size_t src1_vtcm_total = n_threads * src1_row_size_aligned;+ if (available_vtcm > src1_vtcm_total) {+ available_vtcm -= src1_vtcm_total;+ } else {+ available_vtcm = 0;+ }+ vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned);+ vtcm_row_per_thread = available_vtcm / (n_threads * vtcm_size_per_row);+ } else {+ vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned + src1_row_size_aligned);+ vtcm_row_per_thread = vtcm_size / (n_threads * vtcm_size_per_row);+ }+ } else {+ vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned);+ vtcm_row_per_thread = vtcm_size / (n_threads * vtcm_size_per_row);+ }++ const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM ||+ op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM);+ uint32_t col_tile = 0;++ if (vtcm_row_per_thread == 0 && !is_reduction) {+ const size_t per_thread_budget = vtcm_size / n_threads;+ const size_t col_tile_bytes = hex_align_down(per_thread_budget / 4, 128);+ col_tile = (uint32_t) (col_tile_bytes / sizeof(float));++ L->src0_bytes = col_tile_bytes * 2;+ L->dst_bytes = col_tile_bytes * 2;+ L->src1_bytes = 0;+ } else {+ L->src0_bytes = src0_row_size_aligned * vtcm_row_per_thread * 2;+ L->dst_bytes = dst_row_size_aligned * vtcm_row_per_thread * 2;+ if (op == HTP_OP_RMS_NORM_MUL) {+ if (broadcast_weight) {+ L->src1_bytes = src1_row_size_aligned;+ } else {+ L->src1_bytes = src1_row_size_aligned * vtcm_row_per_thread * 2;+ }+ } else {+ L->src1_bytes = 0;+ }+ }++ L->off_src0 = 0;+ if (op == HTP_OP_RMS_NORM_MUL) {+ L->off_src1 = L->off_src0 + L->src0_bytes * n_threads;+ L->off_dst = L->off_src1 + L->src1_bytes * n_threads;+ } else {+ L->off_src1 = 0;+ L->off_dst = L->off_src0 + L->src0_bytes * n_threads;+ }++ L->total_bytes = L->off_dst + L->dst_bytes * n_threads;++ *out_col_tile = col_tile;+ *out_vtcm_row_per_thread = vtcm_row_per_thread;+}++#endif /* HTP_UNARY_OPS_H */