Skip to content

LLM inference in C/C++

C++ ggml

ggml-hexagon: add F16 support for unary ops (#28228)

cqderekf027c4f1b025
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index 04fb9a2233..104201daff 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -4005,8 +4005,10 @@ static void ggml_hexagon_precompute_unary_params(
kparams->n_threads = n_threads;
- const size_t src0_data_row_size = src0->ne[0] * sizeof(float);
- const size_t dst_data_row_size = dst->ne[0] * sizeof(float);
+ const size_t elem_size = ggml_type_size(src0->type);
+
+ const size_t src0_data_row_size = src0->ne[0] * elem_size;
+ const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type);
const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);
const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128);
@@ -4020,7 +4022,7 @@ static void ggml_hexagon_precompute_unary_params(
if (op == HTP_OP_RMS_NORM_MUL) {
GGML_ASSERT(src1 != nullptr);
- src1_data_row_size = src1->ne[0] * sizeof(float);
+ src1_data_row_size = src1->ne[0] * ggml_type_size(src1->type);
src1_row_size_aligned = hex_round_up(src1_data_row_size, 128);
broadcast_weight = (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1);
}
@@ -4034,7 +4036,7 @@ static void ggml_hexagon_precompute_unary_params(
htp_unary_vtcm_layout_build(&L, op, src0->ne[0], dst->ne[0],
op == HTP_OP_RMS_NORM_MUL ? src1->ne[0] : 0,
- broadcast_weight, n_threads, sess->vtcm_size,
+ broadcast_weight, n_threads, sess->vtcm_size, elem_size,
&col_tile, &vtcm_row_per_thread);
kparams->col_tile = col_tile;
@@ -4451,15 +4453,39 @@ static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * ses
const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * dst = op;
- if (src0->type != GGML_TYPE_F32) {
+ if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) {
return false;
}
- if (dst->type != GGML_TYPE_F32) {
+ if (dst->type != src0->type) {
return false;
}
if (!ggml_is_contiguous_rows(src0)) {
return false;
}
+
+ // F16 device kernels only cover this explicit whitelist (must stay in sync with
+ // the is_f16 whitelist in execute_op_unary(), unary-ops.c).
+ if (src0->type == GGML_TYPE_F16) {
+ switch (op->op) {
+ case GGML_OP_NORM:
+ case GGML_OP_RMS_NORM:
+ case GGML_OP_L2_NORM:
+ case GGML_OP_SCALE:
+ case GGML_OP_CLAMP:
+ case GGML_OP_SQR:
+ case GGML_OP_SQRT:
+ case GGML_OP_LOG:
+ break;
+ case GGML_OP_UNARY:
+ if (ggml_get_unary_op(op) != GGML_UNARY_OP_ABS) {
+ return false;
+ }
+ break;
+ default:
+ return false;
+ }
+ }
+
if (!ggml_are_same_shape(src0, dst)) {
return false;
}
diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h
index 765c357766..5ef7463426 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-arith.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h
@@ -358,6 +358,54 @@ static inline void hvx_clamp_scalar_f32(uint8_t * restrict dst, const uint8_t *
}
}
+#define HVX_OP_CLAMP_SCALAR_F16(v) \
+ ({ \
+ HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VhfVhf(v, max_vec); \
+ HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VhfVhf(min_vec, v); \
+ HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \
+ Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \
+ })
+
+static inline void hvx_clamp_scalar_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) {
+ const HVX_Vector min_vec = hvx_vec_splat_f16(min);
+ const HVX_Vector max_vec = hvx_vec_splat_f16(max);
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src % 128 == 0);
+ hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR_F16);
+}
+
+static inline void hvx_clamp_scalar_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) {
+ const HVX_Vector min_vec = hvx_vec_splat_f16(min);
+ const HVX_Vector max_vec = hvx_vec_splat_f16(max);
+ assert((unsigned long) dst % 128 == 0);
+ hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR_F16);
+}
+
+static inline void hvx_clamp_scalar_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) {
+ const HVX_Vector min_vec = hvx_vec_splat_f16(min);
+ const HVX_Vector max_vec = hvx_vec_splat_f16(max);
+ assert((unsigned long) src % 128 == 0);
+ hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR_F16);
+}
+
+static inline void hvx_clamp_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) {
+ const HVX_Vector min_vec = hvx_vec_splat_f16(min);
+ const HVX_Vector max_vec = hvx_vec_splat_f16(max);
+ hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR_F16);
+}
+
+static inline void hvx_clamp_scalar_f16(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, const int num_elems) {
+ if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) {
+ hvx_clamp_scalar_f16_aa(dst, src, min, max, num_elems);
+ } else if (hex_is_aligned((void *) dst, 128)) {
+ hvx_clamp_scalar_f16_au(dst, src, min, max, num_elems);
+ } else if (hex_is_aligned((void *) src, 128)) {
+ hvx_clamp_scalar_f16_ua(dst, src, min, max, num_elems);
+ } else {
+ hvx_clamp_scalar_f16_uu(dst, src, min, max, num_elems);
+ }
+}
+
//
// Abs
//
@@ -386,11 +434,69 @@ static inline void hvx_abs_f32_aa(uint8_t * restrict dst, const uint8_t * restri
}
}
+#define hvx_abs_f16_loop_body(dst_type, src_type, vec_store) \
+ do { \
+ dst_type * restrict vdst = (dst_type *) dst; \
+ src_type * restrict vsrc = (src_type *) src; \
+ \
+ const uint32_t elem_size = sizeof(_Float16); \
+ const uint32_t epv = 128 / elem_size; \
+ const uint32_t nvec = n / epv; \
+ const uint32_t nloe = n % epv; \
+ \
+ uint32_t i = 0; \
+ \
+ _Pragma("unroll(4)") \
+ for (; i < nvec; i++) { \
+ vdst[i] = hvx_vec_abs_f16(vsrc[i]); \
+ } \
+ if (nloe) { \
+ HVX_Vector v = hvx_vec_abs_f16(vsrc[i]); \
+ vec_store((void *) &vdst[i], nloe * elem_size, v); \
+ } \
+ } while(0)
+
+static inline void hvx_abs_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src % 128 == 0);
+ hvx_abs_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
+}
+
+static inline void hvx_abs_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ hvx_abs_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
+}
+
+static inline void hvx_abs_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) src % 128 == 0);
+ hvx_abs_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
+}
+
+static inline void hvx_abs_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ hvx_abs_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
+}
+
+static inline void hvx_abs_f16(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) {
+ if (hex_is_aligned((void *) dst, 128)) {
+ if (hex_is_aligned((void *) src, 128)) {
+ hvx_abs_f16_aa(dst, src, num_elems);
+ } else {
+ hvx_abs_f16_au(dst, src, num_elems);
+ }
+ } else {
+ if (hex_is_aligned((void *) src, 128)) {
+ hvx_abs_f16_ua(dst, src, num_elems);
+ } else {
+ hvx_abs_f16_uu(dst, src, num_elems);
+ }
+ }
+}
+
//
// Square
//
-#define hvx_sqr_f32_loop_body(dst_type, src_type, vec_store) \
+#define hvx_sqr_f32_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
@@ -404,10 +510,10 @@ static inline void hvx_abs_f32_aa(uint8_t * restrict dst, const uint8_t * restri
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
- vdst[i] = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \
+ vdst[i] = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \
} \
if (nloe) { \
- HVX_Vector v = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \
+ HVX_Vector v = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \
vec_store((void *) &vdst[i], nloe * elem_size, v); \
} \
} while(0)
@@ -448,6 +554,64 @@ static inline void hvx_sqr_f32(uint8_t * restrict dst, const uint8_t * restrict
}
}
+#define hvx_sqr_f16_loop_body(dst_type, src_type, vec_store) \
+ do { \
+ dst_type * restrict vdst = (dst_type *) dst; \
+ src_type * restrict vsrc = (src_type *) src; \
+ \
+ const uint32_t elem_size = sizeof(_Float16); \
+ const uint32_t epv = 128 / elem_size; \
+ const uint32_t nvec = n / epv; \
+ const uint32_t nloe = n % epv; \
+ \
+ uint32_t i = 0; \
+ \
+ _Pragma("unroll(4)") \
+ for (; i < nvec; i++) { \
+ vdst[i] = HVX_OP_MUL_F16(vsrc[i], vsrc[i]); \
+ } \
+ if (nloe) { \
+ HVX_Vector v = HVX_OP_MUL_F16(vsrc[i], vsrc[i]); \
+ vec_store((void *) &vdst[i], nloe * elem_size, v); \
+ } \
+ } while(0)
+
+static inline void hvx_sqr_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src % 128 == 0);
+ hvx_sqr_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
+}
+
+static inline void hvx_sqr_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ hvx_sqr_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
+}
+
+static inline void hvx_sqr_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) src % 128 == 0);
+ hvx_sqr_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
+}
+
+static inline void hvx_sqr_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ hvx_sqr_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
+}
+
+static inline void hvx_sqr_f16(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) {
+ if (hex_is_aligned((void *) dst, 128)) {
+ if (hex_is_aligned((void *) src, 128)) {
+ hvx_sqr_f16_aa(dst, src, num_elems);
+ } else {
+ hvx_sqr_f16_au(dst, src, num_elems);
+ }
+ } else {
+ if (hex_is_aligned((void *) src, 128)) {
+ hvx_sqr_f16_ua(dst, src, num_elems);
+ } else {
+ hvx_sqr_f16_uu(dst, src, num_elems);
+ }
+ }
+}
+
#undef HVX_OP_ADD_F32
#undef HVX_OP_SUB_F32
#undef HVX_OP_MUL_F32
@@ -464,6 +628,7 @@ static inline void hvx_sqr_f32(uint8_t * restrict dst, const uint8_t * restrict
#undef hvx_scalar_loop_body
#undef HVX_OP_MIN_SCALAR
#undef HVX_OP_CLAMP_SCALAR
+#undef HVX_OP_CLAMP_SCALAR_F16
#undef DEFINE_HVX_BINARY_OP_VARIANTS
#undef HVX_BINARY_DISPATCHER
#undef UNUSED
diff --git a/ggml/src/ggml-hexagon/htp/hvx-log.h b/ggml/src/ggml-hexagon/htp/hvx-log.h
index a209f88d55..491041d5ad 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-log.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-log.h
@@ -86,4 +86,33 @@ static inline void hvx_log_f32_aa(uint8_t * restrict dst, const uint8_t * restri
}
}
+// Compute log(x) for f16 by promoting to f32, applying hvx_vec_log_f32, and narrowing back.
+static inline void hvx_log_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src % 128 == 0);
+
+ HVX_Vector * restrict vdst = (HVX_Vector *) dst;
+ HVX_Vector * restrict vsrc = (HVX_Vector *) src;
+
+ const uint32_t nvec = n / VLEN_FP16;
+ const uint32_t nloe = n % VLEN_FP16;
+
+ uint32_t i = 0;
+
+ _Pragma("unroll(4)")
+ for (; i < nvec; i++) {
+ HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
+ HVX_Vector r0 = hvx_vec_log_f32(Q6_V_lo_W(p));
+ HVX_Vector r1 = hvx_vec_log_f32(Q6_V_hi_W(p));
+ vdst[i] = hvx_vec_f32_to_f16(r0, r1);
+ }
+ if (nloe) {
+ HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
+ HVX_Vector r0 = hvx_vec_log_f32(Q6_V_lo_W(p));
+ HVX_Vector r1 = hvx_vec_log_f32(Q6_V_hi_W(p));
+ HVX_Vector v = hvx_vec_f32_to_f16(r0, r1);
+ hvx_vec_store_a((void *) &vdst[i], nloe * SIZEOF_FP16, v);
+ }
+}
+
#endif /* HVX_LOG_H */
diff --git a/ggml/src/ggml-hexagon/htp/hvx-norm.h b/ggml/src/ggml-hexagon/htp/hvx-norm.h
index a8645e412d..7ea945a339 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-norm.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-norm.h
@@ -254,4 +254,201 @@ static inline void hvx_fast_l2_norm_f32(const uint8_t * restrict src,
}
}
+// F16 norm kernels: reduce and scale in f32 (via promote/narrow), matching the
+// precision-preserving pattern used by the flash-attn f16 kernels.
+
+static inline void hvx_fast_rms_norm_f16(const uint8_t * restrict src,
+ uint8_t * restrict dst,
+ const int num_elems,
+ float epsilon) {
+
+ const HVX_Vector * restrict v_src = (HVX_Vector *) src;
+ HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
+
+ const int nvec = num_elems / VLEN_FP16; // number of full f16 vectors
+ const int nloe = num_elems % VLEN_FP16; // leftover elements
+
+ HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000);
+ HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);
+
+ #pragma unroll(4)
+ for (int i = 0; i < nvec; i++) {
+ HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
+ HVX_Vector p0 = Q6_V_lo_W(p);
+ HVX_Vector p1 = Q6_V_hi_W(p);
+ sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
+ sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
+ }
+
+ if (nloe > 0) {
+ HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
+ HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
+ HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
+ HVX_Vector p0 = Q6_V_lo_W(p);
+ HVX_Vector p1 = Q6_V_hi_W(p);
+ sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
+ sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
+ }
+
+ sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));
+
+ HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);
+ HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);
+ HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v);
+ HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v);
+
+ HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v));
+
+ #pragma unroll(4)
+ for (int i = 0; i < nvec; i++) {
+ HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
+ HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v));
+ HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v));
+ v_dst[i] = hvx_vec_f32_to_f16(r0, r1);
+ }
+
+ if (nloe > 0) {
+ HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
+ HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
+ HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
+ HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v));
+ HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v));
+ HVX_Vector result = hvx_vec_f32_to_f16(r0, r1);
+ hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result);
+ }
+}
+
+static inline void hvx_fast_norm_f16(const uint8_t * restrict src,
+ uint8_t * restrict dst,
+ const int num_elems,
+ float epsilon) {
+
+ const HVX_Vector * restrict v_src = (HVX_Vector *) src;
+ HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
+
+ const int nvec = num_elems / VLEN_FP16;
+ const int nloe = num_elems % VLEN_FP16;
+
+ HVX_Vector sum_sq_v = Q6_V_vsplat_R(0x00000000);
+ HVX_Vector sum_x_v = Q6_V_vsplat_R(0x00000000);
+ HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);
+
+ #pragma unroll(4)
+ for (int i = 0; i < nvec; i++) {
+ HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
+ HVX_Vector p0 = Q6_V_lo_W(p);
+ HVX_Vector p1 = Q6_V_hi_W(p);
+ sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
+ sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
+ sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p0, Q6_V_vzero()));
+ sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p1, Q6_V_vzero()));
+ }
+
+ if (nloe > 0) {
+ HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
+ HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
+ HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
+ HVX_Vector p0 = Q6_V_lo_W(p);
+ HVX_Vector p1 = Q6_V_hi_W(p);
+ sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
+ sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
+ sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p0, Q6_V_vzero()));
+ sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p1, Q6_V_vzero()));
+ }
+
+ sum_sq_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_sq_v));
+ sum_x_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_x_v));
+
+ HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);
+ HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);
+ HVX_Vector mean_sq_v = Q6_Vqf32_vmpy_VsfVsf(sum_sq_v, denom_v);
+ HVX_Vector mean_x_v = Q6_Vqf32_vmpy_VsfVsf(sum_x_v, denom_v);
+ HVX_Vector mean_x_sq_v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(mean_x_v), Q6_Vsf_equals_Vqf32(mean_x_v));
+ HVX_Vector var_v = Q6_Vqf32_vsub_Vqf32Vqf32(mean_sq_v, mean_x_sq_v);
+ HVX_Vector var_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(var_v, epsilon_v);
+
+ HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(var_epsilon_v));
+ HVX_Vector mean_x_b = hvx_vec_repl_f32(Q6_Vsf_equals_Vqf32(mean_x_v));
+
+ #pragma unroll(4)
+ for (int i = 0; i < nvec; i++) {
+ HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
+ HVX_Vector d0 = Q6_Vqf32_vsub_VsfVsf(Q6_V_lo_W(p), mean_x_b);
+ HVX_Vector d1 = Q6_Vqf32_vsub_VsfVsf(Q6_V_hi_W(p), mean_x_b);
+ HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d0), scale_v));
+ HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d1), scale_v));
+ v_dst[i] = hvx_vec_f32_to_f16(r0, r1);
+ }
+
+ if (nloe > 0) {
+ HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
+ HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
+ HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
+ HVX_Vector d0 = Q6_Vqf32_vsub_VsfVsf(Q6_V_lo_W(p), mean_x_b);
+ HVX_Vector d1 = Q6_Vqf32_vsub_VsfVsf(Q6_V_hi_W(p), mean_x_b);
+ HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d0), scale_v));
+ HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d1), scale_v));
+ HVX_Vector result = hvx_vec_f32_to_f16(r0, r1);
+ hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result);
+ }
+}
+
+static inline void hvx_fast_l2_norm_f16(const uint8_t * restrict src,
+ uint8_t * restrict dst,
+ const int num_elems,
+ float epsilon) {
+
+ const HVX_Vector * restrict v_src = (HVX_Vector *) src;
+ HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
+
+ const int nvec = num_elems / VLEN_FP16;
+ const int nloe = num_elems % VLEN_FP16;
+
+ HVX_Vector sum_v = hvx_vec_splat_f32(0.0f);
+
+ #pragma unroll(4)
+ for (int i = 0; i < nvec; i++) {
+ HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
+ HVX_Vector p0 = Q6_V_lo_W(p);
+ HVX_Vector p1 = Q6_V_hi_W(p);
+ sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
+ sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
+ }
+
+ if (nloe > 0) {
+ HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
+ HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
+ HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
+ HVX_Vector p0 = Q6_V_lo_W(p);
+ HVX_Vector p1 = Q6_V_hi_W(p);
+ sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
+ sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
+ }
+
+ HVX_Vector sum_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));
+ HVX_Vector rsqrt_v = hvx_vec_rsqrt_f32(sum_sf);
+ HVX_Vector sqrt_v = hvx_vec_inverse_f32(rsqrt_v);
+ HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);
+ HVX_Vector denom_v = Q6_Vsf_vmax_VsfVsf(sqrt_v, epsilon_v);
+ HVX_Vector scale_v = hvx_vec_inverse_f32(denom_v);
+
+ #pragma unroll(4)
+ for (int i = 0; i < nvec; i++) {
+ HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
+ HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v));
+ HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v));
+ v_dst[i] = hvx_vec_f32_to_f16(r0, r1);
+ }
+
+ if (nloe > 0) {
+ HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
+ HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
+ HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
+ HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v));
+ HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v));
+ HVX_Vector result = hvx_vec_f32_to_f16(r0, r1);
+ hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result);
+ }
+}
+
#endif // HVX_NORM_H
diff --git a/ggml/src/ggml-hexagon/htp/hvx-scale.h b/ggml/src/ggml-hexagon/htp/hvx-scale.h
index c65c98639d..9b1a28f529 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-scale.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-scale.h
@@ -130,4 +130,70 @@ static inline void hvx_scale_offset_f32(uint8_t * restrict dst, const uint8_t *
}
}
+// Scale+offset computed by promoting f16 -> f32, then narrowing the result back to f16.
+#define hvx_scale_offset_f16_loop_body(dst_type, src_type, vec_store) \
+ do { \
+ dst_type * restrict vdst = (dst_type *) dst; \
+ src_type * restrict vsrc = (src_type *) src; \
+ \
+ HVX_Vector vs = hvx_vec_splat_f32(scale); \
+ HVX_Vector vo = hvx_vec_splat_f32(offset); \
+ \
+ const uint32_t nvec = n / VLEN_FP16; \
+ const uint32_t nloe = n % VLEN_FP16; \
+ \
+ uint32_t i = 0; \
+ \
+ _Pragma("unroll(4)") \
+ for (; i < nvec; ++i) { \
+ HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \
+ HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), vs), vo)); \
+ HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), vs), vo)); \
+ vdst[i] = hvx_vec_f32_to_f16(r0, r1); \
+ } \
+ if (nloe) { \
+ HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \
+ HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), vs), vo)); \
+ HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), vs), vo)); \
+ HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); \
+ vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, v); \
+ } \
+ } while(0)
+
+static inline void hvx_scale_offset_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
+ assert((size_t) dst % 128 == 0);
+ assert((size_t) src % 128 == 0);
+ hvx_scale_offset_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
+}
+
+static inline void hvx_scale_offset_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
+ assert((size_t) dst % 128 == 0);
+ hvx_scale_offset_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
+}
+
+static inline void hvx_scale_offset_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
+ assert((size_t) src % 128 == 0);
+ hvx_scale_offset_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
+}
+
+static inline void hvx_scale_offset_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
+ hvx_scale_offset_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
+}
+
+static inline void hvx_scale_offset_f16(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
+ if (((size_t) dst & 127) == 0) {
+ if (((size_t) src & 127) == 0) {
+ hvx_scale_offset_f16_aa(dst, src, n, scale, offset);
+ } else {
+ hvx_scale_offset_f16_au(dst, src, n, scale, offset);
+ }
+ } else {
+ if (((size_t) src & 127) == 0) {
+ hvx_scale_offset_f16_ua(dst, src, n, scale, offset);
+ } else {
+ hvx_scale_offset_f16_uu(dst, src, n, scale, offset);
+ }
+ }
+}
+
#endif // HVX_SCALE_H
diff --git a/ggml/src/ggml-hexagon/htp/hvx-sqrt.h b/ggml/src/ggml-hexagon/htp/hvx-sqrt.h
index e31a1006d2..abdded5ce6 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-sqrt.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-sqrt.h
@@ -123,4 +123,67 @@ static inline void hvx_sqrt_f32(uint8_t * restrict dst, const uint8_t * restrict
}
}
+// Compute sqrt(x) for f16 by promoting to f32, applying hvx_vec_rsqrt_f32, and narrowing back.
+#define hvx_sqrt_f16_loop_body(dst_type, src_type, vec_store) \
+ do { \
+ dst_type * restrict vdst = (dst_type *) dst; \
+ src_type * restrict vsrc = (src_type *) src; \
+ \
+ const uint32_t nvec = n / VLEN_FP16; \
+ const uint32_t nloe = n % VLEN_FP16; \
+ \
+ uint32_t i = 0; \
+ \
+ _Pragma("unroll(4)") \
+ for (; i < nvec; i++) { \
+ HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \
+ HVX_Vector r0 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_lo_W(p)), Q6_V_lo_W(p)); \
+ HVX_Vector r1 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_hi_W(p)), Q6_V_hi_W(p)); \
+ vdst[i] = hvx_vec_f32_to_f16(r0, r1); \
+ } \
+ if (nloe) { \
+ HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \
+ HVX_Vector r0 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_lo_W(p)), Q6_V_lo_W(p)); \
+ HVX_Vector r1 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_hi_W(p)), Q6_V_hi_W(p)); \
+ HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); \
+ vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, v); \
+ } \
+ } while(0)
+
+static inline void hvx_sqrt_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ assert((unsigned long) src % 128 == 0);
+ hvx_sqrt_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
+}
+
+static inline void hvx_sqrt_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) dst % 128 == 0);
+ hvx_sqrt_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
+}
+
+static inline void hvx_sqrt_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ assert((unsigned long) src % 128 == 0);
+ hvx_sqrt_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
+}
+
+static inline void hvx_sqrt_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+ hvx_sqrt_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
+}
+
+static inline void hvx_sqrt_f16(uint8_t * restrict dst, const uint8_t * restrict src, const int num_elems) {
+ if ((unsigned long) dst % 128 == 0) {
+ if ((unsigned long) src % 128 == 0) {
+ hvx_sqrt_f16_aa(dst, src, num_elems);
+ } else {
+ hvx_sqrt_f16_au(dst, src, num_elems);
+ }
+ } else {
+ if ((unsigned long) src % 128 == 0) {
+ hvx_sqrt_f16_ua(dst, src, num_elems);
+ } else {
+ hvx_sqrt_f16_uu(dst, src, num_elems);
+ }
+ }
+}
+
#endif /* HVX_SQRT_H */
diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c
index 1a632bf563..5e62b4a9bd 100644
--- a/ggml/src/ggml-hexagon/htp/unary-ops.c
+++ b/ggml/src/ggml-hexagon/htp/unary-ops.c
@@ -234,6 +234,146 @@ static void sqrt_f32(const float * restrict src,
}
}
+static void scale_f16(const _Float16 * restrict src,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+ float scale = 0.f;
+ float bias = 0.f;
+ memcpy(&scale, &op_params[0], sizeof(float));
+ memcpy(&bias, &op_params[1], sizeof(float));
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+ hvx_scale_offset_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0, scale, bias);
+ }
+}
+
+static void clamp_f16(const _Float16 * restrict src,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+ float min = 0.f;
+ float max = 0.f;
+ memcpy(&min, &op_params[0], sizeof(float));
+ memcpy(&max, &op_params[1], sizeof(float));
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+ hvx_clamp_scalar_f16(dst_local, src_local, (_Float16) min, (_Float16) max, ne0);
+ }
+}
+
+static void rms_norm_f16(const _Float16 * restrict src,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+ float epsilon = 0.f;
+ memcpy(&epsilon, op_params, sizeof(float));
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+ hvx_fast_rms_norm_f16((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon);
+ }
+}
+
+static void norm_f16(const _Float16 * restrict src,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+ float epsilon = 0.f;
+ memcpy(&epsilon, op_params, sizeof(float));
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+ hvx_fast_norm_f16((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon);
+ }
+}
+
+static void sqr_f16(const _Float16 * restrict src,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+ hvx_sqr_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
+ }
+}
+
+static void sqrt_f16(const _Float16 * restrict src,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+ hvx_sqrt_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
+ }
+}
+
+static void abs_f16(const _Float16 * restrict src,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+ hvx_abs_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
+ }
+}
+
+static void log_f16(const _Float16 * restrict src,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+ hvx_log_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
+ }
+}
+
+static void l2_norm_f16(const _Float16 * restrict src,
+ _Float16 * restrict dst,
+ const uint32_t num_rows,
+ const struct htp_unary_context * uctx) {
+ htp_unary_op_preamble;
+ float epsilon = 0.f;
+ memcpy(&epsilon, op_params, sizeof(float));
+
+ for (uint32_t ir = 0; ir < num_rows; ir++) {
+ const uint8_t * restrict src_f = (const uint8_t *)src + (ir * src0_row_size_aligned);
+ uint8_t * restrict dst_f = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+ hvx_fast_l2_norm_f16((const uint8_t *)src_f, (uint8_t *)dst_f, ne0, epsilon);
+ }
+}
+
static void neg_f32(const float * restrict src,
float * restrict dst,
const uint32_t num_rows,
@@ -471,8 +611,8 @@ static void log_f32(const float * restrict src,
}
}
-#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \
-static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * data) { \
+#define DEFINE_UNARY_TASK_IMPL(NAME, TYPE, SUFFIX, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \
+static void unary_task_##SUFFIX##_##NAME(unsigned int nth, unsigned int ith, void * data) { \
const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \
struct htp_ops_context * octx = uctx->octx; \
const struct htp_tensor * src = octx->src[0]; \
@@ -536,7 +676,7 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
const uint32_t dst_max_block = block_dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1); \
const uint32_t BLOCK = MIN(src0_max_block, dst_max_block); \
if (BLOCK == 0) { \
- FARF(ERROR, "unary-f32 : current VTCM reservation %zu is too small, needed at least %zu\n", \
+ FARF(ERROR, "unary-" #SUFFIX " : current VTCM reservation %zu is too small, needed at least %zu\n", \
uctx->vtcm_src0_size_per_thread, src0_row_size_aligned); \
return; \
} \
@@ -578,11 +718,11 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig, \
ne01, div_ne01); \
\
- float * dst_vtcm = (float *) dma_queue_pop(dma_queue).src; \
- float * src0_vtcm = (float *) dma_queue_pop(dma_queue).dst; \
- float * src1_vtcm = NULL; \
+ TYPE * dst_vtcm = (TYPE *) dma_queue_pop(dma_queue).src; \
+ TYPE * src0_vtcm = (TYPE *) dma_queue_pop(dma_queue).dst; \
+ TYPE * src1_vtcm = NULL; \
if ((IS_RMS_NORM_MUL) && !uctx->broadcast_weight) { \
- src1_vtcm = (float *) dma_queue_pop(dma_queue).dst; \
+ src1_vtcm = (TYPE *) dma_queue_pop(dma_queue).dst; \
} \
\
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \
@@ -625,6 +765,10 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
dma_queue_flush(dma_queue); \
}
+// F32 unary task: row-block DMA/VTCM plumbing, float-typed VTCM buffers.
+#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \
+ DEFINE_UNARY_TASK_IMPL(NAME, float, f32, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR)
+
DEFINE_UNARY_TASK(norm, false, false, norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(rms_norm, false, false, rms_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(rms_norm_mul, true, false, rms_norm_mul_f32(src0_vtcm, uctx->broadcast_weight ? (const float *) src1_vtcm_data : src1_vtcm, dst_vtcm, block_size, uctx))
@@ -644,6 +788,18 @@ DEFINE_UNARY_TASK(unary_log, false, false, log_f32(src0_vtcm, dst_vtcm, blo
DEFINE_UNARY_TASK(l2_norm, false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(tri, false, true, tri_f32(src0_vtcm, dst_vtcm, block_size, ir, uctx))
+// F16 unary tasks: same DMA/VTCM plumbing as DEFINE_UNARY_TASK, but VTCM buffers are
+// _Float16-typed. None of the current F16 ops need RMS_NORM_MUL or TRI support.
+DEFINE_UNARY_TASK_IMPL(norm, _Float16, f16, false, false, norm_f16(src0_vtcm, dst_vtcm, block_size, uctx))
+DEFINE_UNARY_TASK_IMPL(rms_norm, _Float16, f16, false, false, rms_norm_f16(src0_vtcm, dst_vtcm, block_size, uctx))
+DEFINE_UNARY_TASK_IMPL(scale, _Float16, f16, false, false, scale_f16(src0_vtcm, dst_vtcm, block_size, uctx))
+DEFINE_UNARY_TASK_IMPL(clamp, _Float16, f16, false, false, clamp_f16(src0_vtcm, dst_vtcm, block_size, uctx))
+DEFINE_UNARY_TASK_IMPL(sqr, _Float16, f16, false, false, sqr_f16(src0_vtcm, dst_vtcm, block_size, uctx))
+DEFINE_UNARY_TASK_IMPL(sqrt, _Float16, f16, false, false, sqrt_f16(src0_vtcm, dst_vtcm, block_size, uctx))
+DEFINE_UNARY_TASK_IMPL(l2_norm, _Float16, f16, false, false, l2_norm_f16(src0_vtcm, dst_vtcm, block_size, uctx))
+DEFINE_UNARY_TASK_IMPL(unary_abs, _Float16, f16, false, false, abs_f16(src0_vtcm, dst_vtcm, block_size, uctx))
+DEFINE_UNARY_TASK_IMPL(unary_log, _Float16, f16, false, false, log_f16(src0_vtcm, dst_vtcm, block_size, uctx))
+
// Apply a pointwise unary op to one column tile that is already in VTCM.
#define DEFINE_UNARY_TILED_TASK(NAME, IS_TRI, CORE_TILE_EXPR) \
static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void * data) { \
@@ -892,50 +1048,76 @@ DEFINE_UNARY_TILED_TASK(unary_abs, false, hvx_abs_f32_aa(dst_vtcm, src_vtcm
DEFINE_UNARY_TILED_TASK(unary_log, false, hvx_log_f32_aa(dst_vtcm, src_vtcm, tw))
DEFINE_UNARY_TILED_TASK(tri, true, tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype))
-static int execute_op_unary_f32(struct htp_ops_context * octx) {
+static int execute_op_unary(struct htp_ops_context * octx) {
int err = HTP_STATUS_OK;
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
+ const bool is_f16 = (src0->type == HTP_TYPE_F16);
+
const char * op_type = NULL;
switch (octx->op) {
- case HTP_OP_NORM: op_type = "norm-f32"; break;
- case HTP_OP_RMS_NORM: op_type = "rmsnorm-f32"; break;
- case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break;
- case HTP_OP_SCALE: op_type = "scale-f32"; break;
- case HTP_OP_CLAMP: op_type = "clamp-f32"; break;
- case HTP_OP_SQR: op_type = "sqr-f32"; break;
- case HTP_OP_SQRT: op_type = "sqrt-f32"; break;
- case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break;
- case HTP_OP_UNARY_EXP: op_type = "exp-f32"; break;
- case HTP_OP_UNARY_SIGMOID: op_type = "sigmoid-f32"; break;
- case HTP_OP_UNARY_SILU: op_type = "silu-f32"; break;
- case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break;
- case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break;
- case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break;
- case HTP_OP_UNARY_ABS: op_type = "abs-f32"; break;
- case HTP_OP_UNARY_LOG: op_type = "log-f32"; break;
- case HTP_OP_L2_NORM: op_type = "l2norm-f32"; break;
- case HTP_OP_TRI: op_type = "tri-f32"; break;
+ case HTP_OP_NORM: op_type = is_f16 ? "norm-f16" : "norm-f32"; break;
+ case HTP_OP_RMS_NORM: op_type = is_f16 ? "rmsnorm-f16" : "rmsnorm-f32"; break;
+ case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break;
+ case HTP_OP_SCALE: op_type = is_f16 ? "scale-f16" : "scale-f32"; break;
+ case HTP_OP_CLAMP: op_type = is_f16 ? "clamp-f16" : "clamp-f32"; break;
+ case HTP_OP_SQR: op_type = is_f16 ? "sqr-f16" : "sqr-f32"; break;
+ case HTP_OP_SQRT: op_type = is_f16 ? "sqrt-f16" : "sqrt-f32"; break;
+ case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break;
+ case HTP_OP_UNARY_EXP: op_type = "exp-f32"; break;
+ case HTP_OP_UNARY_SIGMOID: op_type = "sigmoid-f32"; break;
+ case HTP_OP_UNARY_SILU: op_type = "silu-f32"; break;
+ case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break;
+ case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break;
+ case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break;
+ case HTP_OP_UNARY_ABS: op_type = is_f16 ? "abs-f16" : "abs-f32"; break;
+ case HTP_OP_UNARY_LOG: op_type = is_f16 ? "log-f16" : "log-f32"; break;
+ case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break;
+ case HTP_OP_TRI: op_type = "tri-f32"; break;
default:
FARF(ERROR, "Unsupported unary Op %u\n", octx->op);
return HTP_STATUS_NO_SUPPORT;
}
+ // F16 only has row-block kernels for this subset of ops (see the dispatch switch
+ // below) - reject everything else up front, before touching kparams/VTCM.
+ if (is_f16) {
+ switch (octx->op) {
+ case HTP_OP_NORM:
+ case HTP_OP_RMS_NORM:
+ case HTP_OP_SCALE:
+ case HTP_OP_CLAMP:
+ case HTP_OP_SQR:
+ case HTP_OP_SQRT:
+ case HTP_OP_L2_NORM:
+ case HTP_OP_UNARY_ABS:
+ case HTP_OP_UNARY_LOG:
+ break;
+ default:
+ FARF(ERROR, "unary-%s: not supported for F16\n", op_type);
+ return HTP_STATUS_NO_SUPPORT;
+ }
+ }
+
const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params;
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const uint32_t n_threads = kparams->n_threads;
- const size_t src0_data_row_size = src0->ne[0] * sizeof(float);
- const size_t dst_data_row_size = dst->ne[0] * sizeof(float);
+ const size_t elem_size = is_f16 ? sizeof(_Float16) : sizeof(float);
+
+ const size_t src0_data_row_size = src0->ne[0] * elem_size;
+ const size_t dst_data_row_size = dst->ne[0] * elem_size;
const size_t src0_row_size_aligned = kparams->src0_row_size_aligned;
const size_t dst_row_size_aligned = kparams->dst_row_size_aligned;
+ // Always 0 for F16 - htp_unary_vtcm_layout_build() keeps F16 on the row-block path,
+ // since only F32 has unary_task_f32_tiled_* kernels.
const uint32_t col_tile = kparams->col_tile;
size_t src1_data_row_size = 0;
@@ -943,6 +1125,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
bool broadcast_weight = kparams->broadcast_weight;
const struct htp_tensor * src1 = NULL;
+ // RMS_NORM_MUL fusion is F32-only (its weight tensor is always F32; see
+ // try_fuse_node()'s type guard), so this never triggers when is_f16 is true.
if (octx->op == HTP_OP_RMS_NORM_MUL) {
src1 = octx->src[1];
src1_data_row_size = src1->ne[0] * sizeof(float);
@@ -987,7 +1171,7 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
.block = kparams->block,
.nc = src0->ne[0],
- .col_tile = (uint32_t) kparams->col_tile,
+ .col_tile = col_tile,
.broadcast_weight = broadcast_weight,
.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, 0),
@@ -1020,6 +1204,19 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
case HTP_OP_TRI: task_func = unary_task_f32_tiled_tri; break;
default: break;
}
+ } else if (is_f16) {
+ switch (octx->op) {
+ case HTP_OP_NORM: task_func = unary_task_f16_norm; break;
+ case HTP_OP_RMS_NORM: task_func = unary_task_f16_rms_norm; break;
+ case HTP_OP_SCALE: task_func = unary_task_f16_scale; break;
+ case HTP_OP_CLAMP: task_func = unary_task_f16_clamp; break;
+ case HTP_OP_SQR: task_func = unary_task_f16_sqr; break;
+ case HTP_OP_SQRT: task_func = unary_task_f16_sqrt; break;
+ case HTP_OP_L2_NORM: task_func = unary_task_f16_l2_norm; break;
+ case HTP_OP_UNARY_ABS: task_func = unary_task_f16_unary_abs; break;
+ case HTP_OP_UNARY_LOG: task_func = unary_task_f16_unary_log; break;
+ default: break;
+ }
} else {
switch (octx->op) {
case HTP_OP_NORM: task_func = unary_task_f32_norm; break;
@@ -1047,7 +1244,7 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
if (task_func) {
worker_pool_run_func(octx->ctx->worker_pool, task_func, &uctx, n_threads);
} else {
- FARF(ERROR, "execute_op_unary_f32: task function is NULL for op %d\n", octx->op);
+ FARF(ERROR, "execute_op_unary: task function is NULL for op %d\n", octx->op);
err = HTP_STATUS_NO_SUPPORT;
}
}
@@ -1058,7 +1255,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
int op_unary(struct htp_ops_context * octx) {
switch (octx->src[0]->type) {
case HTP_TYPE_F32:
- return execute_op_unary_f32(octx);
+ case HTP_TYPE_F16:
+ return execute_op_unary(octx);
default:
return HTP_STATUS_NO_SUPPORT;
diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h
index 458218ff44..116a591c2d 100644
--- a/ggml/src/ggml-hexagon/htp/unary-ops.h
+++ b/ggml/src/ggml-hexagon/htp/unary-ops.h
@@ -85,17 +85,19 @@ static inline void htp_unary_vtcm_layout_build(
bool broadcast_weight,
uint32_t n_threads,
size_t vtcm_size,
+ size_t elem_size,
uint32_t * out_col_tile,
uint32_t * out_vtcm_row_per_thread
) {
- const size_t src0_data_row_size = ne00 * sizeof(float);
- const size_t dst_data_row_size = ne10 * sizeof(float);
+ const size_t src0_data_row_size = ne00 * elem_size;
+ const size_t dst_data_row_size = ne10 * elem_size;
const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);
const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128);
size_t src1_row_size_aligned = 0;
if (op == HTP_OP_RMS_NORM_MUL) {
+ // RMS_NORM_MUL fusion is F32-only; its weight tensor is always F32.
const size_t src1_data_row_size = ne11 * sizeof(float);
src1_row_size_aligned = hex_round_up(src1_data_row_size, 128);
}
@@ -125,12 +127,19 @@ static inline void htp_unary_vtcm_layout_build(
const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM ||
op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM);
+ // The tiled fallback path below only has F32 task functions (unary_task_f32_tiled_*);
+ // F16 has no tiled kernels, so it must stay on the row-block path like reduction ops.
+ // NOTE: if F16 ends up with vtcm_row_per_thread == 0 here (row too large for the VTCM
+ // budget), execute_op_unary() will see BLOCK == 0 and skip computation for that op
+ // (logged via FARF(ERROR, ...)) since there is no F16 tiled fallback. This is a known
+ // limitation; supporting it would require adding F16 tiled kernels.
+ const bool is_f16 = (elem_size == sizeof(_Float16));
uint32_t col_tile = 0;
- if (vtcm_row_per_thread == 0 && !is_reduction) {
+ if (vtcm_row_per_thread == 0 && !is_reduction && !is_f16) {
const size_t per_thread_budget = vtcm_size / n_threads;
const size_t col_tile_bytes = hex_align_down(per_thread_budget / 4, 128);
- col_tile = (uint32_t) (col_tile_bytes / sizeof(float));
+ col_tile = (uint32_t) (col_tile_bytes / elem_size);
L->src0_bytes = col_tile_bytes * 2;
L->dst_bytes = col_tile_bytes * 2;