LLM inference in C/C++
C++ ggml
hexagon: rope updates (#28628)
Todor Boinovski6d9c82ea2bb3
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex a39df2a878..112e9bae60 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -56,6 +56,7 @@#include "htp/unary-ops.h"#include "htp/get-rows-ops.h"#include "htp/set-rows-ops.h"+#include "htp/rope-ops.h"#include "htp_iface.h"#include "htp-drv.h"@@ -299,6 +300,12 @@ static void ggml_hexagon_precompute_set_rows_params(struct htp_set_rows_kernel_params * kparams);+static void ggml_hexagon_precompute_rope_params(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_rope_kernel_params * kparams+);+static void ggml_hexagon_precompute_fused_mmnx_params(const struct ggml_hexagon_session * sess,const struct ggml_tensor * src0,@@ -4148,6 +4155,36 @@ static void ggml_hexagon_precompute_set_rows_params(kparams->vtcm_size = vtcm_layout.total_bytes;}+static void ggml_hexagon_precompute_rope_params(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_rope_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * dst = op;++ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];+ const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows);++ struct htp_rope_vtcm_layout layout;+ htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads);++ kparams->n_threads = n_threads;+ kparams->src0_nrows = src0_nrows;+ kparams->src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;+ kparams->vtcm_size = (uint32_t) layout.total_bytes;+ kparams->spad_per_thread = (uint32_t) layout.bytes_per_thread;+ kparams->theta_cache_offset = (uint32_t) layout.theta_cache_size_aligned;+ kparams->src0_row_size_aligned = (uint32_t) layout.src0_row_size_aligned;++ if (src0_nrows > 0) {+ kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]);+ kparams->div_ne1 = init_fastdiv_values(dst->ne[1]);+ }+}+static void ggml_hexagon_precompute_fused_mmnx_params(const struct ggml_hexagon_session * sess,const struct ggml_tensor * src0, // W0@@ -4706,56 +4743,82 @@ static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * s}static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const int32_t * op_params = &op->op_params[0];+ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * src1 = op->src[1];+ const struct ggml_tensor * src2 = op->src[2];+ const struct ggml_tensor * dst = op;- // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems)- if (op_params[15] % 32 != 0) {+ if (!ggml_are_same_shape(src0, dst)) {return false;}- int mode = op_params[2];+ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32) {+ return false;+ }- // n_dims == ne0/2, so the rotation spans the full row- if (mode == GGML_ROPE_TYPE_VISION) {- const int n_dims = op_params[1];- if (n_dims != (int) (op->src[0]->ne[0] / 2)) {- return false;- }+ if (src0->ne[0] <= 0) {+ return false;}- if (mode & 1) {++ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];+ if (src0_nrows == 0) {return false;}- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * src1 = op->src[1];- const struct ggml_tensor * src2 = op->src[2];- const struct ggml_tensor * dst = op;+ const int32_t * op_params = &op->op_params[0];+ const int n_dims = op_params[1];+ const int mode = op_params[2];+ const int n_offs = op_params[15];- if (src0->type != GGML_TYPE_F32) {- return false; // FIXME: add support for GGML_TYPE_F16 for src0+ if (n_dims <= 0 || n_dims % 2 != 0) {+ return false;}- if (dst->type != GGML_TYPE_F32) {++ // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems)+ if (n_offs < 0 || (n_offs % 32 != 0) || (n_offs + n_dims > src0->ne[0])) {return false;}- if (src1->type != GGML_TYPE_I32) {++ float freq_base;+ memcpy(&freq_base, op_params + 5, sizeof(float));+ if (freq_base <= 0.0f) {return false;}- if (src2) {- if (src2->type != GGML_TYPE_F32) {++ if (mode != GGML_ROPE_TYPE_NORMAL &&+ mode != GGML_ROPE_TYPE_NEOX &&+ mode != GGML_ROPE_TYPE_MROPE &&+ mode != GGML_ROPE_TYPE_VISION &&+ mode != GGML_ROPE_TYPE_IMROPE) {+ return false;+ }++ const bool is_mrope = (mode & GGML_ROPE_TYPE_MROPE) != 0;++ // n_dims == ne0/2, so the rotation spans the full row+ if (mode == GGML_ROPE_TYPE_VISION) {+ if (n_dims != (int) (src0->ne[0] / 2) || n_offs != 0) {return false;}- int n_dims = op_params[1];- if (src2->ne[0] < (n_dims / 2)) {+ }++ if (is_mrope) {+ const int32_t * sections = op_params + 11;+ if (sections[0] <= 0 && sections[1] <= 0 && sections[2] <= 0) {return false;}}+ const int64_t min_pos_len = (is_mrope || mode == GGML_ROPE_TYPE_VISION) ? src0->ne[2] * 4 : src0->ne[2];+ if (src1->ne[0] < min_pos_len || !ggml_is_contiguous(src1)) {+ return false;+ }+if (src2) {- if (!ggml_is_contiguous(src1) || !ggml_is_contiguous(src2)) {+ if (src2->type != GGML_TYPE_F32 || !ggml_is_contiguous(src2)) {return false;}- } else {- if (!ggml_is_contiguous(src1)) {+ if (src2->ne[0] < (n_dims / 2)) {return false;}}@@ -4768,9 +4831,16 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sessif (src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) {return false;}- return true;- GGML_UNUSED(sess);+ const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows);++ struct htp_rope_vtcm_layout layout;+ htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads);+ if (layout.total_bytes > sess->vtcm_size) {+ return false;+ }++ return true;}static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -5206,6 +5276,11 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggnode.node->src[0], node.node->src[1], node.dst(),(struct htp_set_rows_kernel_params *)node.kernel_params);+ } else if (node.opcode == HTP_OP_ROPE) {+ ggml_hexagon_precompute_rope_params(sess,+ node.node,+ (struct htp_rope_kernel_params *)node.kernel_params+ );}computed_nodes.push_back(std::move(node));}diff --git a/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h b/ggml/src/ggml-hexagon/htp/hvx-sin-cos.hindex c5b9a5d47c..8648af0e5b 100644--- a/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h+++ b/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h@@ -4,87 +4,75 @@#include "hvx-base.h"#include "hvx-floor.h"-static inline HVX_Vector hvx_vec_cos_f32(HVX_Vector x) {- HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f);- HVX_Vector const_half = hvx_vec_splat_f32(0.5f);- HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f);- HVX_Vector const_one = hvx_vec_splat_f32(1.0f);+// Range-reduce x to y in [-pi/2, pi/2] and the quadrant sign (-1)^n.+// Floor/truncate need IEEE bits, so convert qf32 back to sf before them.+static inline void hvx_vec_sincos_reduce_f32(HVX_Vector x, HVX_Vector * y, HVX_Vector * sign) {+ HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f);+ HVX_Vector const_half = hvx_vec_splat_f32(0.5f);+ HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f);+ HVX_Vector const_one = hvx_vec_splat_f32(1.0f);HVX_Vector const_neg_one = hvx_vec_splat_f32(-1.0f);+ HVX_Vector const_one_i = Q6_V_vsplat_R(1);++ HVX_Vector x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(x, const_inv_pi));+ x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(x_over_pi, const_half));- // n = floor(x * (1/pi) + 0.5)- HVX_Vector n_float = hvx_vec_floor_f32(hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(x, const_inv_pi), const_half));+ HVX_Vector n_float = hvx_vec_floor_f32(x_over_pi);+ HVX_Vector n_int = hvx_vec_truncate_f32(n_float);- // y = x - n * pi- HVX_Vector y = hvx_vec_sub_f32_f32(x, hvx_vec_mul_f32_f32(n_float, const_pi));+ HVX_Vector n_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(n_float, const_pi));+ *y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(x, n_pi));- // Sign determination: if n is odd, sign is -1.0f, else 1.0f- // half_n = n * 0.5f- HVX_Vector half_n = hvx_vec_mul_f32_f32(n_float, const_half);- // floor_half_n = floor(half_n)- HVX_Vector floor_half_n = hvx_vec_floor_f32(half_n);- // is_odd = half_n > floor_half_n- HVX_VectorPred is_odd = Q6_Q_vcmp_gt_VsfVsf(half_n, floor_half_n);- // sign = vmux(is_odd, -1.0f, 1.0f)- HVX_Vector sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one);+ HVX_VectorPred is_odd = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(n_int, const_one_i), const_one_i);+ *sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one);+}- // z = y^2- HVX_Vector z = hvx_vec_mul_f32_f32(y, y);+static inline void hvx_vec_sincos_f32(HVX_Vector x, HVX_Vector * vcos, HVX_Vector * vsin) {+ HVX_Vector y;+ HVX_Vector sign;+ hvx_vec_sincos_reduce_f32(x, &y, &sign);++ HVX_Vector z = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, y));- // Chebyshev approximation for cos(y)HVX_Vector c4 = hvx_vec_splat_f32(2.3557242013849433e-05f);HVX_Vector c3 = hvx_vec_splat_f32(-0.0013871428263450528f);HVX_Vector c2 = hvx_vec_splat_f32(0.041665895266688284f);HVX_Vector c1 = hvx_vec_splat_f32(-0.4999999360426369f);HVX_Vector c0 = hvx_vec_splat_f32(0.9999999999071725f);- HVX_Vector cos_y = hvx_vec_add_f32_f32(c3, hvx_vec_mul_f32_f32(z, c4));- cos_y = hvx_vec_add_f32_f32(c2, hvx_vec_mul_f32_f32(z, cos_y));- cos_y = hvx_vec_add_f32_f32(c1, hvx_vec_mul_f32_f32(z, cos_y));- cos_y = hvx_vec_add_f32_f32(c0, hvx_vec_mul_f32_f32(z, cos_y));-- return hvx_vec_mul_f32_f32(cos_y, sign);-}--static inline HVX_Vector hvx_vec_sin_f32(HVX_Vector x) {- HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f);- HVX_Vector const_half = hvx_vec_splat_f32(0.5f);- HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f);- HVX_Vector const_one = hvx_vec_splat_f32(1.0f);- HVX_Vector const_neg_one = hvx_vec_splat_f32(-1.0f);-- // n = floor(x * (1/pi) + 0.5)- HVX_Vector n_float = hvx_vec_floor_f32(hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(x, const_inv_pi), const_half));+ HVX_Vector cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, c4))));+ cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y))));+ cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y))));+ cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y))));- // y = x - n * pi- HVX_Vector y = hvx_vec_sub_f32_f32(x, hvx_vec_mul_f32_f32(n_float, const_pi));-- // Sign determination: if n is odd, sign is -1.0f, else 1.0f- // half_n = n * 0.5f- HVX_Vector half_n = hvx_vec_mul_f32_f32(n_float, const_half);- // floor_half_n = floor(half_n)- HVX_Vector floor_half_n = hvx_vec_floor_f32(half_n);- // is_odd = half_n > floor_half_n- HVX_VectorPred is_odd = Q6_Q_vcmp_gt_VsfVsf(half_n, floor_half_n);- // sign = vmux(is_odd, -1.0f, 1.0f)- HVX_Vector sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one);-- // z = y^2- HVX_Vector z = hvx_vec_mul_f32_f32(y, y);-- // Chebyshev approximation for sin(y)HVX_Vector s4 = hvx_vec_splat_f32(2.642186986152672e-06f);HVX_Vector s3 = hvx_vec_splat_f32(-0.00019825318964070864f);HVX_Vector s2 = hvx_vec_splat_f32(0.00833326283319605f);HVX_Vector s1 = hvx_vec_splat_f32(-0.16666666082087775f);HVX_Vector s0 = hvx_vec_splat_f32(0.999999999915155f);- HVX_Vector sin_y = hvx_vec_add_f32_f32(s3, hvx_vec_mul_f32_f32(z, s4));- sin_y = hvx_vec_add_f32_f32(s2, hvx_vec_mul_f32_f32(z, sin_y));- sin_y = hvx_vec_add_f32_f32(s1, hvx_vec_mul_f32_f32(z, sin_y));- sin_y = hvx_vec_add_f32_f32(s0, hvx_vec_mul_f32_f32(z, sin_y));- sin_y = hvx_vec_mul_f32_f32(y, sin_y);+ HVX_Vector sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, s4))));+ sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y))));+ sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y))));+ sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y))));+ sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, sin_y));++ *vcos = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(cos_y, sign));+ *vsin = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(sin_y, sign));+}- return hvx_vec_mul_f32_f32(sin_y, sign);+static inline HVX_Vector hvx_vec_cos_f32(HVX_Vector x) {+ HVX_Vector vcos;+ HVX_Vector vsin;+ hvx_vec_sincos_f32(x, &vcos, &vsin);+ return vcos;+}++static inline HVX_Vector hvx_vec_sin_f32(HVX_Vector x) {+ HVX_Vector vcos;+ HVX_Vector vsin;+ hvx_vec_sincos_f32(x, &vcos, &vsin);+ return vsin;}#endif /* HVX_SIN_COS_H */diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.cindex 6c68982493..0a4b31ccb1 100644--- a/ggml/src/ggml-hexagon/htp/rope-ops.c+++ b/ggml/src/ggml-hexagon/htp/rope-ops.c@@ -17,8 +17,8 @@#include "ggml-common.h"#include "htp-ctx.h"#include "htp-ops.h"-#include "htp-ops.h"#include "htp-tensor.h"+#include "rope-ops.h"// Redefined the rope type constants as we can't include ggml.h#define HTP_ROPE_TYPE_NORMAL 0@@ -27,9 +27,6 @@#define HTP_ROPE_TYPE_VISION 24#define HTP_ROPE_TYPE_IMROPE 40-#define HTP_ROPE_SPAD_NROWS 16-#define HTP_ROPE_SPAD_BLOCK (HTP_ROPE_SPAD_NROWS/2)-#define htp_rope_preamble \const uint32_t ne00 = src0->ne[0]; \const uint32_t ne01 = src0->ne[1]; \@@ -65,26 +62,27 @@ struct htp_rope_context {float beta_fast;float beta_slow;float theta_scale;+ float theta_scale_32;+ float theta_powers[32];float corr_dims[2];uint32_t src0_nrows_per_thread;- size_t spad_stride;struct htp_ops_context * octx;+ uint8_t * vtcm_base;+ size_t spad_per_thread;+ size_t theta_cache_offset;+size_t src0_row_size;size_t src0_row_stride;size_t dst_row_size;size_t dst_row_stride;size_t src0_row_size_aligned;- size_t dst_row_size_aligned;- size_t theta_cache_offset;uint32_t src0_nrows;struct fastdiv_values div_ne2_ne1;struct fastdiv_values div_ne1;-- uint64_t t_start;};static float rope_yarn_ramp(const float low, const float high, const int i0) {@@ -112,94 +110,80 @@ static inline void rope_yarn_one(float theta, float freq_scale, float * corr_dimmscale_final *= 1.0f + 0.1f * logf(1.0f / freq_scale);}- cache[i0 + 0] = cosf(theta_final) * mscale_final;- cache[i0 + 1] = sinf(theta_final) * mscale_final;+ const uint32_t b = i0 / 64;+ const uint32_t k = (i0 % 64) / 2;+ cache[b * 64 + k] = cosf(theta_final) * mscale_final;+ cache[b * 64 + 32 + k] = sinf(theta_final) * mscale_final;+}++// 32 thetas -> 32 deinterleaved pairs [cos[32] | sin[32]] at cache[i0].+static inline void rope_cache_hvx_32(float * cache, uint32_t i0,+ HVX_Vector v_theta,+ const float * freq_factors,+ HVX_Vector v_freq_scale,+ HVX_Vector v_mscale) {+ if (freq_factors) {+ HVX_Vector v_ff = hvx_vmemu(freq_factors + i0 / 2);+ v_theta = hvx_vec_mul_f32_f32(v_theta, hvx_vec_inverse_f32(v_ff));+ }++ HVX_Vector v_theta_final = hvx_vec_mul_f32_f32(v_theta, v_freq_scale);+ HVX_Vector vcos;+ HVX_Vector vsin;+ hvx_vec_sincos_f32(v_theta_final, &vcos, &vsin);+ vcos = hvx_vec_mul_f32_f32(vcos, v_mscale);+ vsin = hvx_vec_mul_f32_f32(vsin, v_mscale);++ if (((uintptr_t) (cache + i0)) % 128 == 0) {+ hvx_vmem(cache + i0 + 0) = vcos;+ hvx_vmem(cache + i0 + 32) = vsin;+ } else {+ hvx_vec_store_u(cache + i0 + 0, 32 * sizeof(float), vcos);+ hvx_vec_store_u(cache + i0 + 32, 32 * sizeof(float), vsin);+ }}static __attribute__((noinline)) void rope_cache_init(const float theta_base,const float freq_scale,const float * freq_factors,float * corr_dims,- const uint32_t ne0,+ const uint32_t n_cache,const float ext_factor,const float mscale,float * cache,- const float theta_scale) {+ const float theta_scale,+ const float * theta_powers,+ const float theta_scale_32) {// ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py-#if __HVX_ARCH__ >= 79- const bool is_v79_or_newer = true;-#else- const bool is_v79_or_newer = false;-#endif-- if (is_v79_or_newer && ext_factor == 0.0f) {+ if (ext_factor == 0.0f) {// Fast path: fully vectorized// We process 32 pairs (64 elements) per iteration.- const uint32_t n_blocks = ne0 / 64;-- // Initialize theta scale powers: [1.0f, theta_scale, theta_scale^2, ..., theta_scale^31]- float __attribute__((aligned(128))) theta_powers[32];- theta_powers[0] = 1.0f;- for (int j = 1; j < 32; j++) {- theta_powers[j] = theta_powers[j - 1] * theta_scale;- }- HVX_Vector v_theta_powers = hvx_vmem(theta_powers);+ const uint32_t n_blocks = n_cache / 64;+ HVX_Vector v_theta_powers = hvx_vmemu(theta_powers);HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale);HVX_Vector v_mscale = hvx_vec_splat_f32(mscale);- // Base theta starts at theta_basefloat theta_block = theta_base;- // The scale factor for the next block is theta_scale^32- float theta_scale_32 = 1.0f;- for (int j = 0; j < 32; j++) {- theta_scale_32 *= theta_scale;- }for (uint32_t b = 0; b < n_blocks; b++) {uint32_t i0 = b * 64;HVX_Vector v_theta_base = hvx_vec_splat_f32(theta_block);HVX_Vector v_theta = hvx_vec_mul_f32_f32(v_theta_base, v_theta_powers);-- if (freq_factors) {- // Load 32 elements of freq_factors- HVX_Vector v_ff = hvx_vmemu(freq_factors + i0 / 2);- HVX_Vector v_inv_ff = hvx_vec_inverse_f32(v_ff);- v_theta = hvx_vec_mul_f32_f32(v_theta, v_inv_ff);- }-- HVX_Vector v_theta_final = hvx_vec_mul_f32_f32(v_theta, v_freq_scale);-- HVX_Vector vcos = hvx_vec_cos_f32(v_theta_final);- HVX_Vector vsin = hvx_vec_sin_f32(v_theta_final);-- vcos = hvx_vec_mul_f32_f32(vcos, v_mscale);- vsin = hvx_vec_mul_f32_f32(vsin, v_mscale);-- HVX_VectorPair vstore = Q6_W_vshuff_VVR(vsin, vcos, -4);-- if (((uintptr_t)cache) % 128 == 0) {- hvx_vmem(cache + i0 + 0) = Q6_V_lo_W(vstore);- hvx_vmem(cache + i0 + 32) = Q6_V_hi_W(vstore);- } else {- hvx_vec_store_u(cache + i0 + 0, 32 * sizeof(float), Q6_V_lo_W(vstore));- hvx_vec_store_u(cache + i0 + 32, 32 * sizeof(float), Q6_V_hi_W(vstore));- }-+ rope_cache_hvx_32(cache, i0, v_theta, freq_factors, v_freq_scale, v_mscale);theta_block *= theta_scale_32;}// Leftoversfloat theta = theta_block;- for (uint32_t i0 = n_blocks * 64; i0 < ne0; i0 += 2) {+ for (uint32_t i0 = n_blocks * 64; i0 < n_cache; i0 += 2) {const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f;rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache);theta *= theta_scale;}} else {- // Fallback to original scalar loopfloat theta = theta_base;- for (uint32_t i0 = 0; i0 < ne0; i0 += 2) {+ for (uint32_t i0 = 0; i0 < n_cache; i0 += 2) {const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f;rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache);theta *= theta_scale;@@ -207,6 +191,72 @@ static __attribute__((noinline)) void rope_cache_init(const float theta_base,}}+static inline float mrope_pick_theta(float theta_t, float theta_h, float theta_w, float theta_e,+ int sector, const int32_t sections[4], int sec_w, int sec_e,+ bool is_imrope) {+ if (is_imrope) {+ if (sector % 3 == 0 && sector < 3 * sections[0]) { return theta_t; }+ else if (sector % 3 == 1 && sector < 3 * sections[1]) { return theta_h; }+ else if (sector % 3 == 2 && sector < 3 * sections[2]) { return theta_w; }+ else { return theta_e; }+ }+ if (sector < sections[0]) { return theta_t; }+ else if (sector < sec_w) { return theta_h; }+ else if (sector < sec_e) { return theta_w; }+ else { return theta_e; }+}++// lane j is 1 when (j % 3) == rem+static const float __attribute__((aligned(128))) mrope_mod3_eq0[32] = {+ 1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0+};+static const float __attribute__((aligned(128))) mrope_mod3_eq1[32] = {+ 0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1+};+static const float __attribute__((aligned(128))) mrope_mod3_eq2[32] = {+ 0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0+};++static const float __attribute__((aligned(128))) mrope_k_ramp[32] = {+ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,+ 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31+};++static inline HVX_VectorPred mrope_mask_eq1(const float * m) {+ return Q6_Q_vcmp_gt_VsfVsf(hvx_vmemu(m), Q6_V_vzero());+}++// IMROPE without wrap: theta[k] = pos[k % 3] * scale^k+static inline HVX_Vector mrope_thetas_imrope_mod3(float pos_t, float pos_h, float pos_w,+ uint32_t k0, HVX_Vector v_powers, float scale_block) {+ const int r = (int) (k0 % 3);+ const float * mt = (r == 0) ? mrope_mod3_eq0 : (r == 1) ? mrope_mod3_eq2 : mrope_mod3_eq1;+ const float * mh = (r == 0) ? mrope_mod3_eq1 : (r == 1) ? mrope_mod3_eq0 : mrope_mod3_eq2;++ HVX_Vector v = hvx_vec_splat_f32(pos_w);+ v = Q6_V_vmux_QVV(mrope_mask_eq1(mh), hvx_vec_splat_f32(pos_h), v);+ v = Q6_V_vmux_QVV(mrope_mask_eq1(mt), hvx_vec_splat_f32(pos_t), v);+ v = hvx_vec_mul_f32_f32(v, v_powers);+ return hvx_vec_mul_f32_f32(v, hvx_vec_splat_f32(scale_block));+}++// Contiguous MROPE without wrap: theta[k] = pos[section(k)] * scale^k+static inline HVX_Vector mrope_thetas_contig(float pos_t, float pos_h, float pos_w, float pos_e,+ uint32_t k0, int s0, int sec_w, int sec_e,+ HVX_Vector v_powers, float scale_block) {+ HVX_Vector v_k = hvx_vec_add_f32_f32(hvx_vec_splat_f32((float) k0), hvx_vmemu(mrope_k_ramp));+ HVX_VectorPred lt_s0 = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) s0), v_k);+ HVX_VectorPred lt_sw = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) sec_w), v_k);+ HVX_VectorPred lt_se = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) sec_e), v_k);++ HVX_Vector v = hvx_vec_splat_f32(pos_e);+ v = Q6_V_vmux_QVV(lt_se, hvx_vec_splat_f32(pos_w), v);+ v = Q6_V_vmux_QVV(lt_sw, hvx_vec_splat_f32(pos_h), v);+ v = Q6_V_vmux_QVV(lt_s0, hvx_vec_splat_f32(pos_t), v);+ v = hvx_vec_mul_f32_f32(v, v_powers);+ return hvx_vec_mul_f32_f32(v, hvx_vec_splat_f32(scale_block));+}+// pos_t/h/w/e: the four position ids for this sequence step (t=time, h=height, w=width, e=extra).// sections[4]: number of head dims assigned to each position component.static __attribute__((noinline)) void mrope_cache_init(const float pos_t,@@ -219,23 +269,71 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t,const float freq_scale,const float * freq_factors,float * corr_dims,- const uint32_t ne0,+ const uint32_t n_cache,const float ext_factor,const float mscale,float * cache,- const float theta_scale) {+ const float theta_scale,+ const float * theta_powers,+ const float theta_scale_32) {const int sect_dims = sections[0] + sections[1] + sections[2] + sections[3];const int sec_w = sections[0] + sections[1];const int sec_e = sec_w + sections[2];+ const uint32_t n_pairs = n_cache / 2;++ const bool no_wrap = (sect_dims > 0) && (n_pairs <= (uint32_t) sect_dims);+ const bool imrope_mod3 = is_imrope && !indep_sects && no_wrap+ && sections[0] > 0 && sections[1] > 0 && sections[2] > 0+ && n_pairs <= (uint32_t) (3 * sections[0])+ && n_pairs <= (uint32_t) (3 * sections[1])+ && n_pairs <= (uint32_t) (3 * sections[2]);+ const bool contig = !is_imrope && !indep_sects && no_wrap;++ if (ext_factor == 0.0f && (imrope_mod3 || contig)) {+ HVX_Vector v_powers = hvx_vmemu(theta_powers);+ HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale);+ HVX_Vector v_mscale = hvx_vec_splat_f32(mscale);+ float scale_block = 1.0f;+ const uint32_t n_blocks = n_cache / 64;++ for (uint32_t b = 0; b < n_blocks; b++) {+ const uint32_t i0 = b * 64;+ const uint32_t k0 = b * 32;+ HVX_Vector v_theta = imrope_mod3+ ? mrope_thetas_imrope_mod3(pos_t, pos_h, pos_w, k0, v_powers, scale_block)+ : mrope_thetas_contig(pos_t, pos_h, pos_w, pos_e, k0, sections[0], sec_w, sec_e,+ v_powers, scale_block);+ rope_cache_hvx_32(cache, i0, v_theta, freq_factors, v_freq_scale, v_mscale);+ scale_block *= theta_scale_32;+ }++ float theta_k = scale_block;+ for (uint32_t k = n_blocks * 32; k < n_pairs; k++) {+ const uint32_t i0 = 2 * k;+ const float pos = mrope_pick_theta(pos_t, pos_h, pos_w, pos_e,+ (int) k, sections, sec_w, sec_e, is_imrope);+ const float ff = freq_factors ? freq_factors[k] : 1.0f;+ rope_yarn_one(pos * theta_k / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache);+ theta_k *= theta_scale;+ }+ return;+ }float theta_t = pos_t;float theta_h = pos_h;float theta_w = pos_w;float theta_e = pos_e;- for (uint32_t i0 = 0; i0 < ne0; i0 += 2) {- const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f;- const int sector = (i0 / 2) % sect_dims;+ const bool use_hvx = (ext_factor == 0.0f);+ float __attribute__((aligned(128))) thetas[32];+ uint32_t n_thetas = 0;+ uint32_t block_i0 = 0;++ HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale);+ HVX_Vector v_mscale = hvx_vec_splat_f32(mscale);++ for (uint32_t i0 = 0; i0 < n_cache; i0 += 2) {+ const int sector = (i0 / 2) % sect_dims;if (indep_sects) {// Reset theta when crossing into a new section.@@ -245,28 +343,34 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t,else if (sector == sec_e) { theta_e = pos_e; }}- float theta;- if (is_imrope) {- // Interleaved: sector mod 3 selects component- if (sector % 3 == 0 && sector < 3 * sections[0]) { theta = theta_t; }- else if (sector % 3 == 1 && sector < 3 * sections[1]) { theta = theta_h; }- else if (sector % 3 == 2 && sector < 3 * sections[2]) { theta = theta_w; }- else { theta = theta_e; }+ const float theta = mrope_pick_theta(theta_t, theta_h, theta_w, theta_e,+ sector, sections, sec_w, sec_e, is_imrope);++ if (use_hvx) {+ if (n_thetas == 0) {+ block_i0 = i0;+ }+ thetas[n_thetas++] = theta;+ if (n_thetas == 32) {+ rope_cache_hvx_32(cache, block_i0, hvx_vmemu(thetas), freq_factors, v_freq_scale, v_mscale);+ n_thetas = 0;+ }} else {- // Contiguous sections- if (sector < sections[0]) { theta = theta_t; }- else if (sector < sec_w) { theta = theta_h; }- else if (sector < sec_e) { theta = theta_w; }- else { theta = theta_e; }+ const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f;+ rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache);}- rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache);-theta_t *= theta_scale;theta_h *= theta_scale;theta_w *= theta_scale;theta_e *= theta_scale;}++ for (uint32_t k = 0; k < n_thetas; k++) {+ const uint32_t i0 = block_i0 + 2 * k;+ const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f;+ rope_yarn_one(thetas[k] / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache);+ }}#define M_PI 3.1415926535897932384626433@@ -283,52 +387,54 @@ static void rope_corr_dims(int n_dims,dims[1] = MIN(n_dims - 1, end);}+static inline void hvx_rope_neox_mul(HVX_Vector v0, HVX_Vector v1, HVX_Vector vcos, HVX_Vector vsin,+ HVX_Vector * o0, HVX_Vector * o1) {+ HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, vcos);+ HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, vsin);+ HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, vcos);+ HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, vsin);+ *o0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s));+ *o1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c));+}++// theta_cache full 32-pair blocks are deinterleaved [cos | sin].static inline void hvx_rope_neox_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) {const uint32_t he = ne / 2;const uint32_t nvec = he / 32;const uint32_t nloe = he % 32;- for (uint32_t i = 0; i < nvec; i++) {- HVX_Vector v0 = ((const HVX_Vector *) src0)[i];- HVX_Vector v1 = hvx_vmemu(src0 + he + i * 32);-- HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[i * 2 + 0];- HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[i * 2 + 1];-- HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4);-- HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_lo_W(vcos_sin));- HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_hi_W(vcos_sin));- HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_lo_W(vcos_sin));- HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_hi_W(vcos_sin));-- HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s);- HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c);-- ((HVX_Vector *) dst)[i] = Q6_Vsf_equals_Vqf32(v4);- hvx_vmemu(dst + he + i * 32) = Q6_Vsf_equals_Vqf32(v5);+ if (nloe == 0) {+ const HVX_Vector * vs = (const HVX_Vector *) src0;+ const HVX_Vector * vt = (const HVX_Vector *) theta_cache;+ HVX_Vector * vd = (HVX_Vector *) dst;+ for (uint32_t i = 0; i < nvec; i++) {+ HVX_Vector o0, o1;+ hvx_rope_neox_mul(vs[i], vs[nvec + i], vt[i * 2 + 0], vt[i * 2 + 1], &o0, &o1);+ vd[i] = o0;+ vd[nvec + i] = o1;+ }+ return;}- if (nloe > 0) {- HVX_Vector v0 = hvx_vmemu(src0 + nvec * 32);- HVX_Vector v1 = hvx_vmemu(src0 + he + nvec * 32);-- HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[nvec * 2 + 0];- HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[nvec * 2 + 1];-- HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4);-- HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_lo_W(vcos_sin));- HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_hi_W(vcos_sin));- HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_lo_W(vcos_sin));- HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_hi_W(vcos_sin));-- HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s);- HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c);-- hvx_vec_store_u(dst + nvec * 32, nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v4));- hvx_vec_store_u(dst + he + nvec * 32, nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v5));+ for (uint32_t i = 0; i < nvec; i++) {+ HVX_Vector o0, o1;+ hvx_rope_neox_mul(((const HVX_Vector *) src0)[i],+ hvx_vmemu(src0 + he + i * 32),+ ((const HVX_Vector *) theta_cache)[i * 2 + 0],+ ((const HVX_Vector *) theta_cache)[i * 2 + 1],+ &o0, &o1);+ ((HVX_Vector *) dst)[i] = o0;+ hvx_vmemu(dst + he + i * 32) = o1;}++ HVX_Vector v0 = hvx_vmemu(src0 + nvec * 32);+ HVX_Vector v1 = hvx_vmemu(src0 + he + nvec * 32);+ HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64);+ HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32);+ HVX_Vector o0, o1;+ hvx_rope_neox_mul(v0, v1, vcos, vsin, &o0, &o1);+ hvx_vec_store_u(dst + nvec * 32, nloe * sizeof(float), o0);+ hvx_vec_store_u(dst + he + nvec * 32, nloe * sizeof(float), o1);}static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) {@@ -339,16 +445,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrictHVX_Vector v0 = ((const HVX_Vector *) src0)[i * 2 + 0];HVX_Vector v1 = ((const HVX_Vector *) src0)[i * 2 + 1];- HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[i * 2 + 0];- HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[i * 2 + 1];+ HVX_Vector vcos = ((const HVX_Vector *) theta_cache)[i * 2 + 0];+ HVX_Vector vsin = ((const HVX_Vector *) theta_cache)[i * 2 + 1];- HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4);- HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4);+ HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4);- HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin));- HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin));- HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin));- HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin));+ HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos);+ HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin);+ HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos);+ HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin);HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s);HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c);@@ -362,15 +467,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrictif (nloe > 0) {if (nloe <= 32) {HVX_Vector v0 = hvx_vmemu(src0 + nvec * 64);- HVX_Vector v2 = hvx_vmemu(theta_cache + nvec * 64);+ HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64);+ HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32);- HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(Q6_V_vzero(), v0, -4);- HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(Q6_V_vzero(), v2, -4);+ HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(Q6_V_vzero(), v0, -4);- HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin));- HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin));- HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin));- HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin));+ HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos);+ HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin);+ HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos);+ HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin);HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s);HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c);@@ -382,16 +487,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrictHVX_Vector v0 = hvx_vmemu(src0 + nvec * 64);HVX_Vector v1 = hvx_vmemu(src0 + nvec * 64 + 32);- HVX_Vector v2 = hvx_vmemu(theta_cache + nvec * 64);- HVX_Vector v3 = hvx_vmemu(theta_cache + nvec * 64 + 32);+ HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64);+ HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32);- HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4);- HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4);+ HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4);- HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin));- HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin));- HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin));- HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin));+ HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos);+ HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin);+ HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos);+ HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin);HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s);HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c);@@ -404,54 +508,23 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict}}-static void inline rope_basic_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src,- uint32_t nr, uint32_t ne0, const float * restrict theta_cache) {- const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op)- #pragma unroll(4)- for (uint32_t i = 0; i < nr; i++) {- float * d = (float *) (dst + i * rctx->dst_row_size_aligned);- float * s = (float *) (src + i * rctx->src0_row_size_aligned);-- hvx_rope_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache);-- // fill the remain channels with data from src tensor- if (n_offs > 0) {- hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs);- }- if (n_offs + rctx->n_dims < ne0) {- hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims);- }- }-}--static void inline rope_neox_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src,- uint32_t nr, uint32_t ne0, const float * restrict theta_cache) {- const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op)+static void inline rope_basic_f32_inplace(struct htp_rope_context * rctx, uint8_t * src,+ uint32_t nr, const float * restrict theta_cache) {+ const uint32_t n_offs = rctx->n_offs;#pragma unroll(4)for (uint32_t i = 0; i < nr; i++) {- float * d = (float *) (dst + i * rctx->dst_row_size_aligned);float * s = (float *) (src + i * rctx->src0_row_size_aligned);-- hvx_rope_neox_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache);-- // fill the remain channels with data from src tensor- if (n_offs > 0) {- hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs);- }- if (n_offs + rctx->n_dims < ne0) {- hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims);- }+ hvx_rope_f32_aa(s + n_offs, s + n_offs, rctx->n_dims, theta_cache);}}-static void inline rope_vision_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src,- uint32_t nr, uint32_t ne0, const float * restrict theta_cache) {+static void inline rope_neox_f32_inplace(struct htp_rope_context * rctx, uint8_t * src,+ uint32_t nr, uint32_t ne, const float * restrict theta_cache) {+ const uint32_t n_offs = rctx->n_offs;#pragma unroll(4)for (uint32_t i = 0; i < nr; i++) {- float * d = (float *) (dst + i * rctx->dst_row_size_aligned);float * s = (float *) (src + i * rctx->src0_row_size_aligned);-- hvx_rope_neox_f32_aa(d, s, ne0, theta_cache);+ hvx_rope_neox_f32_aa(s + n_offs, s + n_offs, ne, theta_cache);}}@@ -477,20 +550,18 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {return;}- uint64_t tt = HAP_perf_get_qtimer_count();-const int32_t mode = rctx->mode;// MROPE, IMROPE and VISION use NEOX-style pairing for the rotationconst bool is_neox = (mode & HTP_ROPE_TYPE_NEOX) || (mode & HTP_ROPE_TYPE_MROPE);const bool is_vision = (mode == HTP_ROPE_TYPE_VISION);// VTCM setup- uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);+ uint8_t * src0_spad_base = rctx->vtcm_base + (ith * rctx->spad_per_thread);float * theta_cache = (float *) (src0_spad_base);src0_spad_base = src0_spad_base + rctx->theta_cache_offset;- uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread);dma_queue * dma_queue = octx->ctx->dma[ith];+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];const int32_t * pos = (const int32_t *) src1->data;const float * freq_factors = src2 ? (const float *) src2->data : NULL;@@ -501,6 +572,7 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {uint32_t ir = src0_start_row;uint32_t prev_i2 = (uint32_t) -1;+ uint32_t cur_slot = 0;for (uint32_t i3 = i3_start; i3 < ne3; i3++) { // batchconst uint32_t i2_init = (i3 == i3_start) ? i2_start : 0;@@ -513,35 +585,30 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {const uint32_t nrows = MIN(src0_end_row - ir, ne1 - i1);// Depth before prefetch- uint32_t dma_depth = dma_queue_depth(dma_queue);-- // FARF(HIGH, "rope-block %u: ir %u n-rows %u dma-depth %u : usec %u", ith, ir, nrows, dma_depth,- // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start));+ const uint32_t dma_depth = dma_queue_depth(dma_queue);- // Prefetch loop- for (uint32_t pnr = 0, pr = 0; pr < nrows && pr < HTP_ROPE_SPAD_NROWS; pr += pnr) {- pnr = MIN(nrows - pr, HTP_ROPE_SPAD_BLOCK);+ // Prefetch up to 2 blocks+ const uint32_t p_nrows = MIN(nrows, 2 * HTP_ROPE_SPAD_BLOCK);+ for (uint32_t pr = 0; pr < p_nrows; pr += HTP_ROPE_SPAD_BLOCK) {+ const uint32_t pnr = MIN(nrows - pr, HTP_ROPE_SPAD_BLOCK);+ const uint32_t slot = (cur_slot + pr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS;+ uint8_t * spad_slot = rope_spad_slot(src0_spad_base, slot, rctx->src0_row_size_aligned);+ const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + (i1 + pr) * nb01;- uint32_t pi1 = i1 + pr;- uint32_t pir = ir + pr;+ // Dummy DMA transaction for sequencing (interleaving wr, rd, wr, rd, ...)+ dma_queue_push(dma_queue, dma_make_ptr((void *) dst->data, spad_slot), 0, 0, 0, 0);- // Dummy DMA transaction for sequencing (interleaving dst,src,dst,...)- dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr((void *) dst->data, dst_spad_base + pr * rctx->dst_row_size_aligned), 0, 0, 0);-- const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01;- uint8_t * src_spad = src0_spad_base + pr * rctx->src0_row_size_aligned;-- // Copy only the row payload while striding the DDR source- dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr),+ dma_queue_push(dma_queue, dma_make_ptr(spad_slot, src_addr),rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr);-- // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr);}// Update theta cacheif (i2 != prev_i2) {prev_i2 = i2;+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, i2);+ // VISION rotates the full row; other modes only rotate n_dims.+ const uint32_t n_cache = is_vision ? ne0 : (uint32_t) rctx->n_dims;const bool is_mrope = (rctx->mode & HTP_ROPE_TYPE_MROPE) != 0;if (is_mrope) {// src1 holds four position arrays stacked along ne0:@@ -554,66 +621,71 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {(float) pos[i2 + ne2 * 3],rctx->sections, is_imrope, is_vision,rctx->freq_scale, freq_factors, rctx->corr_dims,- ne0, rctx->ext_factor, rctx->attn_factor,- theta_cache, rctx->theta_scale);+ n_cache, rctx->ext_factor, rctx->attn_factor,+ theta_cache, rctx->theta_scale, rctx->theta_powers, rctx->theta_scale_32);} else {rope_cache_init(pos[i2], rctx->freq_scale, freq_factors, rctx->corr_dims,- ne0, rctx->ext_factor, rctx->attn_factor,- theta_cache, rctx->theta_scale);+ n_cache, rctx->ext_factor, rctx->attn_factor,+ theta_cache, rctx->theta_scale, rctx->theta_powers, rctx->theta_scale_32);}+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, i2);}// Skip output DMA transactions from prev block (if any)- // No need to wait for those here since we're explicitly waiting for the latest prefecthes below.- for (uint32_t d=0; d < dma_depth; d++) { dma_queue_pop_nowait(dma_queue); }+ for (uint32_t d = 0; d < dma_depth; d++) { dma_queue_pop_nowait(dma_queue); }// Compute loop- for (uint32_t cnr = 0, cr = 0; cr < nrows; cr += cnr, ir += cnr, i1 += cnr) {- // Number of rows to compute- cnr = MIN(nrows - cr, HTP_ROPE_SPAD_BLOCK);+ const uint32_t ne = is_vision ? ne0 : rctx->n_dims;+ const uint32_t base_i1 = i1;+ const uint32_t base_ir = ir;- uint8_t * dst_spad = (uint8_t *) dma_queue_pop(dma_queue).src;- uint8_t * src_spad = (uint8_t *) dma_queue_pop(dma_queue).dst;+ for (uint32_t cnr = 0, cr = 0; cr < nrows; cr += cnr) {+ cnr = MIN(nrows - cr, HTP_ROPE_SPAD_BLOCK);+ const uint32_t slot = (cur_slot + cr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS;+ const uint32_t cur_ir = base_ir + cr;+ const uint32_t cur_i1 = base_i1 + cr;- // FARF(HIGH, "rope-compute %u: ir %u i1 %u i2 %u i3 %u src-spad %p cnr %u : usec %u", ith, ir, i1, i2, i3, src_spad, cnr,- // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start));+ dma_queue_pop(dma_queue);+ uint8_t * cur_spad = (uint8_t *) dma_queue_pop(dma_queue).dst;- if (is_vision) {- rope_vision_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache);- } else if (is_neox) {- rope_neox_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache);+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur_ir);+ if (is_neox || is_vision) {+ rope_neox_f32_inplace(rctx, cur_spad, cnr, ne, theta_cache);} else {- rope_basic_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache);+ rope_basic_f32_inplace(rctx, cur_spad, cnr, theta_cache);}+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur_ir);- uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + i1 * nb1;-- // Write only the row payload while striding the DDR dst- dma_queue_push(dma_queue, dma_make_ptr(dst_addr, dst_spad),- rctx->dst_row_stride, rctx->dst_row_size_aligned, rctx->dst_row_size, cnr);+ uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + cur_i1 * nb1;+ dma_queue_push(dma_queue, dma_make_ptr(dst_addr, cur_spad),+ rctx->dst_row_stride, rctx->src0_row_size_aligned, rctx->dst_row_size, cnr);- // Prefetch more rows (if any)- if ((cr + HTP_ROPE_SPAD_NROWS) < nrows) {- uint32_t pnr = MIN(nrows - (cr + HTP_ROPE_SPAD_NROWS), HTP_ROPE_SPAD_BLOCK);- uint32_t pi1 = i1 + HTP_ROPE_SPAD_NROWS;- uint32_t pir = ir + HTP_ROPE_SPAD_NROWS;+ // Prefetch 2 blocks ahead into the slot just freed+ if ((cr + 2 * HTP_ROPE_SPAD_BLOCK) < nrows) {+ const uint32_t p_cr = cr + 2 * HTP_ROPE_SPAD_BLOCK;+ const uint32_t pnr = MIN(nrows - p_cr, HTP_ROPE_SPAD_BLOCK);+ const uint32_t p_slot = (cur_slot + p_cr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS;+ uint8_t * p_spad = rope_spad_slot(src0_spad_base, p_slot, rctx->src0_row_size_aligned);+ const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + (base_i1 + p_cr) * nb01;- const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01;- dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr),+ dma_queue_push(dma_queue, dma_make_ptr(p_spad, src_addr),rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr);-- // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr);}}++ const uint32_t n_chunks = (nrows + HTP_ROPE_SPAD_BLOCK - 1) / HTP_ROPE_SPAD_BLOCK;+ cur_slot = (cur_slot + n_chunks) % HTP_ROPE_SPAD_NSLOTS;++ ir += nrows;+ i1 += nrows;}}}done:dma_queue_flush(dma_queue);- tt = HAP_perf_get_qtimer_count() - tt;- FARF(HIGH, "rope-f32: %d/%d: (%u:%u) usec %u\n", ith, nth, src0_start_row, src0_end_row, (unsigned) HAP_perf_qtimer_count_to_us(tt));+ FARF(HIGH, "rope-f32: %d/%d: (%u:%u)\n", ith, nth, src0_start_row, src0_end_row);}static int execute_op_rope_f32(struct htp_ops_context * octx) {@@ -624,8 +696,6 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {const struct htp_tensor * src2 = octx->src[2];const struct htp_tensor * dst = octx->dst;- const char * op_type = "rope-f32";-switch (octx->op) {case HTP_OP_ROPE:break;@@ -635,48 +705,23 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {return HTP_STATUS_NO_SUPPORT;}- const uint32_t ne0 = dst->ne[0];- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];- const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);+ const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params;+ assert(kparams->n_threads > 0);+ assert(octx->ctx->vtcm_size >= kparams->vtcm_size);+ const uint32_t ne0 = dst->ne[0];const size_t src0_row_size = src0->ne[0] * sizeof(float);const size_t src0_row_stride = src0->nb[1];const size_t dst_row_size = dst->ne[0] * sizeof(float);const size_t dst_row_stride = dst->nb[1];- // Aligned row sizes for VTCM- const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);- const size_t dst_row_size_aligned = hex_round_up(dst_row_stride, VLEN);- const size_t theta_cache_size_aligned = hex_round_up(src0->ne[0] * sizeof(float), 256);-- // Calculate spad sizes per thread- size_t src0_spad_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned;- size_t dst_spad_per_thread = HTP_ROPE_SPAD_NROWS * dst_row_size_aligned;- size_t spad_per_thread = src0_spad_per_thread + dst_spad_per_thread;-- // Check if we fit in VTCM- size_t total_vtcm_needed = spad_per_thread * n_threads;- if (octx->ctx->vtcm_size < total_vtcm_needed) {- FARF(ERROR, "%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, total_vtcm_needed);- return HTP_STATUS_VTCM_TOO_SMALL;- }-- octx->src0_spad.size_per_thread = src0_spad_per_thread;- octx->dst_spad.size_per_thread = dst_spad_per_thread;- octx->src0_spad.size = n_threads * src0_spad_per_thread;- octx->dst_spad.size = n_threads * dst_spad_per_thread;- octx->src1_spad.size = 0;-- octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL;- octx->src1_spad.data = NULL; octx->src1_spad.src = NULL;- octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->dst_spad.src = NULL;-struct htp_rope_context rctx;memset(&rctx, 0, sizeof(struct htp_rope_context));- rctx.t_start = HAP_perf_get_qtimer_count();-- rctx.octx = octx;+ rctx.octx = octx;+ rctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base;+ rctx.spad_per_thread = kparams->spad_per_thread;+ rctx.theta_cache_offset = kparams->theta_cache_offset;const int32_t * op_params = &octx->op_params[0];rctx.n_dims = ((const int32_t *) op_params)[1];@@ -693,31 +738,29 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {memcpy(&rctx.sections, (int32_t *) op_params + 11, sizeof(int) * 4);rctx.theta_scale = powf(rctx.freq_base, -2.0f / rctx.n_dims);+ rctx.theta_powers[0] = 1.0f;+ for (int j = 1; j < 32; j++) {+ rctx.theta_powers[j] = rctx.theta_powers[j - 1] * rctx.theta_scale;+ }+ rctx.theta_scale_32 = rctx.theta_powers[31] * rctx.theta_scale;rope_corr_dims(rctx.n_dims, rctx.n_ctx_orig, rctx.freq_base, rctx.beta_fast, rctx.beta_slow, rctx.corr_dims);- rctx.src0_row_size = src0_row_size;- rctx.src0_row_stride = src0_row_stride;- rctx.dst_row_size = dst_row_size;- rctx.dst_row_stride = dst_row_stride;- rctx.src0_row_size_aligned = src0_row_size_aligned;- rctx.dst_row_size_aligned = dst_row_size_aligned;- rctx.theta_cache_offset = theta_cache_size_aligned;+ rctx.src0_row_size = src0_row_size;+ rctx.src0_row_stride = src0_row_stride;+ rctx.dst_row_size = dst_row_size;+ rctx.dst_row_stride = dst_row_stride;+ rctx.src0_row_size_aligned = kparams->src0_row_size_aligned;- rctx.src0_nrows = src0_nrows;- rctx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;-- if (src0_nrows > 0) {- rctx.div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]);- rctx.div_ne1 = init_fastdiv_values(dst->ne[1]);- }+ rctx.src0_nrows = kparams->src0_nrows;+ rctx.src0_nrows_per_thread = kparams->src0_nrows_per_thread;+ rctx.div_ne2_ne1 = kparams->div_ne2_ne1;+ rctx.div_ne1 = kparams->div_ne1;FARF(HIGH, "rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\n", rctx.src0_nrows, rctx.n_dims, ne0,rctx.ext_factor, rctx.theta_scale, rctx.attn_factor);- if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {- worker_pool_run_func(octx->ctx->worker_pool, rope_job_f32, &rctx, n_threads);- }+ work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, kparams->n_threads);return err;}diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.h b/ggml/src/ggml-hexagon/htp/rope-ops.hnew file mode 100644index 0000000000..476653d05d--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/rope-ops.h@@ -0,0 +1,56 @@+#ifndef HTP_ROPE_OPS_H+#define HTP_ROPE_OPS_H++#include "hex-common.h"+#include "hex-fastdiv.h"++#define HTP_ROPE_SPAD_BLOCK 8+#define HTP_ROPE_SPAD_NSLOTS 4+#define HTP_ROPE_SPAD_NROWS (HTP_ROPE_SPAD_BLOCK * HTP_ROPE_SPAD_NSLOTS)++struct htp_rope_kernel_params {+ uint32_t n_threads;+ uint32_t src0_nrows;+ uint32_t src0_nrows_per_thread;+ uint32_t vtcm_size;+ uint32_t spad_per_thread;+ uint32_t theta_cache_offset;+ uint32_t src0_row_size_aligned;++ struct fastdiv_values div_ne2_ne1;+ struct fastdiv_values div_ne1;+};++#if defined(__cplusplus)+static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_params is too large for kernel_params blob");+#else+_Static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_params is too large for kernel_params blob");+#endif++struct htp_rope_vtcm_layout {+ size_t total_bytes;+ size_t bytes_per_thread;+ size_t theta_cache_size_aligned;+ size_t src0_row_size_aligned;+};++static inline void htp_rope_vtcm_layout_build(+ struct htp_rope_vtcm_layout * layout,+ uint32_t ne00,+ uint32_t n_threads+) {+ const size_t src0_row_size = ne00 * sizeof(float);+ const size_t src0_row_size_aligned = hex_round_up((uint32_t) src0_row_size, 128);+ const size_t theta_cache_size_aligned = hex_round_up((uint32_t) src0_row_size, 256);++ layout->src0_row_size_aligned = src0_row_size_aligned;+ layout->theta_cache_size_aligned = theta_cache_size_aligned;+ layout->bytes_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned;+ layout->total_bytes = layout->bytes_per_thread * n_threads;+}++static inline uint8_t * rope_spad_slot(uint8_t * base, uint32_t slot, size_t row_size_aligned) {+ return base + (slot * HTP_ROPE_SPAD_BLOCK) * row_size_aligned;+}++#endif // HTP_ROPE_OPS_Hdiff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 8030186fb4..2deb90f6ab 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -10282,6 +10282,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 32, GGML_ROPE_TYPE_NEOX, 512, 1.4245f, 0.7465f, 1.4245f, false, 0, true, true, 32));}+ // Real-model RoPE: F32 forward, packed Q, 512-token prefill.+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 0.8B+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 16, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 4B+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global+for (int v : { 0, 1, 2, 3 }) {for (int dim : { 0, 1, 2, 3, }) {test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {11, 12, 13, 14}, 7, dim, v));@@ -11178,6 +11184,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {}}+ // Real-model RoPE: F32 forward, packed Q, 512-token prefill.+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 0.8B+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 16, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 4B+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding+ test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global+std::vector<std::array<int64_t, 4>> reduce_rows_cases = {{ 8192, 1, 1, 1 },{ 8192, 8192, 1, 1 },