LLM inference in C/C++
C++ ggml
hexagon: add VISION RoPE support (#25216)
Aparna M Pf2d1c2f3984c
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex 3d9fc9b56d..0c78879cbb 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -3105,8 +3105,12 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sessint mode = op_params[2];+ // n_dims == ne0/2, so the rotation spans the full rowif (mode == GGML_ROPE_TYPE_VISION) {- return false;+ const int n_dims = op_params[1];+ if (n_dims != (int) (op->src[0]->ne[0] / 2)) {+ return false;+ }}if (mode & 1) {return false;@@ -3137,16 +3141,23 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess}if (src2) {- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(src2) ||- !ggml_is_contiguous(dst)) {+ if (!ggml_is_contiguous(src1) || !ggml_is_contiguous(src2)) {return false;}} else {- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {+ if (!ggml_is_contiguous(src1)) {return false;}}+ // src0/dst elements within a row must be contiguous (nb[0] == sizeof(float)).+ // nb[1] may exceed ne[0]*sizeof(float) when the tensor is a strided view of a larger one+ if (src0->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) {+ return false;+ }+ if (src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) {+ return false;+ }return true;}diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.cindex c839044b84..d16dc7d38e 100644--- a/ggml/src/ggml-hexagon/htp/rope-ops.c+++ b/ggml/src/ggml-hexagon/htp/rope-ops.c@@ -23,6 +23,7 @@#define HTP_ROPE_TYPE_NORMAL 0#define HTP_ROPE_TYPE_NEOX 2#define HTP_ROPE_TYPE_MROPE 8+#define HTP_ROPE_TYPE_VISION 24#define HTP_ROPE_TYPE_IMROPE 40#define HTP_ROPE_SPAD_NROWS 16@@ -70,7 +71,9 @@ struct htp_rope_context {struct htp_ops_context * octx;size_t src0_row_size;+ size_t src0_row_stride;size_t dst_row_size;+ size_t dst_row_stride;size_t src0_row_size_aligned;size_t dst_row_size_aligned;size_t theta_cache_offset;@@ -210,6 +213,7 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t,const float pos_e,const int32_t sections[4],const bool is_imrope,+ const bool indep_sects,const float freq_scale,const float * freq_factors,float * corr_dims,@@ -231,6 +235,14 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t,const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f;const int sector = (i0 / 2) % sect_dims;+ if (indep_sects) {+ // Reset theta when crossing into a new section.+ if (sector == 0) { theta_t = pos_t; }+ else if (sector == sections[0]) { theta_h = pos_h; }+ else if (sector == sec_w) { theta_w = pos_w; }+ else if (sector == sec_e) { theta_e = pos_e; }+ }+float theta;if (is_imrope) {// Interleaved: sector mod 3 selects component@@ -422,6 +434,17 @@ static void inline rope_neox_f32(struct htp_rope_context * rctx, uint8_t * restr}}+static void inline rope_vision_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src,+ uint32_t nr, uint32_t ne0, const float * restrict theta_cache) {+ #pragma unroll(4)+ for (uint32_t i = 0; i < nr; i++) {+ float * d = (float *) (dst + i * rctx->dst_row_size_aligned);+ float * s = (float *) (src + i * rctx->src0_row_size_aligned);++ hvx_rope_neox_f32_aa(d, s, ne0, theta_cache);+ }+}+static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {struct htp_rope_context * rctx = (struct htp_rope_context *) data;struct htp_ops_context * octx = rctx->octx;@@ -447,8 +470,9 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {uint64_t tt = HAP_perf_get_qtimer_count();const int32_t mode = rctx->mode;- // MROPE and IMROPE use NEOX-style pairing for the rotation+ // MROPE, IMROPE and VISION use NEOX-style pairing for the rotationconst bool is_neox = (mode & HTP_ROPE_TYPE_NEOX) || (mode & HTP_ROPE_TYPE_MROPE);+ const bool is_vision = (mode == HTP_ROPE_TYPE_VISION);// VTCM setupuint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);@@ -496,8 +520,10 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01;uint8_t * src_spad = src0_spad_base + pr * rctx->src0_row_size_aligned;- dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src_spad, src_addr),- rctx->src0_row_size_aligned, rctx->src0_row_size, pnr);++ // Copy only the row payload while striding the DDR source+ dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr),+ rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr);// FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr);}@@ -516,7 +542,7 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {(float) pos[i2 + ne2],(float) pos[i2 + ne2 * 2],(float) pos[i2 + ne2 * 3],- rctx->sections, is_imrope,+ rctx->sections, is_imrope, is_vision,rctx->freq_scale, freq_factors, rctx->corr_dims,ne0, rctx->ext_factor, rctx->attn_factor,theta_cache, rctx->theta_scale);@@ -542,14 +568,19 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {// FARF(HIGH, "rope-compute %u: ir %u i1 %u i2 %u i3 %u src-spad %p cnr %u : usec %u", ith, ir, i1, i2, i3, src_spad, cnr,// (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start));- if (is_neox) {+ if (is_vision) {+ rope_vision_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache);+ } else if (is_neox) {rope_neox_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache);} else {rope_basic_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache);}uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + i1 * nb1;- dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(dst_addr, dst_spad), rctx->dst_row_size, rctx->dst_row_size_aligned, cnr);++ // Write only the row payload while striding the DDR dst+ dma_queue_push(dma_queue, dma_make_ptr(dst_addr, dst_spad),+ rctx->dst_row_stride, rctx->dst_row_size_aligned, rctx->dst_row_size, cnr);// Prefetch more rows (if any)if ((cr + HTP_ROPE_SPAD_NROWS) < nrows) {@@ -558,8 +589,8 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {uint32_t pir = ir + HTP_ROPE_SPAD_NROWS;const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01;- dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src_spad, src_addr),- rctx->src0_row_size_aligned, rctx->src0_row_size, pnr);+ dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr),+ rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr);// FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr);}@@ -598,12 +629,14 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);- const size_t src0_row_size = src0->nb[1];- const size_t dst_row_size = dst->nb[1];+ const size_t src0_row_size = src0->ne[0] * sizeof(float);+ const size_t src0_row_stride = src0->nb[1];+ const size_t dst_row_size = dst->ne[0] * sizeof(float);+ const size_t dst_row_stride = dst->nb[1];// Aligned row sizes for VTCMconst size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);- const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);+ const size_t dst_row_size_aligned = hex_round_up(dst_row_stride, VLEN);const size_t theta_cache_size_aligned = hex_round_up(src0->ne[0] * sizeof(float), 256);// Calculate spad sizes per thread@@ -652,8 +685,10 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {rope_corr_dims(rctx.n_dims, rctx.n_ctx_orig, rctx.freq_base, rctx.beta_fast, rctx.beta_slow, rctx.corr_dims);- rctx.src0_row_size = src0_row_size;- rctx.dst_row_size = dst_row_size;+ rctx.src0_row_size = src0_row_size;+ rctx.src0_row_stride = src0_row_stride;+ rctx.dst_row_size = dst_row_size;+ rctx.dst_row_stride = dst_row_stride;rctx.src0_row_size_aligned = src0_row_size_aligned;rctx.dst_row_size_aligned = dst_row_size_aligned;rctx.theta_cache_offset = theta_cache_size_aligned;