Skip to content

LLM inference in C/C++

C++ ggml

hexagon: add VISION RoPE support (#25216)

Aparna M Pf2d1c2f3984c
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index 3d9fc9b56d..0c78879cbb 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -3105,8 +3105,12 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess
int mode = op_params[2];
+ // n_dims == ne0/2, so the rotation spans the full row
if (mode == GGML_ROPE_TYPE_VISION) {
- return false;
+ const int n_dims = op_params[1];
+ if (n_dims != (int) (op->src[0]->ne[0] / 2)) {
+ return false;
+ }
}
if (mode & 1) {
return false;
@@ -3137,16 +3141,23 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess
}
if (src2) {
- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(src2) ||
- !ggml_is_contiguous(dst)) {
+ if (!ggml_is_contiguous(src1) || !ggml_is_contiguous(src2)) {
return false;
}
} else {
- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {
+ if (!ggml_is_contiguous(src1)) {
return false;
}
}
+ // src0/dst elements within a row must be contiguous (nb[0] == sizeof(float)).
+ // nb[1] may exceed ne[0]*sizeof(float) when the tensor is a strided view of a larger one
+ if (src0->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) {
+ return false;
+ }
+ if (src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) {
+ return false;
+ }
return true;
}
diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c
index c839044b84..d16dc7d38e 100644
--- a/ggml/src/ggml-hexagon/htp/rope-ops.c
+++ b/ggml/src/ggml-hexagon/htp/rope-ops.c
@@ -23,6 +23,7 @@
#define HTP_ROPE_TYPE_NORMAL 0
#define HTP_ROPE_TYPE_NEOX 2
#define HTP_ROPE_TYPE_MROPE 8
+#define HTP_ROPE_TYPE_VISION 24
#define HTP_ROPE_TYPE_IMROPE 40
#define HTP_ROPE_SPAD_NROWS 16
@@ -70,7 +71,9 @@ struct htp_rope_context {
struct htp_ops_context * octx;
size_t src0_row_size;
+ size_t src0_row_stride;
size_t dst_row_size;
+ size_t dst_row_stride;
size_t src0_row_size_aligned;
size_t dst_row_size_aligned;
size_t theta_cache_offset;
@@ -210,6 +213,7 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t,
const float pos_e,
const int32_t sections[4],
const bool is_imrope,
+ const bool indep_sects,
const float freq_scale,
const float * freq_factors,
float * corr_dims,
@@ -231,6 +235,14 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t,
const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f;
const int sector = (i0 / 2) % sect_dims;
+ if (indep_sects) {
+ // Reset theta when crossing into a new section.
+ if (sector == 0) { theta_t = pos_t; }
+ else if (sector == sections[0]) { theta_h = pos_h; }
+ else if (sector == sec_w) { theta_w = pos_w; }
+ else if (sector == sec_e) { theta_e = pos_e; }
+ }
+
float theta;
if (is_imrope) {
// Interleaved: sector mod 3 selects component
@@ -422,6 +434,17 @@ static void inline rope_neox_f32(struct htp_rope_context * rctx, uint8_t * restr
}
}
+static void inline rope_vision_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src,
+ uint32_t nr, uint32_t ne0, const float * restrict theta_cache) {
+ #pragma unroll(4)
+ for (uint32_t i = 0; i < nr; i++) {
+ float * d = (float *) (dst + i * rctx->dst_row_size_aligned);
+ float * s = (float *) (src + i * rctx->src0_row_size_aligned);
+
+ hvx_rope_neox_f32_aa(d, s, ne0, theta_cache);
+ }
+}
+
static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {
struct htp_rope_context * rctx = (struct htp_rope_context *) data;
struct htp_ops_context * octx = rctx->octx;
@@ -447,8 +470,9 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {
uint64_t tt = HAP_perf_get_qtimer_count();
const int32_t mode = rctx->mode;
- // MROPE and IMROPE use NEOX-style pairing for the rotation
+ // MROPE, IMROPE and VISION use NEOX-style pairing for the rotation
const bool is_neox = (mode & HTP_ROPE_TYPE_NEOX) || (mode & HTP_ROPE_TYPE_MROPE);
+ const bool is_vision = (mode == HTP_ROPE_TYPE_VISION);
// VTCM setup
uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);
@@ -496,8 +520,10 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {
const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01;
uint8_t * src_spad = src0_spad_base + pr * rctx->src0_row_size_aligned;
- dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src_spad, src_addr),
- rctx->src0_row_size_aligned, rctx->src0_row_size, pnr);
+
+ // Copy only the row payload while striding the DDR source
+ dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr),
+ rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr);
// FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr);
}
@@ -516,7 +542,7 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {
(float) pos[i2 + ne2],
(float) pos[i2 + ne2 * 2],
(float) pos[i2 + ne2 * 3],
- rctx->sections, is_imrope,
+ rctx->sections, is_imrope, is_vision,
rctx->freq_scale, freq_factors, rctx->corr_dims,
ne0, rctx->ext_factor, rctx->attn_factor,
theta_cache, rctx->theta_scale);
@@ -542,14 +568,19 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {
// FARF(HIGH, "rope-compute %u: ir %u i1 %u i2 %u i3 %u src-spad %p cnr %u : usec %u", ith, ir, i1, i2, i3, src_spad, cnr,
// (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start));
- if (is_neox) {
+ if (is_vision) {
+ rope_vision_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache);
+ } else if (is_neox) {
rope_neox_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache);
} else {
rope_basic_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache);
}
uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + i1 * nb1;
- dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(dst_addr, dst_spad), rctx->dst_row_size, rctx->dst_row_size_aligned, cnr);
+
+ // Write only the row payload while striding the DDR dst
+ dma_queue_push(dma_queue, dma_make_ptr(dst_addr, dst_spad),
+ rctx->dst_row_stride, rctx->dst_row_size_aligned, rctx->dst_row_size, cnr);
// Prefetch more rows (if any)
if ((cr + HTP_ROPE_SPAD_NROWS) < nrows) {
@@ -558,8 +589,8 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {
uint32_t pir = ir + HTP_ROPE_SPAD_NROWS;
const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01;
- dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src_spad, src_addr),
- rctx->src0_row_size_aligned, rctx->src0_row_size, pnr);
+ dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr),
+ rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr);
// FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr);
}
@@ -598,12 +629,14 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);
- const size_t src0_row_size = src0->nb[1];
- const size_t dst_row_size = dst->nb[1];
+ const size_t src0_row_size = src0->ne[0] * sizeof(float);
+ const size_t src0_row_stride = src0->nb[1];
+ const size_t dst_row_size = dst->ne[0] * sizeof(float);
+ const size_t dst_row_stride = dst->nb[1];
// Aligned row sizes for VTCM
const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);
- const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);
+ const size_t dst_row_size_aligned = hex_round_up(dst_row_stride, VLEN);
const size_t theta_cache_size_aligned = hex_round_up(src0->ne[0] * sizeof(float), 256);
// Calculate spad sizes per thread
@@ -652,8 +685,10 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
rope_corr_dims(rctx.n_dims, rctx.n_ctx_orig, rctx.freq_base, rctx.beta_fast, rctx.beta_slow, rctx.corr_dims);
- rctx.src0_row_size = src0_row_size;
- rctx.dst_row_size = dst_row_size;
+ rctx.src0_row_size = src0_row_size;
+ rctx.src0_row_stride = src0_row_stride;
+ rctx.dst_row_size = dst_row_size;
+ rctx.dst_row_stride = dst_row_stride;
rctx.src0_row_size_aligned = src0_row_size_aligned;
rctx.dst_row_size_aligned = dst_row_size_aligned;
rctx.theta_cache_offset = theta_cache_size_aligned;