LLM inference in C/C++
C++ ggml
hexagon: optimize CPY/CONCAT — round 3, short-rows edition (#30341)
Max Krasnyansky8bb3cf0d216a
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex 533f51a0e4..a32ed59190 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -6593,6 +6593,37 @@ static bool ggml_hexagon_precompute_concat_params(}}+ const uint32_t max_gather_ne0 = 128 / type_size;+ const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);+ const bool b_dense = (src1->nb[0] == type_size);+ const bool gather_strides_ok =+ src0->nb[1] >= (size_t) src0->ne[0] * type_size &&+ (b_dense ? src1->nb[1] >= (size_t) src1->ne[0] * type_size+ : src1->nb[0] >= (size_t) src1->ne[1] * type_size);++ if (dim == 0 &&+ (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16) &&+ dst->ne[0] <= max_gather_ne0 &&+ dst->nb[0] == type_size && dst->nb[1] == (size_t) dst->ne[0] * type_size &&+ src0->nb[0] == type_size &&+ (src1->nb[0] == type_size || src1->nb[1] == type_size) &&+ gather_strides_ok &&+ htp_gather_rows_tensor_outer_contiguous(dst) &&+ htp_gather_rows_tensor_outer_contiguous(src0) &&+ htp_gather_rows_tensor_outer_contiguous(src1)) {++ if (htp_gather_rows_solve_layout(&kparams->u.gather,+ (uint32_t) src0->ne[0], (uint32_t) src0->nb[1],+ (uint32_t) src1->ne[0], (uint32_t) src1->nb[1], b_dense,+ type_size, total_rows,+ sess->n_threads, sess->mdev.count, sess->vtcm_size)) {+ kparams->kernel_type = HTP_CONCAT_KERNEL_GATHER_ROWS;+ kparams->n_threads = (uint8_t) sess->n_threads;+ kparams->vtcm_size = kparams->u.gather.slice * sess->n_threads;+ return true;+ }+ }+const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size);if (dma_strides_ok) {@@ -6616,11 +6647,11 @@ static bool ggml_hexagon_precompute_concat_params(return false;}- kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED;- kparams->n_threads = n_threads;- kparams->vtcm_size = layout.total_bytes;- kparams->spad0_size_per_thread = layout.src0_spad_size_per_thread;- kparams->spad1_size_per_thread = layout.src1_spad_size_per_thread;+ kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED;+ kparams->n_threads = n_threads;+ kparams->vtcm_size = layout.total_bytes;+ kparams->u.transposed.spad0_size_per_thread = layout.src0_spad_size_per_thread;+ kparams->u.transposed.spad1_size_per_thread = layout.src1_spad_size_per_thread;return true;}@@ -6706,6 +6737,27 @@ static bool ggml_hexagon_precompute_cpy_params(return true;}+ const uint32_t max_gather_ne0 = 128 / src_type_size;+ const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);+ if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) &&+ dst_is_contiguous && !src_is_contiguous &&+ src0->nb[0] == src_type_size && src0->ne[0] <= max_gather_ne0 &&+ src0->nb[1] >= (size_t) src0->ne[0] * src_type_size &&+ total_rows > 1 &&+ htp_gather_rows_tensor_outer_contiguous(src0)) {++ if (htp_gather_rows_solve_layout(&kparams->u.gather,+ (uint32_t) src0->ne[0], (uint32_t) src0->nb[1],+ 0, 0, false,+ src_type_size, total_rows,+ sess->n_threads, sess->mdev.count, sess->vtcm_size)) {+ kparams->kernel_type = HTP_COPY_KERNEL_GATHER_ROWS;+ kparams->n_threads = (uint8_t) sess->n_threads;+ kparams->vtcm_size = kparams->u.gather.slice * sess->n_threads;+ return true;+ }+ }+if (sameshape) {kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE;kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);@@ -6737,7 +6789,7 @@ static bool ggml_hexagon_precompute_cpy_params(const uint32_t n_threads = sess->n_threads > 0 ? (uint32_t) sess->n_threads : 4;struct htp_copy_convert_vtcm_layout layout;- htp_copy_convert_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], src_type_size, dst_type_size, n_threads);+ htp_copy_convert_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1], src_type_size, dst_type_size, n_threads);if (sess->vtcm_size > 0 && layout.total_bytes > sess->vtcm_size) {return false;@@ -6751,6 +6803,9 @@ static bool ggml_hexagon_precompute_cpy_params(kparams->u.convert.dst_buf_size = layout.dst_buf_size;kparams->u.convert.spad0_size_per_thread = layout.spad0_size_per_thread;kparams->u.convert.spad1_size_per_thread = layout.spad1_size_per_thread;+ kparams->u.convert.src0_row_stride = layout.src0_row_stride;+ kparams->u.convert.dst_row_stride = layout.dst_row_stride;+ kparams->u.convert.blk_rows = layout.blk_rows;kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);kparams->u.convert.div_ne02_ne01 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1]));diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txtindex 787badf079..bc1474ae83 100644--- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt+++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt@@ -33,6 +33,7 @@ add_library(${HTP_LIB} SHAREDset-rows-ops.cget-rows-ops.ccpy-ops.c+ gather-rows.crepeat-ops.cssm-conv.ccumsum-ops.cdiff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.cindex f7dda1408b..40fa09151b 100644--- a/ggml/src/ggml-hexagon/htp/concat-ops.c+++ b/ggml/src/ggml-hexagon/htp/concat-ops.c@@ -13,6 +13,7 @@#include "htp-vtcm.h"#include "hvx-utils.h"#include "hvx_hexagon_protos.h"+#include "gather-rows.h"#include <string.h>@@ -302,9 +303,9 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_construct htp_concat_context cctx;cctx.octx = octx;cctx.spad0_base = vtcm_base;- cctx.spad1_base = vtcm_base + n_threads * kparams->spad0_size_per_thread;- cctx.spad0_size_per_thread = kparams->spad0_size_per_thread;- cctx.spad1_size_per_thread = kparams->spad1_size_per_thread;+ cctx.spad1_base = vtcm_base + n_threads * kparams->u.transposed.spad0_size_per_thread;+ cctx.spad0_size_per_thread = kparams->u.transposed.spad0_size_per_thread;+ cctx.spad1_size_per_thread = kparams->u.transposed.spad1_size_per_thread;cctx.row_start = row_start;cctx.nrows = nrows;cctx.nplanes = dst->ne[2] * dst->ne[3];@@ -318,10 +319,13 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_conint op_concat(struct htp_ops_context * octx) {const struct htp_concat_kernel_params * kparams = (const struct htp_concat_kernel_params *) octx->kernel_params;- const struct htp_tensor * dst = octx->dst;- const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;+ const struct htp_tensor * src0 = octx->src[0];+ const struct htp_tensor * src1 = octx->src[1];+ const struct htp_tensor * dst = octx->dst;int status = HTP_STATUS_OK;+ const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;+switch (kparams->kernel_type) {case HTP_CONCAT_KERNEL_REGULAR:status = concat_regular(octx, kparams->dim, type_size);@@ -331,6 +335,10 @@ int op_concat(struct htp_ops_context * octx) {status = concat_transposed(octx, kparams, type_size);break;+ case HTP_CONCAT_KERNEL_GATHER_ROWS:+ status = htp_gather_rows(octx, src0, src1, &kparams->u.gather, kparams->n_threads);+ break;+default:status = HTP_STATUS_NO_SUPPORT;break;@@ -338,9 +346,5 @@ int op_concat(struct htp_ops_context * octx) {htp_ops_context_set_status(octx, status);- if (octx->ctx->mdev.count > 1) {- htp_mdev_group_barrier(octx);- }-return octx->status;}diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.h b/ggml/src/ggml-hexagon/htp/concat-ops.hindex 1d7218441f..4bfba9894a 100644--- a/ggml/src/ggml-hexagon/htp/concat-ops.h+++ b/ggml/src/ggml-hexagon/htp/concat-ops.h@@ -2,12 +2,19 @@#define HTP_CONCAT_OPS_H#include "hex-common.h"+#include "gather-rows.h"#include <stdint.h>enum htp_concat_kernel_type {HTP_CONCAT_KERNEL_UNSUPPORTED = 0,HTP_CONCAT_KERNEL_REGULAR = 1,HTP_CONCAT_KERNEL_TRANSPOSED = 2,+ HTP_CONCAT_KERNEL_GATHER_ROWS = 3,+};++struct htp_concat_transposed_params {+ uint32_t spad0_size_per_thread;+ uint32_t spad1_size_per_thread;};struct htp_concat_kernel_params {@@ -17,8 +24,11 @@ struct htp_concat_kernel_params {uint8_t pad;uint32_t vtcm_size;- uint32_t spad0_size_per_thread;- uint32_t spad1_size_per_thread;++ union {+ struct htp_concat_transposed_params transposed;+ struct htp_gather_rows_params gather;+ } u;};#if defined(__cplusplus)diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.cindex d366acb759..63ebb38e68 100644--- a/ggml/src/ggml-hexagon/htp/cpy-ops.c+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c@@ -18,6 +18,7 @@#include "htp-ops.h"#include "htp-tensor.h"#include "hvx-utils.h"+#include "gather-rows.h"struct htp_copy_context {struct htp_ops_context * octx;@@ -141,6 +142,25 @@ DEFINE_CPY_RESHAPE(f32, float, 4)DEFINE_CPY_RESHAPE(f16, __fp16, 2)DEFINE_CPY_RESHAPE(i32, int32_t, 4)+struct cpy_row_cursor {+ uint32_t r;+ uint32_t i01;+ uint32_t i02;+ uint32_t i03;+};++static inline void cpy_cursor_advance(struct cpy_row_cursor * c, uint32_t n, uint32_t ne01, uint32_t ne02) {+ c->r += n;+ c->i01 += n;+ if (c->i01 == ne01) {+ c->i01 = 0;+ if (++c->i02 == ne02) {+ c->i02 = 0;+ c->i03++;+ }+ }+}+#define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC) \static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \struct htp_copy_context * ct = (struct htp_copy_context *) data; \@@ -151,7 +171,6 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, voconst uint32_t ir0 = ct->row_start + dr * ith; \const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \if (ir0 >= ir1) return; \- const uint32_t nrows_thread = ir1 - ir0; \\dma_queue * dma_q = octx->ctx->dma[ith]; \struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \@@ -159,6 +178,9 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, voconst struct htp_copy_convert_params * cvt = &ct->kparams->u.convert; \const uint32_t src0_buf_size = cvt->src0_buf_size; \const uint32_t dst_buf_size = cvt->dst_buf_size; \+ const uint32_t src0_row_stride = cvt->src0_row_stride; \+ const uint32_t dst_row_stride = cvt->dst_row_stride; \+ const uint32_t blk_rows = cvt->blk_rows; \uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * cvt->spad0_size_per_thread; \uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * cvt->spad1_size_per_thread; \const uint32_t src0_row_size = ne00 * ct->kparams->src0_type_size; \@@ -170,62 +192,43 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vouint32_t i02 = fastdiv(rem, &cvt->div_ne01); \uint32_t i01 = rem - i02 * ne01; \\- uint32_t f_i01 = i01, f_i02 = i02, f_i03 = i03; \- dma_addr_t f_src0_addr = src0->data + f_i01*nb01 + f_i02*nb02 + f_i03*nb03; \+ struct cpy_row_cursor f = { ir0, i01, i02, i03 }; \+ struct cpy_row_cursor c = f; \\- uint32_t c_i01 = i01, c_i02 = i02, c_i03 = i03; \- dma_addr_t c_dst_addr = dst->data + c_i01*nb1 + c_i02*nb2 + c_i03*nb3; \- \- for (uint32_t r = 0; r < nrows_thread && r < 2; ++r) { \- uint8_t * src_spad = vtcm_src0_base + r * src0_buf_size; \- uint8_t * dst_spad = vtcm_dst_base + r * dst_buf_size; \+ for (uint32_t b = 0; b < 2 && f.r < ir1; ++b) { \+ uint8_t * src_spad = vtcm_src0_base + b * src0_buf_size; \+ uint8_t * dst_spad = vtcm_dst_base + b * dst_buf_size; \+ const uint32_t nr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \+ dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \dma_queue_push(dma_q, dma_make_data(dst->data, dst_spad), \dst_row_size, dst_buf_size, dst_row_size, 0); \dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \- src0_buf_size, src0_row_size, src0_row_size, 1); \- f_src0_addr += nb01; \- if (++f_i01 == ne01) { \- f_i01 = 0; \- if (++f_i02 == ne02) { \- f_i02 = 0; \- f_i03++; \- } \- f_src0_addr = src0->data + f_i02*nb02 + f_i03*nb03; \- } \+ src0_row_stride, nb01, src0_row_size, nr); \+ cpy_cursor_advance(&f, nr, ne01, ne02); \} \\- for (uint32_t r = 0; r < nrows_thread; ++r) { \+ for (uint32_t blk = 0; c.r < ir1; ++blk) { \+ const uint32_t nr = MIN(blk_rows, MIN(ne01 - c.i01, ir1 - c.r)); \uint8_t * dst_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).src; \uint8_t * src_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; \\- htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \- CONV_FUNC(dst_spad, src_spad, ne00); \- htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \+ for (uint32_t r = 0; r < nr; ++r) { \+ CONV_FUNC(dst_spad + r * dst_row_stride, src_spad + r * src0_row_stride, ne00); \+ } \+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \\+ dma_addr_t c_dst_addr = dst->data + c.i01*nb1 + c.i02*nb2 + c.i03*nb3; \dma_queue_push(dma_q, dma_make_data(c_dst_addr, dst_spad), \- dst_row_size, dst_buf_size, dst_row_size, 1); \- c_dst_addr += nb1; \- if (++c_i01 == ne01) { \- c_i01 = 0; \- if (++c_i02 == ne02) { \- c_i02 = 0; \- c_i03++; \- } \- c_dst_addr = dst->data + c_i02*nb2 + c_i03*nb3; \- } \+ nb1, dst_row_stride, dst_row_size, nr); \+ cpy_cursor_advance(&c, nr, ne01, ne02); \\- if (r + 2 < nrows_thread) { \+ if (f.r < ir1) { \+ const uint32_t fnr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \+ dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \- src0_buf_size, src0_row_size, src0_row_size, 1); \- f_src0_addr += nb01; \- if (++f_i01 == ne01) { \- f_i01 = 0; \- if (++f_i02 == ne02) { \- f_i02 = 0; \- f_i03++; \- } \- f_src0_addr = src0->data + f_i02*nb02 + f_i03*nb03; \- } \+ src0_row_stride, nb01, src0_row_size, fnr); \+ cpy_cursor_advance(&f, fnr, ne01, ne02); \} \} \dma_queue_flush(dma_q); \@@ -337,10 +340,6 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htpconst struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * dst = octx->dst;- if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {- return HTP_STATUS_NO_SUPPORT;- }-if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {return HTP_STATUS_INVAL_PARAMS;}@@ -398,10 +397,6 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kernconst struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * dst = octx->dst;- if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {- return HTP_STATUS_NO_SUPPORT;- }-if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {return HTP_STATUS_INVAL_PARAMS;}@@ -461,6 +456,9 @@ int op_cpy(struct htp_ops_context * octx) {case HTP_COPY_KERNEL_RESHAPE:status = cpy_reshape(octx, kparams);break;+ case HTP_COPY_KERNEL_GATHER_ROWS:+ status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_threads);+ break;default:status = HTP_STATUS_NO_SUPPORT;break;@@ -468,11 +466,11 @@ int op_cpy(struct htp_ops_context * octx) {htp_ops_context_set_status(octx, status);- if (octx->ctx->mdev.count > 1) {- htp_mdev_group_barrier(octx);- }-if (octx->op == HTP_OP_CPY_FENCE) {+ if (octx->ctx->mdev.count > 1) {+ htp_mdev_group_barrier(octx);+ }+if (octx->ctx->mdev.idx == 0) {const struct htp_tensor * sync = octx->src[1];const uint32_t seq = (uint32_t) octx->op_params[0];diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.h b/ggml/src/ggml-hexagon/htp/cpy-ops.hindex 4cf7dc13b6..16695c2ab6 100644--- a/ggml/src/ggml-hexagon/htp/cpy-ops.h+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.h@@ -3,6 +3,7 @@#include "hex-common.h"#include "hex-fastdiv.h"+#include "gather-rows.h"#include <stdint.h>enum htp_copy_kernel_type {@@ -12,6 +13,7 @@ enum htp_copy_kernel_type {HTP_COPY_KERNEL_SAMESHAPE_CONVERT = 3,HTP_COPY_KERNEL_RESHAPE = 4,HTP_COPY_KERNEL_SCALAR = 5,+ HTP_COPY_KERNEL_GATHER_ROWS = 6,};struct htp_copy_convert_params {@@ -19,6 +21,9 @@ struct htp_copy_convert_params {uint32_t dst_buf_size;uint32_t spad0_size_per_thread;uint32_t spad1_size_per_thread;+ uint32_t src0_row_stride; // row pitch in the VTCM buffers+ uint32_t dst_row_stride;+ uint32_t blk_rows; // rows moved per DMA descriptorstruct fastdiv_values div_ne01;struct fastdiv_values div_ne02_ne01;};@@ -45,10 +50,19 @@ struct htp_copy_kernel_params {union {struct htp_copy_convert_params convert;struct htp_copy_reshape_params reshape;+ struct htp_gather_rows_params gather;} u;};+// Size of one VTCM staging buffer: big enough that a DMA descriptor moves many short+// rows at once. The copy kernels keep four per thread (two in flight each way), about+// 1 MB of VTCM with 8 threads.+#define HTP_COPY_VTCM_BLK_BYTES (32 * 1024)+struct htp_copy_convert_vtcm_layout {+ uint32_t src0_row_stride;+ uint32_t dst_row_stride;+ uint32_t blk_rows;uint32_t src0_buf_size;uint32_t dst_buf_size;uint32_t spad0_size_per_thread;@@ -56,15 +70,30 @@ struct htp_copy_convert_vtcm_layout {uint32_t total_bytes;};+// ne01 bounds a block: rows of one block share the dim-1 stride.static inline void htp_copy_convert_vtcm_layout_build(struct htp_copy_convert_vtcm_layout * layout,uint32_t ne00,+ uint32_t ne01,uint32_t src_type_size,uint32_t dst_type_size,uint32_t n_threads) {- layout->src0_buf_size = hex_round_up(ne00 * src_type_size, 256);- layout->dst_buf_size = hex_round_up(ne00 * dst_type_size, 256);+ layout->src0_row_stride = hex_round_up(ne00 * src_type_size, 256);+ layout->dst_row_stride = hex_round_up(ne00 * dst_type_size, 256);++ const uint32_t max_row = layout->src0_row_stride > layout->dst_row_stride ? layout->src0_row_stride : layout->dst_row_stride;+ uint32_t blk_rows = HTP_COPY_VTCM_BLK_BYTES / max_row;+ if (blk_rows > ne01) {+ blk_rows = ne01;+ }+ if (blk_rows == 0) {+ blk_rows = 1;+ }+ layout->blk_rows = blk_rows;++ layout->src0_buf_size = blk_rows * layout->src0_row_stride;+ layout->dst_buf_size = blk_rows * layout->dst_row_stride;layout->spad0_size_per_thread = 2 * layout->src0_buf_size;layout->spad1_size_per_thread = 2 * layout->dst_buf_size;layout->total_bytes = n_threads * (layout->spad0_size_per_thread + layout->spad1_size_per_thread);diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.c b/ggml/src/ggml-hexagon/htp/gather-rows.cnew file mode 100644index 0000000000..1db48e59cd--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/gather-rows.c@@ -0,0 +1,387 @@+#include "gather-rows.h"++#include <assert.h>++#include "hex-common.h"+#include "hex-profile.h"+#include "hvx-utils.h"+#include "dma-queue.h"+#include "htp-ctx.h"+#include "htp-ops.h"+#include "htp-tensor.h"+#include "work-queue.h"++typedef void (*gather_rows_compute_fn_t)(+ uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,+ const struct htp_gather_rows_params * restrict kparams,+ const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur);++struct gather_rows_task {+ struct htp_ops_context * octx;+ const struct htp_tensor * a;+ const struct htp_tensor * b;+ dma_addr_t dst;+ uint32_t r_start;+ uint32_t rows;+ uint32_t rows_per_thread;+ const struct htp_gather_rows_params * kparams;+ gather_rows_compute_fn_t compute;+};++static inline __attribute__((always_inline)) void gather_rows_dma_in(+ dma_queue * q, const struct gather_rows_task * task,+ uint8_t * region, uint32_t r, uint32_t nr, uint32_t elem_size) {+ const struct htp_tensor * a = task->a;+ const struct htp_tensor * b = task->b;+ const struct htp_gather_rows_params * kparams = task->kparams;++ const size_t a_size = (nr - 1) * a->nb[1] + kparams->na * elem_size;+ dma_queue_push(q, dma_make_data(region, a->data + r * a->nb[1]), a_size, a_size, a_size, 1);+ if (b) {+ if (kparams->b_dense) {+ const size_t b_size = (nr - 1) * b->nb[1] + kparams->nb * elem_size;+ dma_queue_push(q, dma_make_data(region + kparams->v1, b->data + r * b->nb[1]), b_size, b_size, b_size, 1);+ } else {+ const size_t b_size = (nr - 1) * b->nb[1] + elem_size;+ for (uint32_t k = 0; k < kparams->nb; k++) {+ dma_queue_push(q, dma_make_data(region + kparams->v1 + k * kparams->span1,+ b->data + k * b->nb[0] + r * b->nb[1]), b_size, b_size, b_size, 1);+ }+ }+ }+}++static inline void gather_rows_dma_out(dma_queue * q, const struct gather_rows_task * task,+ uint8_t * out, uint32_t r, uint32_t nr, uint32_t elem_size) {+ const struct htp_gather_rows_params * kparams = task->kparams;+ const size_t out_size = nr * kparams->ne * elem_size;+ dma_queue_push(q, dma_make_data(task->dst + r * kparams->ne * elem_size, out), out_size, out_size, out_size, 1);+}++static __attribute__((noinline)) void gather_rows_init_tables_f32(+ HVX_Vector * tab, HVX_Vector * inc,+ const struct htp_gather_rows_params * kparams,+ const struct htp_tensor * a, const struct htp_tensor * b) {++ const int32_t s_a = kparams->rg * a->nb[1];+ const int32_t s_b = b ? kparams->rg * b->nb[1] : 0;+ const uint32_t b_nb1 = b ? b->nb[1] : 0;+ uint32_t row = 0;+ uint32_t col = 0;+ int32_t t_buf[32] __attribute__((aligned(128)));+ int32_t s_buf[32] __attribute__((aligned(128)));+ for (uint32_t k = 0; k < kparams->vg; k++) {+ for (uint32_t lane = 0; lane < 32; lane++) {+ if (col < kparams->na) {+ t_buf[lane] = row * a->nb[1] + col * 4;+ s_buf[lane] = s_a;+ } else {+ t_buf[lane] = kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1;+ s_buf[lane] = s_b;+ }+ if (++col == kparams->ne) {+ col = 0;+ row++;+ }+ }+ tab[k] = *(const HVX_Vector *) t_buf;+ inc[k] = *(const HVX_Vector *) s_buf;+ }+}++static __attribute__((noinline)) void gather_rows_init_tables_f16(+ HVX_Vector * tab, HVX_Vector * inc,+ const struct htp_gather_rows_params * kparams,+ const struct htp_tensor * a, const struct htp_tensor * b) {++ const int16_t s_a = (int16_t) (kparams->rg * a->nb[1]);+ const int16_t s_b = (int16_t) (b ? kparams->rg * b->nb[1] : 0);+ const uint32_t b_nb1 = b ? b->nb[1] : 0;+ uint32_t row = 0;+ uint32_t col = 0;+ int16_t t_buf[64] __attribute__((aligned(128)));+ int16_t s_buf[64] __attribute__((aligned(128)));+ for (uint32_t k = 0; k < kparams->vg; k++) {+ for (uint32_t lane = 0; lane < 64; lane++) {+ if (col < kparams->na) {+ t_buf[lane] = (int16_t) (row * a->nb[1] + col * 2);+ s_buf[lane] = s_a;+ } else {+ t_buf[lane] = (int16_t) (kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1);+ s_buf[lane] = s_b;+ }+ if (++col == kparams->ne) {+ col = 0;+ row++;+ }+ }+ tab[k] = *(const HVX_Vector *) t_buf;+ inc[k] = *(const HVX_Vector *) s_buf;+ }+}++static void gather_rows_compute_f32_vg1(+ uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,+ const struct htp_gather_rows_params * restrict kparams,+ const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {+ (void) cur;+ const uint32_t region_size = kparams->region_size;+ HVX_Vector vcur = tab[0];+ const HVX_Vector vinc = inc[0];+ #pragma unroll(4)+ for (uint32_t v = 0; v < nvec; v++) {+ Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur);+ vcur = Q6_Vw_vadd_VwVw(vcur, vinc);+ }+}++static void gather_rows_compute_f32_vgn(+ uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,+ const struct htp_gather_rows_params * restrict kparams,+ const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {+ const uint32_t vg = kparams->vg;+ const uint32_t region_size = kparams->region_size;+ for (uint32_t k = 0; k < vg; k++) {+ cur[k] = tab[k];+ }+ uint32_t k = 0;+ #pragma unroll(4)+ for (uint32_t v = 0; v < nvec; v++) {+ Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]);+ cur[k] = Q6_Vw_vadd_VwVw(cur[k], inc[k]);+ if (++k == vg) {+ k = 0;+ }+ }+}++static void gather_rows_compute_f16_vg1(+ uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,+ const struct htp_gather_rows_params * restrict kparams,+ const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {+ (void) cur;+ const uint32_t region_size = kparams->region_size;+ HVX_Vector vcur = tab[0];+ const HVX_Vector vinc = inc[0];+ #pragma unroll(4)+ for (uint32_t v = 0; v < nvec; v++) {+ Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur);+ vcur = Q6_Vh_vadd_VhVh(vcur, vinc);+ }+}++static void gather_rows_compute_f16_vgn(+ uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,+ const struct htp_gather_rows_params * restrict kparams,+ const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {+ const uint32_t vg = kparams->vg;+ const uint32_t region_size = kparams->region_size;+ for (uint32_t k = 0; k < vg; k++) {+ cur[k] = tab[k];+ }+ uint32_t k = 0;+ #pragma unroll(4)+ for (uint32_t v = 0; v < nvec; v++) {+ Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]);+ cur[k] = Q6_Vh_vadd_VhVh(cur[k], inc[k]);+ if (++k == vg) {+ k = 0;+ }+ }+}++static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * data) {+ (void) nth;+ const struct gather_rows_task * task = (const struct gather_rows_task *) data;+ const struct htp_gather_rows_params * kparams = task->kparams;+ const struct htp_tensor * a = task->a;+ const struct htp_tensor * b = task->b;++ const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows);+ const uint32_t r_beg = task->r_start + r_offset;+ const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows);+ if (r_beg >= r_end) {+ return;+ }++ struct htp_thread_trace * tr = &task->octx->ctx->trace[ith];++ uint8_t * base = task->octx->ctx->vtcm_base + ith * kparams->slice;+ HVX_Vector * tab = (HVX_Vector *) base;+ HVX_Vector * inc = tab + kparams->vg;+ HVX_Vector * cur = inc + kparams->vg;+ uint8_t * region_base = (uint8_t *) (cur + kparams->vg);++ uint8_t * region[2] = { region_base, region_base + kparams->buf_stride };+ uint8_t * out[2] = { region[0] + kparams->out_off, region[1] + kparams->out_off };++ htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);+ gather_rows_init_tables_f32(tab, inc, kparams, a, b);+ htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);++ dma_queue * q = task->octx->ctx->dma[ith];+ const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0);+ const uint32_t nr_max = kparams->nr_max;+ const uint32_t nr0 = MIN(nr_max, r_end - r_beg);++ dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out+ gather_rows_dma_in(q, task, region[0], r_beg, nr0, 4);++ const uint32_t r1 = r_beg + nr_max;+ if (r1 < r_end) {+ const uint32_t nr1 = MIN(nr_max, r_end - r1);+ dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out+ gather_rows_dma_in(q, task, region[1], r1, nr1, 4);+ }++ uint32_t buf = 0;+ for (uint32_t r = r_beg; r < r_end; r += nr_max) {+ const uint32_t nr = MIN(nr_max, r_end - r);+ const uint32_t next_buf = buf ^ 1;++ dma_queue_pop(q); // complete out+ for (uint32_t i = 0; i < n_in; i++) {+ dma_queue_pop(q);+ }++ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);+ const uint32_t nvec = (nr * kparams->ne + 31) >> 5;+ task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur);+ hvx_gather_sync(out[buf]);+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);++ gather_rows_dma_out(q, task, out[buf], r, nr, 4);++ const uint32_t r_prefetch = r + 2 * nr_max;+ if (r_prefetch < r_end) {+ const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch);+ gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 4);+ }++ buf = next_buf;+ }+ dma_queue_flush(q);+}++static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * data) {+ (void) nth;+ const struct gather_rows_task * task = (const struct gather_rows_task *) data;+ const struct htp_gather_rows_params * kparams = task->kparams;+ const struct htp_tensor * a = task->a;+ const struct htp_tensor * b = task->b;++ const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows);+ const uint32_t r_beg = task->r_start + r_offset;+ const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows);+ if (r_beg >= r_end) {+ return;+ }++ struct htp_thread_trace * tr = &task->octx->ctx->trace[ith];++ uint8_t * base = task->octx->ctx->vtcm_base + ith * kparams->slice;+ HVX_Vector * tab = (HVX_Vector *) base;+ HVX_Vector * inc = tab + kparams->vg;+ HVX_Vector * cur = inc + kparams->vg;+ uint8_t * region_base = (uint8_t *) (cur + kparams->vg);++ uint8_t * region[2] = { region_base, region_base + kparams->buf_stride };+ uint8_t * out[2] = { region[0] + kparams->out_off, region[1] + kparams->out_off };++ htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);+ gather_rows_init_tables_f16(tab, inc, kparams, a, b);+ htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);++ dma_queue * q = task->octx->ctx->dma[ith];+ const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0);++ const uint32_t nr_max = kparams->nr_max;+ const uint32_t nr0 = MIN(nr_max, r_end - r_beg);+ dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out+ gather_rows_dma_in(q, task, region[0], r_beg, nr0, 2);++ const uint32_t r1 = r_beg + nr_max;+ if (r1 < r_end) {+ const uint32_t nr1 = MIN(nr_max, r_end - r1);+ dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out+ gather_rows_dma_in(q, task, region[1], r1, nr1, 2);+ }++ uint32_t buf = 0;+ for (uint32_t r = r_beg; r < r_end; r += nr_max) {+ const uint32_t nr = MIN(nr_max, r_end - r);+ const uint32_t next_buf = buf ^ 1;++ dma_queue_pop(q); // complete out+ for (uint32_t i = 0; i < n_in; i++) {+ dma_queue_pop(q);+ }++ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);+ const uint32_t nvec = (nr * kparams->ne + 63) >> 6;+ task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur);+ hvx_gather_sync(out[buf]);+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);++ gather_rows_dma_out(q, task, out[buf], r, nr, 2);++ const uint32_t r_prefetch = r + 2 * nr_max;+ if (r_prefetch < r_end) {+ const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch);+ gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 2);+ }++ buf = next_buf;+ }+ dma_queue_flush(q);+}++int htp_gather_rows(+ struct htp_ops_context * octx,+ const struct htp_tensor * a,+ const struct htp_tensor * b,+ const struct htp_gather_rows_params * kparams,+ uint32_t n_threads) {++ if (!htp_ops_context_set_n_threads(octx, n_threads)) {+ return HTP_STATUS_INVAL_PARAMS;+ }++ assert(kparams->slice * octx->n_threads <= octx->ctx->vtcm_size);++ const uint32_t total_rows = kparams->total_rows;+ uint32_t r_start = 0;+ uint32_t r_count = total_rows;+ if (octx->ctx->mdev.count > 1) {+ const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(+ total_rows, kparams->rg, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);+ r_start = range.start;+ r_count = range.count;+ }++ if (r_count == 0) {+ return HTP_STATUS_OK;+ }++ const uint32_t elem_size = (a->type == HTP_TYPE_F16) ? 2 : 4;++ struct gather_rows_task task;+ task.octx = octx;+ task.a = a;+ task.b = b;+ task.dst = octx->dst->data;+ task.r_start = r_start;+ task.rows = r_count;+ task.kparams = kparams;+ task.rows_per_thread = kparams->rows_per_thread;++ if (elem_size == 2) {+ task.compute = (kparams->vg == 1) ? gather_rows_compute_f16_vg1 : gather_rows_compute_f16_vgn;+ } else {+ task.compute = (kparams->vg == 1) ? gather_rows_compute_f32_vg1 : gather_rows_compute_f32_vgn;+ }++ work_queue_func_t worker_func = (elem_size == 2) ? gather_rows_thread_f16 : gather_rows_thread_f32;+ work_queue_run(octx->ctx->work_queue, worker_func, &task, octx->n_threads);+ return HTP_STATUS_OK;+}diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.h b/ggml/src/ggml-hexagon/htp/gather-rows.hnew file mode 100644index 0000000000..3bbb1c6357--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/gather-rows.h@@ -0,0 +1,160 @@+#ifndef HTP_GATHER_ROWS_H+#define HTP_GATHER_ROWS_H++#include "hex-common.h"+#include <stdint.h>+#include <stdbool.h>++#ifndef GGML_MAX_DIMS+#define GGML_MAX_DIMS 4+#endif++struct htp_gather_rows_params {+ uint32_t na;+ uint32_t nb;+ uint32_t ne;+ uint32_t rg;+ uint32_t vg;+ uint32_t nr_max;+ uint32_t v1;+ uint32_t span1;+ uint32_t region_size;+ uint32_t out_off;+ uint32_t buf_stride;+ uint32_t slice;+ uint32_t total_rows;+ uint32_t rows_per_thread;+ uint8_t b_dense;+ uint8_t pad[3];+};++static inline bool htp_gather_rows_solve_layout(+ struct htp_gather_rows_params * kparams,+ uint32_t na, uint32_t a_nb1,+ uint32_t nb, uint32_t b_nb1, bool b_dense,+ uint32_t elem_size,+ uint32_t total_rows,+ uint32_t n_threads,+ uint32_t mdev_count,+ uint32_t vtcm_size) {++ const uint32_t max_ne0 = 128 / elem_size;+ const uint32_t ne = na + nb;+ if (ne == 0 || ne > max_ne0) {+ return false;+ }++ if (elem_size != 4 && elem_size != 2) {+ return false;+ }++ const uint32_t v_lanes = 128 / elem_size;+ const uint32_t g = hex_gcd_u32(ne, v_lanes);+ const uint32_t rg = v_lanes / g;+ const uint32_t vg = ne / g;++ const uint32_t slice = (vtcm_size / n_threads) & ~127u;+ const uint32_t total_chunks = total_rows / rg;+ const uint32_t chunks_per_dev = (total_chunks + mdev_count - 1) / mdev_count;+ const uint32_t max_rows_per_dev = (mdev_count > 1 && total_chunks >= mdev_count)+ ? (chunks_per_dev * rg + (total_rows % rg))+ : total_rows;+ const uint32_t rows_per_thread = hex_round_up((max_rows_per_dev + n_threads - 1) / n_threads, rg);++ const uint32_t fixed = 3 * vg * 128;+ const uint32_t per_row = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0) + ne * elem_size;+ if (slice <= fixed + 2 * per_row * rg) {+ return false;+ }++ const uint32_t avail_per_buf = (slice - fixed) / 2;+ uint32_t nr_max = MIN(avail_per_buf / per_row, rows_per_thread);+ if (elem_size == 2) {+ const uint32_t per_row_region = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0);+ if (per_row_region > 0) {+ nr_max = MIN(nr_max, (32768u - 256u) / per_row_region);+ }+ }+ nr_max = (nr_max / rg) * rg;++ uint32_t v1 = 0;+ uint32_t span1 = 0;+ uint32_t region_size = 0;+ uint32_t out_off = 0;+ uint32_t buf_stride = 0;++ while (nr_max > 0) {+ v1 = hex_round_up((nr_max - 1) * a_nb1 + na * elem_size, 128);+ if (nb > 0) {+ if (b_dense) {+ span1 = elem_size;+ region_size = v1 + (nr_max - 1) * b_nb1 + nb * elem_size;+ } else {+ span1 = (nr_max - 1) * b_nb1 + elem_size;+ region_size = v1 + nb * span1;+ }+ } else {+ span1 = 0;+ region_size = v1;+ }++ out_off = hex_round_up(region_size, 128);+ buf_stride = hex_round_up(out_off + nr_max * ne * elem_size, 128);++ if ((elem_size != 2 || region_size <= 32768) && fixed + 2 * buf_stride <= slice) {+ break;+ }++ nr_max -= rg;+ }++ if (nr_max == 0) {+ return false;+ }++ kparams->na = na;+ kparams->nb = nb;+ kparams->ne = ne;+ kparams->rg = rg;+ kparams->vg = vg;+ kparams->nr_max = nr_max;+ kparams->v1 = v1;+ kparams->span1 = span1;+ kparams->region_size = region_size;+ kparams->out_off = out_off;+ kparams->buf_stride = buf_stride;+ kparams->slice = slice;+ kparams->total_rows = total_rows;+ kparams->rows_per_thread = rows_per_thread;+ kparams->b_dense = b_dense ? 1 : 0;+ kparams->pad[0] = 0;+ kparams->pad[1] = 0;+ kparams->pad[2] = 0;+ return true;+}++#ifdef __cplusplus+struct ggml_tensor;+static inline bool htp_gather_rows_tensor_outer_contiguous(const struct ggml_tensor * t) {+ size_t next_nb = ((const size_t *) t->nb)[1] * ((const int64_t *) t->ne)[1];+ for (int i = 2; i < GGML_MAX_DIMS; i++) {+ if (((const int64_t *) t->ne)[i] != 1 && ((const size_t *) t->nb)[i] != next_nb) {+ return false;+ }+ next_nb *= ((const int64_t *) t->ne)[i];+ }+ return true;+}+#else+struct htp_ops_context;+struct htp_tensor;++int htp_gather_rows(+ struct htp_ops_context * octx,+ const struct htp_tensor * a,+ const struct htp_tensor * b,+ const struct htp_gather_rows_params * kparams,+ uint32_t n_threads);+#endif++#endif // HTP_GATHER_ROWS_Hdiff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.cindex 6e3482b91e..aff1c6c0f7 100644--- a/ggml/src/ggml-hexagon/htp/main.c+++ b/ggml/src/ggml-hexagon/htp/main.c@@ -287,15 +287,15 @@ AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) {}static void vtcm_acquire(struct htp_context * ctx) {- if (!ctx->vtcm_valid) {+ if (!atomic_load(&ctx->vtcm_valid)) {int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 10000000u);if (err != 0) {FARF(ERROR, "ggml-hex: failed to acquire VTCM: 0x%08x", (unsigned)err);abort();}- ctx->vtcm_needs_release = false;- ctx->vtcm_valid = true;+ atomic_store(&ctx->vtcm_needs_release, false);+ atomic_store(&ctx->vtcm_valid, true);// Drop the priority to make sure we get the release callback from other GGML-HTP and QNN-HTP sessionsHAP_compute_res_update_priority(ctx->vtcm_rctx, ctx->thread_prio + 10);@@ -303,16 +303,16 @@ static void vtcm_acquire(struct htp_context * ctx) {}static void vtcm_release(struct htp_context * ctx) {- if (ctx->vtcm_valid) {- ctx->vtcm_valid = false;- ctx->vtcm_needs_release = false;+ if (atomic_load(&ctx->vtcm_valid)) {+ atomic_store(&ctx->vtcm_valid, false);+ atomic_store(&ctx->vtcm_needs_release, false);HAP_compute_res_release_cached(ctx->vtcm_rctx);}}static int vtcm_release_callback(unsigned int rctx, void * state) {struct htp_context * ctx = (struct htp_context *) state;- ctx->vtcm_needs_release = true;+ atomic_store(&ctx->vtcm_needs_release, true);return 0;}@@ -345,8 +345,8 @@ static int vtcm_alloc(struct htp_context * ctx) {ctx->vtcm_base = (uint8_t *) vtcm_ptr;ctx->vtcm_size = vtcm_size;ctx->vtcm_rctx = rctx;- ctx->vtcm_valid = false;- ctx->vtcm_needs_release = false;+ atomic_store(&ctx->vtcm_valid, false);+ atomic_store(&ctx->vtcm_needs_release, false);return 0;}@@ -1309,7 +1309,7 @@ static void process_ops(struct htp_context * ctx) {vtcm_acquire(ctx);- while (!ctx->vtcm_needs_release && !atomic_load(&ctx->killed)) {+ while (!atomic_load(&ctx->vtcm_needs_release) && !atomic_load(&ctx->killed)) {struct htp_opbatch_req req;uint32_t r_size = sizeof(req);diff --git a/scripts/snapdragon/build.py b/scripts/snapdragon/build.pyindex b36e6361b3..62a114e04c 100755--- a/scripts/snapdragon/build.py+++ b/scripts/snapdragon/build.py@@ -10,6 +10,7 @@ import subprocessimport platformimport shutilimport logging+import globfrom sdk import validate_windows_sdks@@ -51,6 +52,8 @@ def main():parser.add_argument("--no-docker", action="store_true", help="Build natively on the host instead of in a docker container")parser.add_argument("--preset", help="Override the CMake preset to use")parser.add_argument("--debug", action="store_true", help="Build in debug mode (uses -debug presets instead of -release)")+ parser.add_argument("--hex-only", "--hexagon-only", action="store_true", help="Only build and push ggml-hexagon and libggml-htp libraries")+ parser.add_argument("--hex-arch", choices=["v73", "v75", "v79", "v81", "73", "75", "79", "81"], help="Specific Hexagon DSP architecture to build with --hex-only (default: all)")# Push optionsparser.add_argument("--push", action="store_true", help="Push built package to the target device via ADB or SSH/SCP")@@ -129,6 +132,16 @@ def main():jobs = args.jobs if args.jobs else os.cpu_count() or 4+ hex_arch = None+ if args.hex_arch:+ hex_arch = args.hex_arch if args.hex_arch.startswith("v") else f"v{args.hex_arch}"++ if args.hex_only:+ if hex_arch:+ hex_targets = ["ggml-hexagon", f"htp-{hex_arch}"]+ else:+ hex_targets = ["ggml-hexagon", "htp-v73", "htp-v75", "htp-v79", "htp-v81"]+if args.no_docker:# Native/local host buildlogger.info("Running native/local CMake build...")@@ -143,7 +156,10 @@ def main():sys.exit(res.returncode)# Build- build_cmd = ["cmake", "--build", build_dir, "-j", str(jobs)]+ build_cmd = ["cmake", "--build", build_dir]+ if args.hex_only:+ build_cmd += ["--target"] + hex_targets+ build_cmd += ["-j", str(jobs)]logger.info(f"+ {' '.join(build_cmd)}")res = subprocess.run(build_cmd, cwd=repo_root)if res.returncode != 0:@@ -165,9 +181,10 @@ def main():install_prefix_container = f"/workspace/{install_dir}/llama.cpp"+ target_args = f"--target {' '.join(hex_targets)} " if args.hex_only else ""build_sh_cmd = (f"cmake --preset {preset} -B /workspace/{build_dir} && "- f"cmake --build /workspace/{build_dir} -j {jobs} && "+ f"cmake --build /workspace/{build_dir} {target_args}-j {jobs} && "f"cmake --install /workspace/{build_dir} --prefix {install_prefix_container}")@@ -206,58 +223,91 @@ def main():sub_items = [item for item in os.listdir(src_path) if not item.startswith(".")]+ if args.hex_only:+ lib_dir = os.path.join(src_path, "lib")+ hex_libs = glob.glob(os.path.join(lib_dir, "libggml-hexagon*"))+ if hex_arch:+ hex_libs += glob.glob(os.path.join(lib_dir, f"libggml-htp-{hex_arch}*"))+ else:+ hex_libs += glob.glob(os.path.join(lib_dir, "libggml-htp*"))++ if not hex_libs:+ logger.error(f"Error: no hexagon libraries found in {lib_dir} to push.")+ sys.exit(1)+if target_type == "android":- logger.info("\nPushing built artifacts to Android device via ADB...")adb_cmd = ["adb"]if target_val: # serialadb_cmd += ["-s", target_val]- # Clean stale package files on device- if sub_items:- clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)- clean_cmd = adb_cmd + ["shell", f"rm -rf {clean_paths}"]- logger.info(f"+ {' '.join(clean_cmd)}")- subprocess.run(clean_cmd)-- # Android destination directory is target_dir- push_cmd = adb_cmd + ["push", os.path.join(src_path, "."), target_dir]- logger.info(f"+ {' '.join(push_cmd)}")- res = subprocess.run(push_cmd)- if res.returncode != 0:- logger.error("ADB push failed.")- sys.exit(res.returncode)-- chmod_cmd = adb_cmd + ["shell", f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]- logger.info(f"+ {' '.join(chmod_cmd)}")- subprocess.run(chmod_cmd)- logger.info("ADB push completed successfully!")+ if args.hex_only:+ logger.info("\nPushing Hexagon libraries to Android device via ADB...")+ target_lib_dir = f"{target_dir}/lib"+ for lib in hex_libs:+ push_cmd = adb_cmd + ["push", lib, f"{target_lib_dir}/"]+ logger.info(f"+ {' '.join(push_cmd)}")+ res = subprocess.run(push_cmd)+ if res.returncode != 0:+ logger.error("ADB push failed.")+ sys.exit(res.returncode)+ logger.info("Hexagon libraries deployed successfully!")+ else:+ # Clean stale package files on device+ if sub_items:+ clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)+ clean_cmd = adb_cmd + ["shell", f"rm -rf {clean_paths}"]+ logger.info(f"+ {' '.join(clean_cmd)}")+ subprocess.run(clean_cmd)++ # Android destination directory is target_dir+ push_cmd = adb_cmd + ["push", os.path.join(src_path, "."), target_dir]+ logger.info(f"+ {' '.join(push_cmd)}")+ res = subprocess.run(push_cmd)+ if res.returncode != 0:+ logger.error("ADB push failed.")+ sys.exit(res.returncode)++ chmod_cmd = adb_cmd + ["shell", f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]+ logger.info(f"+ {' '.join(chmod_cmd)}")+ subprocess.run(chmod_cmd)+ logger.info("ADB push completed successfully!")elif target_type == "linux":ssh_host = target_valif not ssh_host:logger.error("Error: SSH host not specified in target (e.g. use linux:user@host, lnx:user@host, or ubuntu:user@host). Cannot deploy.")sys.exit(1)- logger.info(f"\nDeploying built artifacts to Linux device {ssh_host} via SSH/SCP...")-- # Clean stale package files on remote host- if sub_items:- clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)- clean_cmd = ["ssh", ssh_host, f"rm -rf {clean_paths}"]- logger.info(f"+ {' '.join(clean_cmd)}")- subprocess.run(clean_cmd)-- # Deploy to target_dir- deploy_cmd = ["scp", "-r", os.path.join(src_path, "."), f"{ssh_host}:{target_dir}"]- logger.info(f"+ {' '.join(deploy_cmd)}")- res = subprocess.run(deploy_cmd)- if res.returncode != 0:- logger.error("SSH/SCP deploy failed.")- sys.exit(res.returncode)-- chmod_cmd = ["ssh", ssh_host, f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]- logger.info(f"+ {' '.join(chmod_cmd)}")- subprocess.run(chmod_cmd)- logger.info("SSH/SCP deploy completed successfully!")++ if args.hex_only:+ logger.info(f"\nDeploying Hexagon libraries to Linux device {ssh_host} via SCP...")+ target_lib_dir = f"{target_dir}/lib"+ deploy_cmd = ["scp"] + hex_libs + [f"{ssh_host}:{target_lib_dir}/"]+ logger.info(f"+ {' '.join(deploy_cmd)}")+ res = subprocess.run(deploy_cmd)+ if res.returncode != 0:+ logger.error("SCP deploy failed.")+ sys.exit(res.returncode)+ logger.info("Hexagon libraries deployed successfully!")+ else:+ # Clean stale package files on remote host+ if sub_items:+ clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)+ clean_cmd = ["ssh", ssh_host, f"rm -rf {clean_paths}"]+ logger.info(f"+ {' '.join(clean_cmd)}")+ subprocess.run(clean_cmd)++ # Deploy to target_dir+ deploy_cmd = ["scp", "-r", os.path.join(src_path, "."), f"{ssh_host}:{target_dir}"]+ logger.info(f"+ {' '.join(deploy_cmd)}")+ res = subprocess.run(deploy_cmd)+ if res.returncode != 0:+ logger.error("SSH/SCP deploy failed.")+ sys.exit(res.returncode)++ chmod_cmd = ["ssh", ssh_host, f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]+ logger.info(f"+ {' '.join(chmod_cmd)}")+ subprocess.run(chmod_cmd)+ logger.info("SSH/SCP deploy completed successfully!")elif target_type == "windows":logger.info("\nPush for Windows on Snapdragon (windows) target is currently a stub.")