LLM inference in C/C++
C++ ggml
hexagon: use DMA for contiguous dim1 CONCAT (#29404)
kurquhared319febb148
diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.cindex 259a2c46d7..382a946600 100644--- a/ggml/src/ggml-hexagon/htp/concat-ops.c+++ b/ggml/src/ggml-hexagon/htp/concat-ops.c@@ -247,6 +247,61 @@ static void concat_generic(unsigned int nth, unsigned int ith, void * data) {}}+static bool concat_dim1_contiguous_dma(struct htp_ops_context * octx, int dim, uint32_t type_size) {+ const struct htp_tensor * src0 = octx->src[0];+ const struct htp_tensor * src1 = octx->src[1];+ const struct htp_tensor * dst = octx->dst;++ if (dim != 1 || octx->ctx->mdev.count > 1 ||+ (dst->type != HTP_TYPE_F32 && dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_I32) ||+ src0->type != dst->type || src1->type != dst->type ||+ src0->ne[0] != dst->ne[0] || src1->ne[0] != dst->ne[0] ||+ src0->ne[2] != dst->ne[2] || src1->ne[2] != dst->ne[2] ||+ src0->ne[3] != dst->ne[3] || src1->ne[3] != dst->ne[3] ||+ dst->ne[1] != src0->ne[1] + src1->ne[1] ||+ !htp_tensor_is_contiguous(src0, type_size) ||+ !htp_tensor_is_contiguous(src1, type_size) ||+ !htp_tensor_is_contiguous(dst, type_size)) {+ return false;+ }++ const uint32_t src0_row_size = src0->ne[0] * type_size;+ const uint32_t src1_row_size = src1->ne[0] * type_size;++ // v75+ dma_queue_push() writes a 2D descriptor directly and does not split overflow.+#if __HVX_ARCH__ >= 75+ if (src0_row_size > 0xffffffu || src1_row_size > 0xffffffu ||+ src0->nb[1] > 0xffffffu || src1->nb[1] > 0xffffffu || dst->nb[1] > 0xffffffu ||+ src0->ne[1] > UINT16_MAX || src1->ne[1] > UINT16_MAX) {+ return false;+ }+#endif++ dma_queue * q = octx->ctx->dma[0];++ for (uint32_t i3 = 0; i3 < dst->ne[3]; ++i3) {+ for (uint32_t i2 = 0; i2 < dst->ne[2]; ++i2) {+ dma_addr_t dst_addr = dst->data + i3 * dst->nb[3] + i2 * dst->nb[2];+ dma_addr_t src0_addr = src0->data + i3 * src0->nb[3] + i2 * src0->nb[2];+ dma_addr_t src1_addr = src1->data + i3 * src1->nb[3] + i2 * src1->nb[2];++ if (!dma_queue_push(q, dma_make_data(dst_addr, src0_addr), dst->nb[1], src0->nb[1], src0_row_size, src0->ne[1])) {+ dma_queue_flush(q);+ dma_queue_push(q, dma_make_data(dst_addr, src0_addr), dst->nb[1], src0->nb[1], src0_row_size, src0->ne[1]);+ }++ dst_addr += src0->ne[1] * dst->nb[1];+ if (!dma_queue_push(q, dma_make_data(dst_addr, src1_addr), dst->nb[1], src1->nb[1], src1_row_size, src1->ne[1])) {+ dma_queue_flush(q);+ dma_queue_push(q, dma_make_data(dst_addr, src1_addr), dst->nb[1], src1->nb[1], src1_row_size, src1->ne[1]);+ }+ }+ }++ dma_queue_flush(q);+ return true;+}+int op_concat(struct htp_ops_context * octx) {const struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * src1 = octx->src[1];@@ -258,6 +313,10 @@ int op_concat(struct htp_ops_context * octx) {bool is_src1_transposed = (src1->nb[0] > src1->nb[1]);bool is_src0_transposed = (src0->nb[0] > src0->nb[1]);+ if (concat_dim1_contiguous_dma(octx, dim, type_size)) {+ return HTP_STATUS_OK;+ }+uint32_t n_threads = octx->n_threads;struct htp_concat_context cctx;cctx.octx = octx;