LLM inference in C/C++
C++ ggml
hexagon: shared strided DMA copy for CPY and CONCAT, any-dim CONCAT via DMA (#29685)
Yiwei Shaodcd387a412ca
diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.cindex 1bb9ce5e3e..4dc1463934 100644--- a/ggml/src/ggml-hexagon/htp/concat-ops.c+++ b/ggml/src/ggml-hexagon/htp/concat-ops.c@@ -1,15 +1,17 @@+#include "dma-queue.h"#include "hex-common.h"+#include "hex-cpy-dma.h"+#include "hex-fastdiv.h"#include "hex-profile.h"+#include "hexagon_protos.h"+#include "hexagon_types.h"#include "htp-ctx.h"#include "htp-ops.h"#include "htp-tensor.h"-#include "hexagon_types.h"-#include "hexagon_protos.h"-#include "hvx_hexagon_protos.h"-#include "dma-queue.h"#include "htp-vtcm.h"#include "hvx-utils.h"-#include "hex-fastdiv.h"+#include "hvx_hexagon_protos.h"+#include <string.h>struct htp_concat_context {@@ -285,73 +287,63 @@ static void concat_generic(unsigned int nth, unsigned int ith, void * data) {}}-static bool concat_dim1_contiguous_dma(struct htp_ops_context * octx, int dim, uint32_t type_size) {+static bool concat_dma(struct htp_ops_context * octx, int dim, uint32_t type_size) {+ if (dim < 0 || dim >= HTP_OP_MAX_DIMS) {+ return false;+ }+const struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * src1 = octx->src[1];const struct htp_tensor * dst = octx->dst;- if (dim != 1 || octx->ctx->mdev.count > 1 ||+ // Not partitioned across devices: the row/element-split paths handle that.+ if (octx->ctx->mdev.count > 1 ||(dst->type != HTP_TYPE_F32 && dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_I32) ||- src0->type != dst->type || src1->type != dst->type ||- src0->ne[0] != dst->ne[0] || src1->ne[0] != dst->ne[0] ||- src0->ne[2] != dst->ne[2] || src1->ne[2] != dst->ne[2] ||- src0->ne[3] != dst->ne[3] || src1->ne[3] != dst->ne[3] ||- dst->ne[1] != src0->ne[1] + src1->ne[1] ||- !htp_tensor_is_contiguous(src0, type_size) ||- !htp_tensor_is_contiguous(src1, type_size) ||- !htp_tensor_is_contiguous(dst, type_size)) {+ src0->type != dst->type || src1->type != dst->type || src0->nb[0] != type_size || src1->nb[0] != type_size ||+ dst->nb[0] != type_size || (size_t) dst->ne[0] * type_size > DMA_MAX_SIZE_24B ||+ dst->nb[1] > DMA_MAX_STRIDE_24B || src0->nb[1] > DMA_MAX_STRIDE_24B || src1->nb[1] > DMA_MAX_STRIDE_24B) {return false;}- const uint32_t src0_row_size = src0->ne[0] * type_size;- const uint32_t src1_row_size = src1->ne[0] * type_size;+ for (int d = 0; d < HTP_OP_MAX_DIMS; d++) {+ const uint32_t ne_d = (d == dim) ? src0->ne[d] + src1->ne[d] : src0->ne[d];+ if (dst->ne[d] != ne_d || (d != dim && src1->ne[d] != dst->ne[d])) {+ return false;+ }+ }- // v75+ dma_queue_push() writes a 2D descriptor directly and does not split overflow.-#if __HVX_ARCH__ >= 75- if (src0_row_size > 0xffffffu || src1_row_size > 0xffffffu ||- src0->nb[1] > 0xffffffu || src1->nb[1] > 0xffffffu || dst->nb[1] > 0xffffffu ||- src0->ne[1] > UINT16_MAX || src1->ne[1] > UINT16_MAX) {- return false;+ // The two views of dst, shaped like the sources.+ struct htp_tensor view0 = *dst;+ struct htp_tensor view1 = *dst;+ for (int d = 0; d < HTP_OP_MAX_DIMS; d++) {+ view0.ne[d] = src0->ne[d];+ view1.ne[d] = src1->ne[d];}-#endif+ view1.data += (uint64_t) src0->ne[dim] * dst->nb[dim];dma_queue * q = octx->ctx->dma[0];- for (uint32_t i3 = 0; i3 < dst->ne[3]; ++i3) {- for (uint32_t i2 = 0; i2 < dst->ne[2]; ++i2) {- dma_addr_t dst_addr = dst->data + i3 * dst->nb[3] + i2 * dst->nb[2];- dma_addr_t src0_addr = src0->data + i3 * src0->nb[3] + i2 * src0->nb[2];- dma_addr_t src1_addr = src1->data + i3 * src1->nb[3] + i2 * src1->nb[2];-- if (!dma_queue_push(q, dma_make_data(dst_addr, src0_addr), dst->nb[1], src0->nb[1], src0_row_size, src0->ne[1])) {- dma_queue_flush(q);- dma_queue_push(q, dma_make_data(dst_addr, src0_addr), dst->nb[1], src0->nb[1], src0_row_size, src0->ne[1]);- }-- dst_addr += src0->ne[1] * dst->nb[1];- if (!dma_queue_push(q, dma_make_data(dst_addr, src1_addr), dst->nb[1], src1->nb[1], src1_row_size, src1->ne[1])) {- dma_queue_flush(q);- dma_queue_push(q, dma_make_data(dst_addr, src1_addr), dst->nb[1], src1->nb[1], src1_row_size, src1->ne[1]);- }- }- }-+ cpy_dma_sametype_sameshape(q, &view0, src0, type_size);+ cpy_dma_sametype_sameshape(q, &view1, src1, type_size);dma_queue_flush(q);return true;}int op_concat(struct htp_ops_context * octx) {+ int dim = octx->op_params[0];+ if (dim < 0 || dim >= HTP_OP_MAX_DIMS) {+ return HTP_STATUS_NO_SUPPORT;+ }+const struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * src1 = octx->src[1];const struct htp_tensor * dst = octx->dst;- int dim = octx->op_params[0];-const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;bool is_src1_transposed = (src1->nb[0] > src1->nb[1]);bool is_src0_transposed = (src0->nb[0] > src0->nb[1]);- if (concat_dim1_contiguous_dma(octx, dim, type_size)) {+ if (concat_dma(octx, dim, type_size)) {return HTP_STATUS_OK;}diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.cindex aef3c04533..df38d3eed3 100644--- a/ggml/src/ggml-hexagon/htp/cpy-ops.c+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c@@ -11,12 +11,12 @@#define GGML_COMMON_DECL_C#include "ggml-common.h"+#include "hex-cpy-dma.h"#include "htp-ctx.h"+#include "htp-fence.h"#include "htp-ops.h"-#include "htp-ops.h"-#include "hvx-utils.h"#include "htp-tensor.h"-#include "htp-fence.h"+#include "hvx-utils.h"struct htp_copy_context {struct htp_ops_context * octx;@@ -49,30 +49,6 @@ struct htp_copy_context {struct fastdiv_values div_ne02_ne01_ne00;};-static inline void cpy_dma_sametype_reshape_contig(- dma_queue * dma_q,- dma_addr_t dst,- dma_addr_t src0,- uint32_t total_bytes-) {- if (total_bytes == 0) {- return;- }-- const uint32_t max_chunk = DMA_SAFE_CHUNK_SIZE;- while (total_bytes > 0) {- const uint32_t chunk = MIN(total_bytes, max_chunk);- if (!dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/ 1)) {- dma_queue_flush(dma_q);- dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/ 1);- }- dst += chunk;- src0 += chunk;- total_bytes -= chunk;- }- dma_queue_flush(dma_q);-}-#define cpy_preamble \const struct htp_tensor *src0 = octx->src[0]; \const struct htp_tensor *dst = octx->dst; \@@ -112,6 +88,7 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vodma_addr_t dst_addr = dst->data + ir0 * ne00 * ELEM_SIZE; \dma_addr_t src0_addr = src0->data + ir0 * ne00 * ELEM_SIZE; \cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (ir1 - ir0) * ne00 * ELEM_SIZE); \+ dma_queue_flush(dma_q); \return; \} \const uint32_t ne02_ne01 = ne02 * ne01; \@@ -157,6 +134,7 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, voiddma_addr_t dst_addr = dst->data + th_start * ELEM_SIZE; \dma_addr_t src0_addr = src0->data + th_start * ELEM_SIZE; \cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (th_end - th_start) * ELEM_SIZE); \+ dma_queue_flush(dma_q); \return; \} \\@@ -381,67 +359,6 @@ static void cpy_thread_f32_i32_sameshape(unsigned int nth, unsigned int ith, voi}}-static inline void cpy_dma_push_2d_chunked(- dma_queue * dma_q,- dma_addr_t dst,- dma_addr_t src,- size_t dst_stride,- size_t src_stride,- size_t row_size,- uint32_t nrows-) {- while (nrows > 0) {- const uint32_t cur_rows = MIN(nrows, DMA_MAX_NROWS);- if (!dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows)) {- dma_queue_flush(dma_q);- dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows);- }- dst += cur_rows * dst_stride;- src += cur_rows * src_stride;- nrows -= cur_rows;- }-}--static inline void cpy_dma_sametype_sameshape(- struct htp_ops_context * octx,- const struct htp_tensor * dst,- const struct htp_tensor * src0,- uint32_t elem_size,- uint32_t ne00, uint32_t ne01, uint32_t ne02, uint32_t ne03,- uint32_t nb01, uint32_t nb02, uint32_t nb03,- uint32_t nb1, uint32_t nb2, uint32_t nb3-) {- const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size);-- dma_queue * dma_q = octx->ctx->dma[0];-- if (contiguous) {- cpy_dma_sametype_reshape_contig(dma_q, dst->data, src0->data, ne00 * elem_size * ne01 * ne02 * ne03);- return;- }-- const bool contiguous_outer =- (ne02 == 1 || (nb02 == ne01 * nb01 && nb2 == ne01 * nb1)) &&- (ne03 == 1 || (nb03 == ne02 * nb02 && nb3 == ne02 * nb2));-- if (contiguous_outer) {- uint32_t total_rows = ne01 * ne02 * ne03;- cpy_dma_push_2d_chunked(dma_q, dst->data, src0->data, nb1, nb01, ne00 * elem_size, total_rows);- dma_queue_flush(dma_q);- return;- }-- for (uint32_t i03 = 0; i03 < ne03; i03++) {- for (uint32_t i02 = 0; i02 < ne02; i02++) {- dma_addr_t dst_data = dst->data + i02 * nb2 + i03 * nb3;- dma_addr_t src0_data = src0->data + i02 * nb02 + i03 * nb03;- cpy_dma_push_2d_chunked(dma_q, dst_data, src0_data, nb1, nb01, ne00 * elem_size, ne01);- }- }-- dma_queue_flush(dma_q);-}-static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {cpy_preamble;*use_dma = false;@@ -551,7 +468,8 @@ static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {if (sametype && (octx->ctx->mdev.count <= 1 || htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {if (octx->ctx->mdev.idx == 0) {*use_dma = true;- cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3);+ cpy_dma_sametype_sameshape(octx->ctx->dma[0], dst, src0, ct.src0_type_size);+ dma_queue_flush(octx->ctx->dma[0]);}} else {work_queue_func_t copy_fun = NULL;@@ -582,6 +500,7 @@ static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {if (octx->ctx->mdev.count <= 1 && dst_is_contiguous && src_is_contiguous) {*use_dma = true;cpy_dma_sametype_reshape_contig(octx->ctx->dma[0], dst->data, src0->data, total_elems * ct.dst_type_size);+ dma_queue_flush(octx->ctx->dma[0]);return HTP_STATUS_OK;}diff --git a/ggml/src/ggml-hexagon/htp/hex-cpy-dma.h b/ggml/src/ggml-hexagon/htp/hex-cpy-dma.hnew file mode 100644index 0000000000..c87f495e28--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/hex-cpy-dma.h@@ -0,0 +1,126 @@+#ifndef HEX_CPY_DMA_H+#define HEX_CPY_DMA_H++// DDR<->DDR DMA copies of same-type, same-shape tensors with arbitrary strides.+// Used by CPY for the copy itself and by CONCAT, which is two such copies into+// two views of its destination. Every helper only pushes descriptors; the+// caller flushes the queue when it needs the data.++#include "dma-queue.h"+#include "hex-common.h"+#include "htp-tensor.h"++#include <stddef.h>+#include <stdint.h>++// Contiguous byte run, as 1d transfers of at most DMA_SAFE_CHUNK_SIZE each.+static inline void cpy_dma_sametype_reshape_contig(dma_queue * dma_q,+ dma_addr_t dst,+ dma_addr_t src0,+ uint32_t total_bytes) {+ if (total_bytes == 0) {+ return;+ }++ const uint32_t max_chunk = DMA_SAFE_CHUNK_SIZE;+ while (total_bytes > 0) {+ const uint32_t chunk = MIN(total_bytes, max_chunk);+ if (!dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/1)) {+ dma_queue_flush(dma_q);+ dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/1);+ }+ dst += chunk;+ src0 += chunk;+ total_bytes -= chunk;+ }+}++// One 2d transfer, split at the 16-bit nrows field.+static inline void cpy_dma_push_2d_chunked(dma_queue * dma_q,+ dma_addr_t dst,+ dma_addr_t src,+ size_t dst_stride,+ size_t src_stride,+ size_t row_size,+ uint32_t nrows) {+ if (row_size == 0 || nrows == 0) {+ return;+ }++ while (nrows > 0) {+ const uint32_t cur_rows = MIN(nrows, DMA_MAX_NROWS);+ if (!dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows)) {+ dma_queue_flush(dma_q);+ dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows);+ }+ dst += cur_rows * dst_stride;+ src += cur_rows * src_stride;+ nrows -= cur_rows;+ }+}++// Copy src0 into dst: same type, same ne[], any nb[] above dim 0, dim 0 dense on+// both sides (nb[0] == elem_size).+static inline void cpy_dma_sametype_sameshape(dma_queue * dma_q,+ const struct htp_tensor * dst,+ const struct htp_tensor * src0,+ uint32_t elem_size) {+ const uint32_t ne00 = src0->ne[0];+ const uint32_t ne01 = src0->ne[1];+ const uint32_t ne02 = src0->ne[2];+ const uint32_t ne03 = src0->ne[3];++ if (ne00 == 0 || ne01 == 0 || ne02 == 0 || ne03 == 0) {+ return;+ }++ const uint32_t nb01 = src0->nb[1];+ const uint32_t nb02 = src0->nb[2];+ const uint32_t nb03 = src0->nb[3];++ const uint32_t nb1 = dst->nb[1];+ const uint32_t nb2 = dst->nb[2];+ const uint32_t nb3 = dst->nb[3];++ const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size);++ if (contiguous) {+ cpy_dma_sametype_reshape_contig(dma_q, dst->data, src0->data, ne00 * elem_size * ne01 * ne02 * ne03);+ return;+ }++ // The single-descriptor path flattens (i01,i02,i03) into one row index, so every+ // row must sit at a constant stride: nb01 on the source, nb1 on the destination.+ // Walk the outer dims and require each to continue that progression. A dim of+ // extent 1 spans no rows, so it is skipped -- but its own stride must NOT then be+ // used to justify the next dim's stride, which is what comparing nb03 against+ // ne02*nb02 did: ggml leaves the stride of an extent-1 dim meaningless, so a view+ // could pass the check while its rows were nowhere near that stride.+ uint32_t exp_src = ne01 * nb01;+ uint32_t exp_dst = ne01 * nb1;+ bool contiguous_outer = true;+ if (ne02 != 1) {+ contiguous_outer = contiguous_outer && (nb02 == exp_src) && (nb2 == exp_dst);+ }+ exp_src *= ne02;+ exp_dst *= ne02;+ if (ne03 != 1) {+ contiguous_outer = contiguous_outer && (nb03 == exp_src) && (nb3 == exp_dst);+ }++ if (contiguous_outer) {+ uint32_t total_rows = ne01 * ne02 * ne03;+ cpy_dma_push_2d_chunked(dma_q, dst->data, src0->data, nb1, nb01, ne00 * elem_size, total_rows);+ return;+ }++ for (uint32_t i03 = 0; i03 < ne03; i03++) {+ for (uint32_t i02 = 0; i02 < ne02; i02++) {+ dma_addr_t dst_data = dst->data + i02 * nb2 + i03 * nb3;+ dma_addr_t src0_data = src0->data + i02 * nb02 + i03 * nb03;+ cpy_dma_push_2d_chunked(dma_q, dst_data, src0_data, nb1, nb01, ne00 * elem_size, ne01);+ }+ }+}++#endif /* HEX_CPY_DMA_H */