Skip to content

LLM inference in C/C++

C++ ggml

hexagon: shared strided DMA copy for CPY and CONCAT, any-dim CONCAT via DMA (#29685)

Yiwei Shaodcd387a412ca
diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c
index 1bb9ce5e3e..4dc1463934 100644
--- a/ggml/src/ggml-hexagon/htp/concat-ops.c
+++ b/ggml/src/ggml-hexagon/htp/concat-ops.c
@@ -1,15 +1,17 @@
+#include "dma-queue.h"
#include "hex-common.h"
+#include "hex-cpy-dma.h"
+#include "hex-fastdiv.h"
#include "hex-profile.h"
+#include "hexagon_protos.h"
+#include "hexagon_types.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
-#include "hexagon_types.h"
-#include "hexagon_protos.h"
-#include "hvx_hexagon_protos.h"
-#include "dma-queue.h"
#include "htp-vtcm.h"
#include "hvx-utils.h"
-#include "hex-fastdiv.h"
+#include "hvx_hexagon_protos.h"
+
#include <string.h>
struct htp_concat_context {
@@ -285,73 +287,63 @@ static void concat_generic(unsigned int nth, unsigned int ith, void * data) {
}
}
-static bool concat_dim1_contiguous_dma(struct htp_ops_context * octx, int dim, uint32_t type_size) {
+static bool concat_dma(struct htp_ops_context * octx, int dim, uint32_t type_size) {
+ if (dim < 0 || dim >= HTP_OP_MAX_DIMS) {
+ return false;
+ }
+
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
- if (dim != 1 || octx->ctx->mdev.count > 1 ||
+ // Not partitioned across devices: the row/element-split paths handle that.
+ if (octx->ctx->mdev.count > 1 ||
(dst->type != HTP_TYPE_F32 && dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_I32) ||
- src0->type != dst->type || src1->type != dst->type ||
- src0->ne[0] != dst->ne[0] || src1->ne[0] != dst->ne[0] ||
- src0->ne[2] != dst->ne[2] || src1->ne[2] != dst->ne[2] ||
- src0->ne[3] != dst->ne[3] || src1->ne[3] != dst->ne[3] ||
- dst->ne[1] != src0->ne[1] + src1->ne[1] ||
- !htp_tensor_is_contiguous(src0, type_size) ||
- !htp_tensor_is_contiguous(src1, type_size) ||
- !htp_tensor_is_contiguous(dst, type_size)) {
+ src0->type != dst->type || src1->type != dst->type || src0->nb[0] != type_size || src1->nb[0] != type_size ||
+ dst->nb[0] != type_size || (size_t) dst->ne[0] * type_size > DMA_MAX_SIZE_24B ||
+ dst->nb[1] > DMA_MAX_STRIDE_24B || src0->nb[1] > DMA_MAX_STRIDE_24B || src1->nb[1] > DMA_MAX_STRIDE_24B) {
return false;
}
- const uint32_t src0_row_size = src0->ne[0] * type_size;
- const uint32_t src1_row_size = src1->ne[0] * type_size;
+ for (int d = 0; d < HTP_OP_MAX_DIMS; d++) {
+ const uint32_t ne_d = (d == dim) ? src0->ne[d] + src1->ne[d] : src0->ne[d];
+ if (dst->ne[d] != ne_d || (d != dim && src1->ne[d] != dst->ne[d])) {
+ return false;
+ }
+ }
- // v75+ dma_queue_push() writes a 2D descriptor directly and does not split overflow.
-#if __HVX_ARCH__ >= 75
- if (src0_row_size > 0xffffffu || src1_row_size > 0xffffffu ||
- src0->nb[1] > 0xffffffu || src1->nb[1] > 0xffffffu || dst->nb[1] > 0xffffffu ||
- src0->ne[1] > UINT16_MAX || src1->ne[1] > UINT16_MAX) {
- return false;
+ // The two views of dst, shaped like the sources.
+ struct htp_tensor view0 = *dst;
+ struct htp_tensor view1 = *dst;
+ for (int d = 0; d < HTP_OP_MAX_DIMS; d++) {
+ view0.ne[d] = src0->ne[d];
+ view1.ne[d] = src1->ne[d];
}
-#endif
+ view1.data += (uint64_t) src0->ne[dim] * dst->nb[dim];
dma_queue * q = octx->ctx->dma[0];
- for (uint32_t i3 = 0; i3 < dst->ne[3]; ++i3) {
- for (uint32_t i2 = 0; i2 < dst->ne[2]; ++i2) {
- dma_addr_t dst_addr = dst->data + i3 * dst->nb[3] + i2 * dst->nb[2];
- dma_addr_t src0_addr = src0->data + i3 * src0->nb[3] + i2 * src0->nb[2];
- dma_addr_t src1_addr = src1->data + i3 * src1->nb[3] + i2 * src1->nb[2];
-
- if (!dma_queue_push(q, dma_make_data(dst_addr, src0_addr), dst->nb[1], src0->nb[1], src0_row_size, src0->ne[1])) {
- dma_queue_flush(q);
- dma_queue_push(q, dma_make_data(dst_addr, src0_addr), dst->nb[1], src0->nb[1], src0_row_size, src0->ne[1]);
- }
-
- dst_addr += src0->ne[1] * dst->nb[1];
- if (!dma_queue_push(q, dma_make_data(dst_addr, src1_addr), dst->nb[1], src1->nb[1], src1_row_size, src1->ne[1])) {
- dma_queue_flush(q);
- dma_queue_push(q, dma_make_data(dst_addr, src1_addr), dst->nb[1], src1->nb[1], src1_row_size, src1->ne[1]);
- }
- }
- }
-
+ cpy_dma_sametype_sameshape(q, &view0, src0, type_size);
+ cpy_dma_sametype_sameshape(q, &view1, src1, type_size);
dma_queue_flush(q);
return true;
}
int op_concat(struct htp_ops_context * octx) {
+ int dim = octx->op_params[0];
+ if (dim < 0 || dim >= HTP_OP_MAX_DIMS) {
+ return HTP_STATUS_NO_SUPPORT;
+ }
+
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
- int dim = octx->op_params[0];
-
const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;
bool is_src1_transposed = (src1->nb[0] > src1->nb[1]);
bool is_src0_transposed = (src0->nb[0] > src0->nb[1]);
- if (concat_dim1_contiguous_dma(octx, dim, type_size)) {
+ if (concat_dma(octx, dim, type_size)) {
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c
index aef3c04533..df38d3eed3 100644
--- a/ggml/src/ggml-hexagon/htp/cpy-ops.c
+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c
@@ -11,12 +11,12 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
+#include "hex-cpy-dma.h"
#include "htp-ctx.h"
+#include "htp-fence.h"
#include "htp-ops.h"
-#include "htp-ops.h"
-#include "hvx-utils.h"
#include "htp-tensor.h"
-#include "htp-fence.h"
+#include "hvx-utils.h"
struct htp_copy_context {
struct htp_ops_context * octx;
@@ -49,30 +49,6 @@ struct htp_copy_context {
struct fastdiv_values div_ne02_ne01_ne00;
};
-static inline void cpy_dma_sametype_reshape_contig(
- dma_queue * dma_q,
- dma_addr_t dst,
- dma_addr_t src0,
- uint32_t total_bytes
-) {
- if (total_bytes == 0) {
- return;
- }
-
- const uint32_t max_chunk = DMA_SAFE_CHUNK_SIZE;
- while (total_bytes > 0) {
- const uint32_t chunk = MIN(total_bytes, max_chunk);
- if (!dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/ 1)) {
- dma_queue_flush(dma_q);
- dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/ 1);
- }
- dst += chunk;
- src0 += chunk;
- total_bytes -= chunk;
- }
- dma_queue_flush(dma_q);
-}
-
#define cpy_preamble \
const struct htp_tensor *src0 = octx->src[0]; \
const struct htp_tensor *dst = octx->dst; \
@@ -112,6 +88,7 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
dma_addr_t dst_addr = dst->data + ir0 * ne00 * ELEM_SIZE; \
dma_addr_t src0_addr = src0->data + ir0 * ne00 * ELEM_SIZE; \
cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (ir1 - ir0) * ne00 * ELEM_SIZE); \
+ dma_queue_flush(dma_q); \
return; \
} \
const uint32_t ne02_ne01 = ne02 * ne01; \
@@ -157,6 +134,7 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void
dma_addr_t dst_addr = dst->data + th_start * ELEM_SIZE; \
dma_addr_t src0_addr = src0->data + th_start * ELEM_SIZE; \
cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (th_end - th_start) * ELEM_SIZE); \
+ dma_queue_flush(dma_q); \
return; \
} \
\
@@ -381,67 +359,6 @@ static void cpy_thread_f32_i32_sameshape(unsigned int nth, unsigned int ith, voi
}
}
-static inline void cpy_dma_push_2d_chunked(
- dma_queue * dma_q,
- dma_addr_t dst,
- dma_addr_t src,
- size_t dst_stride,
- size_t src_stride,
- size_t row_size,
- uint32_t nrows
-) {
- while (nrows > 0) {
- const uint32_t cur_rows = MIN(nrows, DMA_MAX_NROWS);
- if (!dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows)) {
- dma_queue_flush(dma_q);
- dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows);
- }
- dst += cur_rows * dst_stride;
- src += cur_rows * src_stride;
- nrows -= cur_rows;
- }
-}
-
-static inline void cpy_dma_sametype_sameshape(
- struct htp_ops_context * octx,
- const struct htp_tensor * dst,
- const struct htp_tensor * src0,
- uint32_t elem_size,
- uint32_t ne00, uint32_t ne01, uint32_t ne02, uint32_t ne03,
- uint32_t nb01, uint32_t nb02, uint32_t nb03,
- uint32_t nb1, uint32_t nb2, uint32_t nb3
-) {
- const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size);
-
- dma_queue * dma_q = octx->ctx->dma[0];
-
- if (contiguous) {
- cpy_dma_sametype_reshape_contig(dma_q, dst->data, src0->data, ne00 * elem_size * ne01 * ne02 * ne03);
- return;
- }
-
- const bool contiguous_outer =
- (ne02 == 1 || (nb02 == ne01 * nb01 && nb2 == ne01 * nb1)) &&
- (ne03 == 1 || (nb03 == ne02 * nb02 && nb3 == ne02 * nb2));
-
- if (contiguous_outer) {
- uint32_t total_rows = ne01 * ne02 * ne03;
- cpy_dma_push_2d_chunked(dma_q, dst->data, src0->data, nb1, nb01, ne00 * elem_size, total_rows);
- dma_queue_flush(dma_q);
- return;
- }
-
- for (uint32_t i03 = 0; i03 < ne03; i03++) {
- for (uint32_t i02 = 0; i02 < ne02; i02++) {
- dma_addr_t dst_data = dst->data + i02 * nb2 + i03 * nb3;
- dma_addr_t src0_data = src0->data + i02 * nb02 + i03 * nb03;
- cpy_dma_push_2d_chunked(dma_q, dst_data, src0_data, nb1, nb01, ne00 * elem_size, ne01);
- }
- }
-
- dma_queue_flush(dma_q);
-}
-
static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {
cpy_preamble;
*use_dma = false;
@@ -551,7 +468,8 @@ static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {
if (sametype && (octx->ctx->mdev.count <= 1 || htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {
if (octx->ctx->mdev.idx == 0) {
*use_dma = true;
- cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3);
+ cpy_dma_sametype_sameshape(octx->ctx->dma[0], dst, src0, ct.src0_type_size);
+ dma_queue_flush(octx->ctx->dma[0]);
}
} else {
work_queue_func_t copy_fun = NULL;
@@ -582,6 +500,7 @@ static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {
if (octx->ctx->mdev.count <= 1 && dst_is_contiguous && src_is_contiguous) {
*use_dma = true;
cpy_dma_sametype_reshape_contig(octx->ctx->dma[0], dst->data, src0->data, total_elems * ct.dst_type_size);
+ dma_queue_flush(octx->ctx->dma[0]);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/hex-cpy-dma.h b/ggml/src/ggml-hexagon/htp/hex-cpy-dma.h
new file mode 100644
index 0000000000..c87f495e28
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/hex-cpy-dma.h
@@ -0,0 +1,126 @@
+#ifndef HEX_CPY_DMA_H
+#define HEX_CPY_DMA_H
+
+// DDR<->DDR DMA copies of same-type, same-shape tensors with arbitrary strides.
+// Used by CPY for the copy itself and by CONCAT, which is two such copies into
+// two views of its destination. Every helper only pushes descriptors; the
+// caller flushes the queue when it needs the data.
+
+#include "dma-queue.h"
+#include "hex-common.h"
+#include "htp-tensor.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+// Contiguous byte run, as 1d transfers of at most DMA_SAFE_CHUNK_SIZE each.
+static inline void cpy_dma_sametype_reshape_contig(dma_queue * dma_q,
+ dma_addr_t dst,
+ dma_addr_t src0,
+ uint32_t total_bytes) {
+ if (total_bytes == 0) {
+ return;
+ }
+
+ const uint32_t max_chunk = DMA_SAFE_CHUNK_SIZE;
+ while (total_bytes > 0) {
+ const uint32_t chunk = MIN(total_bytes, max_chunk);
+ if (!dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/1)) {
+ dma_queue_flush(dma_q);
+ dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/1);
+ }
+ dst += chunk;
+ src0 += chunk;
+ total_bytes -= chunk;
+ }
+}
+
+// One 2d transfer, split at the 16-bit nrows field.
+static inline void cpy_dma_push_2d_chunked(dma_queue * dma_q,
+ dma_addr_t dst,
+ dma_addr_t src,
+ size_t dst_stride,
+ size_t src_stride,
+ size_t row_size,
+ uint32_t nrows) {
+ if (row_size == 0 || nrows == 0) {
+ return;
+ }
+
+ while (nrows > 0) {
+ const uint32_t cur_rows = MIN(nrows, DMA_MAX_NROWS);
+ if (!dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows)) {
+ dma_queue_flush(dma_q);
+ dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows);
+ }
+ dst += cur_rows * dst_stride;
+ src += cur_rows * src_stride;
+ nrows -= cur_rows;
+ }
+}
+
+// Copy src0 into dst: same type, same ne[], any nb[] above dim 0, dim 0 dense on
+// both sides (nb[0] == elem_size).
+static inline void cpy_dma_sametype_sameshape(dma_queue * dma_q,
+ const struct htp_tensor * dst,
+ const struct htp_tensor * src0,
+ uint32_t elem_size) {
+ const uint32_t ne00 = src0->ne[0];
+ const uint32_t ne01 = src0->ne[1];
+ const uint32_t ne02 = src0->ne[2];
+ const uint32_t ne03 = src0->ne[3];
+
+ if (ne00 == 0 || ne01 == 0 || ne02 == 0 || ne03 == 0) {
+ return;
+ }
+
+ const uint32_t nb01 = src0->nb[1];
+ const uint32_t nb02 = src0->nb[2];
+ const uint32_t nb03 = src0->nb[3];
+
+ const uint32_t nb1 = dst->nb[1];
+ const uint32_t nb2 = dst->nb[2];
+ const uint32_t nb3 = dst->nb[3];
+
+ const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size);
+
+ if (contiguous) {
+ cpy_dma_sametype_reshape_contig(dma_q, dst->data, src0->data, ne00 * elem_size * ne01 * ne02 * ne03);
+ return;
+ }
+
+ // The single-descriptor path flattens (i01,i02,i03) into one row index, so every
+ // row must sit at a constant stride: nb01 on the source, nb1 on the destination.
+ // Walk the outer dims and require each to continue that progression. A dim of
+ // extent 1 spans no rows, so it is skipped -- but its own stride must NOT then be
+ // used to justify the next dim's stride, which is what comparing nb03 against
+ // ne02*nb02 did: ggml leaves the stride of an extent-1 dim meaningless, so a view
+ // could pass the check while its rows were nowhere near that stride.
+ uint32_t exp_src = ne01 * nb01;
+ uint32_t exp_dst = ne01 * nb1;
+ bool contiguous_outer = true;
+ if (ne02 != 1) {
+ contiguous_outer = contiguous_outer && (nb02 == exp_src) && (nb2 == exp_dst);
+ }
+ exp_src *= ne02;
+ exp_dst *= ne02;
+ if (ne03 != 1) {
+ contiguous_outer = contiguous_outer && (nb03 == exp_src) && (nb3 == exp_dst);
+ }
+
+ if (contiguous_outer) {
+ uint32_t total_rows = ne01 * ne02 * ne03;
+ cpy_dma_push_2d_chunked(dma_q, dst->data, src0->data, nb1, nb01, ne00 * elem_size, total_rows);
+ return;
+ }
+
+ for (uint32_t i03 = 0; i03 < ne03; i03++) {
+ for (uint32_t i02 = 0; i02 < ne02; i02++) {
+ dma_addr_t dst_data = dst->data + i02 * nb2 + i03 * nb3;
+ dma_addr_t src0_data = src0->data + i02 * nb02 + i03 * nb03;
+ cpy_dma_push_2d_chunked(dma_q, dst_data, src0_data, nb1, nb01, ne00 * elem_size, ne01);
+ }
+ }
+}
+
+#endif /* HEX_CPY_DMA_H */