LLM inference in C/C++
C++ ggml
hexagon: extend scalar DMA broadcasting support for binary ops (#30245)
kurquhar4413a192410f
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex a32ed59190..1c1c92da20 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -344,7 +344,8 @@ static bool ggml_hexagon_precompute_binary_params(const struct ggml_tensor * src0,const struct ggml_tensor * src1,const struct ggml_tensor * dst,- struct htp_binary_kernel_params * kparams+ struct htp_binary_kernel_params * kparams,+ const struct ggml_tensor * src2 = nullptr);static void ggml_hexagon_precompute_get_rows_params(@@ -5864,7 +5865,8 @@ static bool ggml_hexagon_precompute_binary_params(const struct ggml_tensor * src0,const struct ggml_tensor * src1,const struct ggml_tensor * dst,- struct htp_binary_kernel_params * kparams+ struct htp_binary_kernel_params * kparams,+ const struct ggml_tensor * src2) {memset(kparams, 0, sizeof(*kparams));@@ -5877,14 +5879,17 @@ static bool ggml_hexagon_precompute_binary_params(const size_t src1_row_size_aligned = hex_round_up(src1_row_size, 128);const size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128);+ if (!htp_binary_strides_safe(src0, src1, dst, elem_size)) {+ return false;+ }+const bool is_add_id = op == HTP_OP_ADD_ID;const bool is_scalar = !is_add_id && src1->ne[0] == 1;- const bool is_transposed = src0->nb[1] < src0_row_size || src1->nb[1] < src1_row_size || dst->nb[1] < dst_row_size;- const bool is_row_bcast = !is_add_id && !is_scalar && !is_transposed &&+ const bool is_row_bcast = !is_add_id && !is_scalar &&src1->ne[0] == src0->ne[0] &&(src0->ne[1] > 1 || src0->ne[2] > 1 || src0->ne[3] > 1) &&src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1;- const bool is_same_shape = !is_add_id && !is_scalar && !is_transposed &&+ const bool is_same_shape = !is_add_id && !is_scalar &&src1->ne[0] == src0->ne[0] &&src1->ne[1] == src0->ne[1] &&(src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) &&@@ -5925,28 +5930,42 @@ static bool ggml_hexagon_precompute_binary_params(enum htp_binary_kernel_type kernel_type;size_t src1_size = 0;+ size_t src2_size = 0;if (is_add_id) {+ GGML_ASSERT(src2 != nullptr);kernel_type = HTP_BINARY_KERNEL_ADD_ID;src1_size = hex_round_up(src1->ne[1] * src1_row_size_aligned, 128);+ src2_size = hex_round_up(ggml_nelements(src2) * sizeof(int32_t), 128);+ if (src1_size + src2_size > sess->vtcm_size) {+ return false;+ }} else if (is_row_bcast) {kernel_type = HTP_BINARY_KERNEL_ROW_BCAST;src1_size = src1_row_size_aligned;} else if (is_scalar) {- const bool is_scalar_static = (src1->ne[2] == 1 && src1->ne[3] == 1) &&- (src1->ne[1] == 1 || src1->nb[1] == elem_size);- if (is_scalar_static) {- kernel_type = HTP_BINARY_KERNEL_SCALAR_DMA;- src1_size = hex_round_up(src1->ne[1] * elem_size, 128);- } else {- kernel_type = HTP_BINARY_KERNEL_SCALAR;+ src1_size = hex_round_up(ggml_nelements(src1) * elem_size, 128);+ if (src1_size > sess->vtcm_size) {+ return false;}+ kernel_type = HTP_BINARY_KERNEL_SCALAR;+ kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;} else if (is_same_shape) {kernel_type = HTP_BINARY_KERNEL_SAME_SHAPE;} else if (is_complex) {+ src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128);+ if (src1_size > sess->vtcm_size) {+ return false;+ }kernel_type = HTP_BINARY_KERNEL_COMPLEX;+ kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;} else {+ src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128);+ if (src1_size > sess->vtcm_size) {+ return false;+ }kernel_type = HTP_BINARY_KERNEL_REPEAT;+ kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;}kparams->kernel_type = kernel_type;@@ -5955,6 +5974,39 @@ static bool ggml_hexagon_precompute_binary_params(kparams->src1_row_size_aligned = src1_row_size_aligned;kparams->dst_row_size_aligned = dst_row_size_aligned;kparams->src1_size = src1_size;+ kparams->src2_size = src2_size;++ const bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]);+ const bool src1_contig_dim1 = (src1->nb[2] == src1->ne[1] * src1->nb[1]);+ const bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]);++ const bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]);+ const bool src1_contig_dim2 = (src1->nb[3] == src1->ne[2] * src1->nb[2]);+ const bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]);++ bool split_at_ne01 = false;+ if (src0->ne[2] > 1) {+ if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {+ split_at_ne01 = (src1->ne[2] != src0->ne[2]) || !src0_contig_dim1 || !src1_contig_dim1 || !dst_contig_dim1;+ } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {+ split_at_ne01 = !src0_contig_dim1 || !dst_contig_dim1;+ } else {+ split_at_ne01 = true;+ }+ }+ kparams->split_at_ne01 = split_at_ne01 ? 1 : 0;++ bool split_at_ne02 = false;+ if (src0->ne[3] > 1) {+ if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {+ split_at_ne02 = (src1->ne[3] != src0->ne[3]) || !src0_contig_dim2 || !src1_contig_dim2 || !dst_contig_dim2;+ } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {+ split_at_ne02 = !src0_contig_dim2 || !dst_contig_dim2;+ } else {+ split_at_ne02 = true;+ }+ }+ kparams->split_at_ne02 = split_at_ne02 ? 1 : 0;struct htp_binary_vtcm_layout L;htp_binary_vtcm_layout_build(&L, kparams, sess->vtcm_size);@@ -7139,9 +7191,12 @@ static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * seif (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {return false;}+ if (src2->nb[1] < src2->ne[0] * sizeof(int32_t)) {+ return false;+ }struct htp_binary_kernel_params kparams;- return ggml_hexagon_precompute_binary_params(sess, HTP_OP_ADD_ID, src0, src1, dst, &kparams);+ return ggml_hexagon_precompute_binary_params(sess, HTP_OP_ADD_ID, src0, src1, dst, &kparams, src2);}static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -8504,9 +8559,10 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg);} else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {const ggml_tensor * src1 = node.node->src[1];+ const ggml_tensor * src2 = (node.opcode == HTP_OP_ADD_ID) ? node.node->src[2] : nullptr;GGML_ASSERT(ggml_hexagon_precompute_binary_params(sess,node.opcode, node.node->src[0], src1, node.node,- (struct htp_binary_kernel_params *) node.kernel_params));+ (struct htp_binary_kernel_params *) node.kernel_params, src2));} else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) {ggml_hexagon_precompute_flash_attn_params(sess,node.node,@@ -9391,9 +9447,11 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, conscase GGML_GLU_OP_SWIGLU_OAI:case GGML_GLU_OP_SWIGLU_CLAMP:case GGML_GLU_OP_GEGLU:+ supp = ggml_hexagon_supported_activations(sess, op);+ break;case GGML_GLU_OP_GEGLU_QUICK:case GGML_GLU_OP_GEGLU_ERF:- supp = ggml_hexagon_supported_activations(sess, op);+ supp = (op->src[0]->type == GGML_TYPE_F32) && ggml_hexagon_supported_activations(sess, op);break;default:supp = false;diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.cindex 1286176af5..78ce4bb63d 100644--- a/ggml/src/ggml-hexagon/htp/binary-ops.c+++ b/ggml/src/ggml-hexagon/htp/binary-ops.c@@ -103,13 +103,13 @@ static inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_// Out-of-line compute micro-kernels-typedef void (*compute_scalar_dma_t)(+typedef void (*compute_scalar_t)(uint8_t * dst, const uint8_t * src0, const void * s1_table,uint32_t cur_i11, uint32_t ne11, uint32_t n_rows,size_t dst_stride, size_t src0_stride, uint32_t ne00);-#define DEFINE_COMPUTE_SCALAR_DMA(NAME, TYPE, HVX_STMT) \-static void compute_scalar_dma_##NAME( \+#define DEFINE_COMPUTE_SCALAR(NAME, TYPE, HVX_STMT) \+static void compute_scalar_##NAME( \uint8_t * dst, const uint8_t * src0, const void * s1_table, \uint32_t cur_i11, uint32_t ne11, uint32_t n_rows, \size_t dst_stride, size_t src0_stride, uint32_t ne00) { \@@ -125,31 +125,6 @@ static void compute_scalar_dma_##NAME( \} \}-DEFINE_COMPUTE_SCALAR_DMA(add_f32, float, hvx_add_scalar_f32_aa(r_dst, r_src0, val, ne00))-DEFINE_COMPUTE_SCALAR_DMA(add_f16, _Float16, hvx_add_scalar_f16_aa(r_dst, r_src0, val, ne00))-DEFINE_COMPUTE_SCALAR_DMA(sub_f32, float, hvx_sub_scalar_f32_aa(r_dst, r_src0, val, ne00))-DEFINE_COMPUTE_SCALAR_DMA(sub_f16, _Float16, hvx_sub_scalar_f16_aa(r_dst, r_src0, val, ne00))-DEFINE_COMPUTE_SCALAR_DMA(mul_f32, float, hvx_mul_scalar_f32_aa(r_dst, r_src0, val, ne00))-DEFINE_COMPUTE_SCALAR_DMA(mul_f16, _Float16, hvx_mul_scalar_f16_aa(r_dst, r_src0, val, ne00))-DEFINE_COMPUTE_SCALAR_DMA(div_f32, float, hvx_mul_scalar_f32_aa(r_dst, r_src0, 1.0f / (val), ne00))-DEFINE_COMPUTE_SCALAR_DMA(div_f16, _Float16, hvx_div_scalar_f16_aa(r_dst, r_src0, val, ne00))--typedef void (*compute_scalar_t)(- uint8_t * dst, const uint8_t * src0, const uint8_t * src1_ptr, uint32_t s1_stride,- uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00);--#define DEFINE_COMPUTE_SCALAR(NAME, TYPE, HVX_STMT) \-static void compute_scalar_##NAME( \- uint8_t * dst, const uint8_t * src0, const uint8_t * src1_ptr, uint32_t s1_stride, \- uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { \- for (uint32_t r = 0; r < n_rows; r++) { \- uint8_t * r_dst = dst + r * dst_stride; \- const uint8_t * r_src0 = src0 + r * src0_stride; \- TYPE val = *(const TYPE *)(src1_ptr + r * s1_stride); \- HVX_STMT; \- } \-}-DEFINE_COMPUTE_SCALAR(add_f32, float, hvx_add_scalar_f32_aa(r_dst, r_src0, val, ne00))DEFINE_COMPUTE_SCALAR(add_f16, _Float16, hvx_add_scalar_f16_aa(r_dst, r_src0, val, ne00))DEFINE_COMPUTE_SCALAR(sub_f32, float, hvx_sub_scalar_f32_aa(r_dst, r_src0, val, ne00))@@ -205,17 +180,17 @@ DEFINE_COMPUTE_ROW_BCAST(div_f16, hvx_div_f16_aaa)typedef void (*compute_complex_t)(uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane,- uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11,+ uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride,uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00);#define DEFINE_COMPUTE_COMPLEX(NAME, HVX_FN) \static void compute_complex_##NAME( \uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, \- uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, \+ uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, \uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { \for (uint32_t r = 0; r < n_rows; r++) { \uint32_t i11 = fastmodulo(i01 + r, ne11, div11); \- const uint8_t * r_src1 = src1_plane + i11 * nb11; \+ const uint8_t * r_src1 = src1_plane + i11 * src1_stride; \HVX_FN(dst + r * dst_stride, src0 + r * src0_stride, r_src1, ne00); \} \}@@ -231,17 +206,17 @@ DEFINE_COMPUTE_COMPLEX(div_f16, hvx_div_f16_aau)typedef void (*compute_repeat_t)(uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane,- uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11,+ uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride,uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00, uint32_t ne10);#define DEFINE_COMPUTE_REPEAT(NAME, TYPE, HVX_FN) \static void compute_repeat_##NAME( \uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, \- uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t nb11, \+ uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, \uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00, uint32_t ne10) { \for (uint32_t r = 0; r < n_rows; r++) { \uint32_t i11 = fastmodulo(i01 + r, ne11, div11); \- const uint8_t * r_src1_row = src1_plane + i11 * nb11; \+ const uint8_t * r_src1_row = src1_plane + i11 * src1_stride; \uint8_t * r_dst = dst + r * dst_stride; \const uint8_t * r_src0 = src0 + r * src0_stride; \for (uint32_t c = 0; c < ne00; c += ne10) { \@@ -261,17 +236,17 @@ DEFINE_COMPUTE_REPEAT(div_f32, float, hvx_div_f32_uuu)DEFINE_COMPUTE_REPEAT(div_f16, _Float16, hvx_div_f16_uuu)typedef void (*compute_add_id_t)(- uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const char * src2_data,- uint32_t i01, uint32_t i02, uint32_t nb20, uint32_t nb21, uint32_t src1_stride,+ uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const int32_t * src2_ids,+ uint32_t i01, uint32_t i02, uint32_t ne01, uint32_t src1_stride,uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00);static void compute_add_id_f32(- uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const char * src2_data,- uint32_t i01, uint32_t i02, uint32_t nb20, uint32_t nb21, uint32_t src1_stride,+ uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const int32_t * src2_ids,+ uint32_t i01, uint32_t i02, uint32_t ne01, uint32_t src1_stride,uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) {for (uint32_t r = 0; r < n_rows; r++) {uint32_t r_i01 = i01 + r;- const int32_t idx = *(const int32_t *)(src2_data + r_i01 * nb20 + i02 * nb21);+ const int32_t idx = src2_ids[i02 * ne01 + r_i01];if (idx < 0) {memcpy(dst + r * dst_stride, src0 + r * src0_stride, ne00 * sizeof(float));continue;@@ -283,18 +258,22 @@ static void compute_add_id_f32(}}-// 1a. Scalar src1 in VTCM via DMA (ne10 == 1, ne12 == 1, ne13 == 1)-static void binary_thread_scalar_dma(unsigned int nth, unsigned int ith, void * data) {+// 1. Scalar src1 in VTCM via DMA (ne10 == 1)+static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data) {struct htp_binary_context * bctx = (struct htp_binary_context *) data;struct htp_ops_context * octx = bctx->octx;htp_binary_preamble;+ const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16);+ const size_t plane_stride = (size_t) ne12 * ne11 * elem_size;+ const size_t row_stride = (size_t) ne11 * elem_size;+const uint32_t row_size_bytes = bctx->row_size_bytes;const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);if (start_row >= end_row) return;- FARF(HIGH, "binary-scalar-dma: %d/%d (%u:%u) row-size %u (%u)",+ FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)",ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout;@@ -329,94 +308,6 @@ static void binary_thread_scalar_dma(unsigned int nth, unsigned int ith, void *spad_idx ^= 1;}- struct htp_thread_trace * tr = &octx->ctx->trace[ith];- compute_scalar_dma_t compute = (compute_scalar_dma_t) bctx->compute;-- for (uint32_t ir = start_row; ir < end_row; ) {- uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);-- uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src;- uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst;-- uint32_t i03, i02, i01, rem;- i03 = fastdiv(ir, &bctx->src0_dim12_div);- rem = ir - i03 * (ne02 * ne01);- i02 = fastdiv(rem, &bctx->src0_dim1_div);- i01 = rem - i02 * ne01;-- uint32_t cur_i11 = fastmodulo(i01, ne11, &bctx->src1_dim1_div);-- htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);- compute(d_spad, s0_spad, s1_table, cur_i11, ne11, current_block_size,- bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00);- htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);-- dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;- dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes,- current_block_size);-- if (ir_prefetch < end_row) {- uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);- uint32_t p03, p02, p01, prem;- p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div);- prem = ir_prefetch - p03 * (ne02 * ne01);- p02 = fastdiv(prem, &bctx->src0_dim1_div);- p01 = prem - p02 * ne01;- dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01;- dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes,- next_block_size);- ir_prefetch += next_block_size;- }- ir += current_block_size;- }-- dma_queue_flush(dma_q);-}--// 1b. Scalar src1 dynamic / pointer (ne10 == 1)-static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data) {- struct htp_binary_context * bctx = (struct htp_binary_context *) data;- struct htp_ops_context * octx = bctx->octx;- htp_binary_preamble;-- const uint32_t row_size_bytes = bctx->row_size_bytes;- const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;- const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);- if (start_row >= end_row) return;-- FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)",- ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);-- const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout;- uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread);- uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);- size_t src0_spad_half = layout->src0_spad_half_size;- size_t dst_spad_half = layout->dst_spad_half_size;-- dma_queue * dma_q = octx->ctx->dma[ith];- uint32_t ir_prefetch = start_row;- int spad_idx = 0;-- for (int k = 0; k < 2 && ir_prefetch < end_row; k++) {- uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);- uint32_t i03, i02, i01, rem;- i03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div);- rem = ir_prefetch - i03 * (ne02 * ne01);- i02 = fastdiv(rem, &bctx->src0_dim1_div);- i01 = rem - i02 * ne01;-- dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01;- dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;-- uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half;- uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half;-- dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0);- dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size);- ir_prefetch += current_block_size;- spad_idx ^= 1;- }-struct htp_thread_trace * tr = &octx->ctx->trace[ith];compute_scalar_t compute = (compute_scalar_t) bctx->compute;@@ -435,17 +326,17 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * datauint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div);uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div);uint32_t i11 = fastmodulo(i01, ne11, &bctx->src1_dim1_div);-- const uint8_t * src1_ptr = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11;- uint32_t s1_stride = (ne11 == 1) ? 0 : nb11;+ const uint8_t * scalar_table = (const uint8_t *) s1_table ++ i13 * plane_stride + i12 * row_stride;htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);- compute(d_spad, s0_spad, src1_ptr, s1_stride, current_block_size,+ compute(d_spad, s0_spad, scalar_table, i11, ne11, current_block_size,bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;- dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size);+ dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes,+ current_block_size);if (ir_prefetch < end_row) {uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02);@@ -455,7 +346,8 @@ static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * datap02 = fastdiv(prem, &bctx->src0_dim1_div);p01 = prem - p02 * ne01;dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01;- dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size);+ dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes,+ next_block_size);ir_prefetch += next_block_size;}ir += current_block_size;@@ -668,6 +560,12 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voiuint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);size_t src0_spad_half = layout->src0_spad_half_size;size_t dst_spad_half = layout->dst_spad_half_size;+ const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1);++ const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16);+ const size_t vtcm_row_stride = (size_t) ne10 * elem_size;+ const size_t vtcm_plane_stride = (size_t) ne11 * vtcm_row_stride;+ const size_t vtcm_vol_stride = (size_t) ne12 * vtcm_plane_stride;dma_queue * dma_q = octx->ctx->dma[ith];uint32_t ir_prefetch = start_row;@@ -707,10 +605,10 @@ static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, voiuint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div);uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div);- const uint8_t * src1_plane = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12;+ const uint8_t * src1_plane = vtcm_src1 + i13 * vtcm_vol_stride + i12 * vtcm_plane_stride;htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);- compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, nb11,+ compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, vtcm_row_stride,current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);@@ -750,6 +648,12 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voiuint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);size_t src0_spad_half = layout->src0_spad_half_size;size_t dst_spad_half = layout->dst_spad_half_size;+ const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1);++ const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16);+ const size_t vtcm_row_stride = (size_t) ne10 * elem_size;+ const size_t vtcm_plane_stride = (size_t) ne11 * vtcm_row_stride;+ const size_t vtcm_vol_stride = (size_t) ne12 * vtcm_plane_stride;FARF(HIGH, "binary-repeat: %d/%d (%u:%u) row-size %u (%u)",ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);@@ -792,10 +696,10 @@ static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, voiuint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div);uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div);- const uint8_t * src1_plane = (const uint8_t *)(uintptr_t) src1->data + i13 * nb13 + i12 * nb12;+ const uint8_t * src1_plane = vtcm_src1 + i13 * vtcm_vol_stride + i12 * vtcm_plane_stride;htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);- compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, nb11,+ compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, vtcm_row_stride,current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00, ne10);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);@@ -826,7 +730,6 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void *const struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * src1 = octx->src[1];- const struct htp_tensor * src2 = octx->src[2];const struct htp_tensor * dst = octx->dst;const uint32_t ne00 = src0->ne[0];@@ -852,6 +755,7 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void *uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread);uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1);+ const int32_t * vtcm_src2 = VTCM_LAYOUT_PTR(const int32_t, bctx->vtcm_base, layout->off_src2);size_t src0_spad_half = layout->src0_spad_half_size;size_t dst_spad_half = layout->dst_spad_half_size;@@ -892,8 +796,8 @@ static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void *htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);compute_add_id_t compute = (compute_add_id_t) bctx->compute;- compute(d_spad, s0_spad, vtcm_src1, (const char *)(uintptr_t)src2->data,- i01, i02, src2->nb[0], src2->nb[1], src1_stride,+ compute(d_spad, s0_spad, vtcm_src1, vtcm_src2,+ i01, i02, ne01, src1_stride,current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);@@ -1234,19 +1138,6 @@ static int execute_op_binary(struct htp_ops_context * octx) {const size_t src1_row_size_aligned = kparams->src1_row_size_aligned;const size_t dst_row_size_aligned = kparams->dst_row_size_aligned;- if (htp_tensor_is_extended(src1)) {- if (kparams->kernel_type != HTP_BINARY_KERNEL_SAME_SHAPE &&- kparams->kernel_type != HTP_BINARY_KERNEL_ROW_BCAST &&- kparams->kernel_type != HTP_BINARY_KERNEL_SCALAR_DMA &&- kparams->kernel_type != HTP_BINARY_KERNEL_ADD_ID) {- return HTP_STATUS_NO_SUPPORT;- }- }-- if (octx->op == HTP_OP_ADD_ID && htp_tensor_is_extended(octx->src[2])) {- return HTP_STATUS_NO_SUPPORT;- }-struct htp_binary_context bctx;bctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base;htp_binary_vtcm_layout_build(&bctx.vtcm_layout, kparams, octx->ctx->vtcm_size);@@ -1259,12 +1150,38 @@ static int execute_op_binary(struct htp_ops_context * octx) {uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src1);if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[0] * elem_size, 1);- } else if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR_DMA) {- dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[1] * elem_size, 1);+ } else if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR ||+ kparams->kernel_type == HTP_BINARY_KERNEL_REPEAT ||+ kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX) {+ if (!kparams->src1_strided) {+ const size_t src1_bytes = (size_t) src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3] * elem_size;+ dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1_bytes, 1);+ } else {+ const uint32_t ne11 = src1->ne[1];+ const uint32_t ne12 = src1->ne[2];+ const uint32_t ne13 = src1->ne[3];+ const size_t row_bytes = src1->ne[0] * elem_size;+ for (uint32_t i13 = 0; i13 < ne13; i13++) {+ for (uint32_t i12 = 0; i12 < ne12; i12++) {+ dma_addr_t src_addr = src1->data + i13 * src1->nb[3] + i12 * src1->nb[2];+ uint8_t * dst_ptr = vtcm_src1 + (i13 * ne12 * ne11 + i12 * ne11) * row_bytes;+ while (!dma_queue_push(dma_q, dma_make_data(dst_ptr, src_addr),+ row_bytes, src1->nb[1], row_bytes, ne11)) {+ dma_queue_pop(dma_q);+ }+ }+ }+ }} else if (kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) {+ const struct htp_tensor * src2 = octx->src[2];+ uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2);dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data),kparams->src1_row_size_aligned, src1->nb[1],src1->ne[0] * elem_size, src1->ne[1]);+ const size_t src2_row_bytes = src2->ne[0] * sizeof(int32_t);+ dma_queue_push(dma_q, dma_make_data(vtcm_src2, src2->data),+ src2_row_bytes, src2->nb[1],+ src2_row_bytes, src2->ne[1]);}bctx.octx = octx;@@ -1285,15 +1202,8 @@ static int execute_op_binary(struct htp_ops_context * octx) {bctx.src1_dim2_div = init_fastdiv_values(src1->ne[2]);bctx.src1_dim3_div = init_fastdiv_values(src1->ne[3]);- bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]);- bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]);-- bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]);- bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]);-- bctx.split_at_ne01 = (octx->op == HTP_OP_ADD_ID) ||- ((src0->ne[2] > 1) && ((src1->ne[1] > 1) || (src1->ne[2] > 1) || !src0_contig_dim1 || !dst_contig_dim1));- bctx.split_at_ne02 = (src0->ne[3] > 1) && ((src1->ne[2] > 1) || (src1->ne[3] > 1) || !src0_contig_dim2 || !dst_contig_dim2);+ bctx.split_at_ne01 = kparams->split_at_ne01;+ bctx.split_at_ne02 = kparams->split_at_ne02;worker_callback_t worker_func = NULL;void * compute_func = NULL;@@ -1339,26 +1249,6 @@ static int execute_op_binary(struct htp_ops_context * octx) {}}break;- case HTP_BINARY_KERNEL_SCALAR_DMA:- worker_func = binary_thread_scalar_dma;- if (src0_type == HTP_TYPE_F32) {- switch (octx->op) {- case HTP_OP_ADD: compute_func = compute_scalar_dma_add_f32; break;- case HTP_OP_SUB: compute_func = compute_scalar_dma_sub_f32; break;- case HTP_OP_MUL: compute_func = compute_scalar_dma_mul_f32; break;- case HTP_OP_DIV: compute_func = compute_scalar_dma_div_f32; break;- default: break;- }- } else if (src0_type == HTP_TYPE_F16) {- switch (octx->op) {- case HTP_OP_ADD: compute_func = compute_scalar_dma_add_f16; break;- case HTP_OP_SUB: compute_func = compute_scalar_dma_sub_f16; break;- case HTP_OP_MUL: compute_func = compute_scalar_dma_mul_f16; break;- case HTP_OP_DIV: compute_func = compute_scalar_dma_div_f16; break;- default: break;- }- }- break;case HTP_BINARY_KERNEL_SCALAR:worker_func = binary_thread_scalar;if (src0_type == HTP_TYPE_F32) {@@ -1435,9 +1325,11 @@ static int execute_op_binary(struct htp_ops_context * octx) {bctx.compute = compute_func;if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST ||- kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR_DMA ||+ kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR ||+ kparams->kernel_type == HTP_BINARY_KERNEL_REPEAT ||+ kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX ||kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) {- dma_queue_pop(dma_q);+ dma_queue_flush(dma_q);}work_queue_run(octx->ctx->work_queue, worker_func, &bctx, n_threads);@@ -1446,13 +1338,6 @@ static int execute_op_binary(struct htp_ops_context * octx) {}int op_binary(struct htp_ops_context * octx) {-- // Does not support permutations of src1- const struct htp_tensor * src1 = octx->src[1];- if (src1->nb[1] < src1->nb[0]) {- return HTP_STATUS_NO_SUPPORT;- }-const uint32_t src0_type = octx->src[0]->type;if ((src0_type == HTP_TYPE_F32) || (src0_type == HTP_TYPE_F16)) {return execute_op_binary(octx);diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.h b/ggml/src/ggml-hexagon/htp/binary-ops.hindex aa8ff49599..db3d514895 100644--- a/ggml/src/ggml-hexagon/htp/binary-ops.h+++ b/ggml/src/ggml-hexagon/htp/binary-ops.h@@ -12,7 +12,6 @@enum htp_binary_kernel_type {HTP_BINARY_KERNEL_SAME_SHAPE = 0,HTP_BINARY_KERNEL_ROW_BCAST,- HTP_BINARY_KERNEL_SCALAR_DMA,HTP_BINARY_KERNEL_SCALAR,HTP_BINARY_KERNEL_ADD_ID,HTP_BINARY_KERNEL_COMPLEX,@@ -30,15 +29,41 @@ struct htp_binary_kernel_params {uint32_t dst_row_size_aligned;uint32_t src1_size;+ uint32_t src2_size;uint32_t vtcm_size;uint32_t chunk_size;uint32_t chunk_bytes;uint32_t is_scalar;++ uint8_t split_at_ne01;+ uint8_t split_at_ne02;+ uint8_t src1_strided;+ uint8_t pad;};#if defined(__cplusplus)static_assert(sizeof(struct htp_binary_kernel_params) <= 128, "htp_binary_kernel_params is too large for kernel_params blob");++struct ggml_tensor;+static inline bool htp_binary_strides_safe(+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * dst,+ size_t elem_size+) {+ if ((src0->ne[0] != 1 && src0->nb[0] != elem_size) ||+ (src1->ne[0] != 1 && src1->nb[0] != elem_size) ||+ (dst->ne[0] != 1 && dst->nb[0] != elem_size)) {+ return false;+ }+ if ((src0->ne[1] > 1 && src0->nb[1] < src0->ne[0] * elem_size) ||+ (src1->ne[1] > 1 && src1->nb[1] < src1->ne[0] * elem_size) ||+ (dst->ne[1] > 1 && dst->nb[1] < dst->ne[0] * elem_size)) {+ return false;+ }+ return true;+}#else_Static_assert(sizeof(struct htp_binary_kernel_params) <= 128, "htp_binary_kernel_params is too large for kernel_params blob");#endif@@ -47,6 +72,7 @@ struct htp_binary_vtcm_layout {size_t total_bytes;size_t off_src0;size_t off_src1;+ size_t off_src2;size_t off_dst;size_t src0_bytes_per_thread;@@ -58,6 +84,7 @@ struct htp_binary_vtcm_layout {size_t dst_spad_half_size;size_t src1_size;+ size_t src2_size;uint32_t rows_per_buffer;};@@ -111,17 +138,19 @@ static inline void htp_binary_vtcm_layout_build(? 2 * (kparams->src0_row_size_aligned + kparams->src1_row_size_aligned + kparams->dst_row_size_aligned): 2 * (kparams->src0_row_size_aligned + kparams->dst_row_size_aligned);- if (spad_row_total == 0 || vtcm_size < kparams->src1_size) {+ const size_t static_vtcm_size = kparams->src1_size + kparams->src2_size;+ if (spad_row_total == 0 || vtcm_size < static_vtcm_size) {return;}- const size_t rows_per_buffer = (vtcm_size - kparams->src1_size) / (n_threads * spad_row_total);+ const size_t rows_per_buffer = (vtcm_size - static_vtcm_size) / (n_threads * spad_row_total);if (rows_per_buffer == 0) {return;}L->rows_per_buffer = (uint32_t) rows_per_buffer;L->src1_size = kparams->src1_size;+ L->src2_size = kparams->src2_size;L->src0_bytes_per_thread = rows_per_buffer * 2 * kparams->src0_row_size_aligned;L->dst_bytes_per_thread = rows_per_buffer * 2 * kparams->dst_row_size_aligned;@@ -137,11 +166,13 @@ static inline void htp_binary_vtcm_layout_build(const size_t src1_total = (kparams->src1_size > 0)? kparams->src1_size: n_threads * L->src1_bytes_per_thread;+ const size_t src2_total = kparams->src2_size;const size_t dst_total = n_threads * L->dst_bytes_per_thread;size_t off = 0;VTCM_LAYOUT_ALLOC(off, off_src0, src0_total);VTCM_LAYOUT_ALLOC(off, off_src1, src1_total);+ VTCM_LAYOUT_ALLOC(off, off_src2, src2_total);VTCM_LAYOUT_ALLOC(off, off_dst, dst_total);L->total_bytes = off;diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex bd599e99e5..b54abbea83 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -3303,6 +3303,7 @@ struct test_bin_bcast : public test_case {bool perm1; // permute src1?bool src_overlap; // src0 and src1 are overlapping views of the same bufferconst ggml_type type_b; // type of src1, GGML_TYPE_COUNT -> same as src0+ bool weights;bool run_whole_graph() override { return nf > 1; }@@ -3311,6 +3312,9 @@ struct test_bin_bcast : public test_case {if (type_b != GGML_TYPE_COUNT) {s += "," + VAR_TO_STR(type_b);}+ if (weights) {+ s += ",weights=1";+ }return s;}@@ -3323,28 +3327,38 @@ struct test_bin_bcast : public test_case {std::array<int, 4> nr = {1, 2, 1, 1},int nf = 1,bool perm1 = false, bool src_overlap = false,- ggml_type type_b = GGML_TYPE_COUNT)- : op(op), type(type), ne(ne), nr(nr), nf(nf), perm1(perm1), src_overlap(src_overlap), type_b(type_b) {}+ ggml_type type_b = GGML_TYPE_COUNT, bool weights = false)+ : op(op), type(type), ne(ne), nr(nr), nf(nf), perm1(perm1), src_overlap(src_overlap), type_b(type_b), weights(weights) {}++ bool use_weight_context() override {+ return weights;+ }ggml_tensor * build_graph(ggml_context * ctx) override {+ return build_graph(ctx, ctx);+ }++ ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override {GGML_ASSERT(nf <= 16);ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0]*nr[0], ne[1]*nr[1], ne[2]*nr[2], ne[3]*nr[3]);ggml_set_name(a, "a");const ggml_type tb = type_b == GGML_TYPE_COUNT ? type : type_b;+ ggml_context * ctx_b = weights ? ctx_weights : ctx;+ GGML_ASSERT(ctx_b);ggml_tensor * b[16];for (int i = 0; i < nf; ++i) {if (perm1) {const int p[4] = { 1, 2, 0, 3 }; // hardcoded for now- b[i] = ggml_new_tensor_4d(ctx, tb, ne[p[0]], ne[p[1]], ne[p[2]], ne[p[3]]);- b[i] = ggml_permute(ctx, b[i], p[0], p[1], p[2], p[3]);+ b[i] = ggml_new_tensor_4d(ctx_b, tb, ne[p[0]], ne[p[1]], ne[p[2]], ne[p[3]]);+ b[i] = ggml_permute(ctx_b, b[i], p[0], p[1], p[2], p[3]);} else if (src_overlap) {b[i] = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], 2 * (ne[3] / 3), a->nb[1], a->nb[2], a->nb[3], (ne[3] / 3) * a->nb[3]);} else {- b[i] = ggml_new_tensor(ctx, tb, 4, ne.data());+ b[i] = ggml_new_tensor(ctx_b, tb, 4, ne.data());}ggml_set_name(b[i], (std::string("b") + std::to_string(i)).c_str());}@@ -10387,6 +10401,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {//add_test_bin_bcast(type, {3, 3, 2560, 1280}, {2, 1, 1, 1});}+ for (auto op : {ggml_add, ggml_mul}) {+ test_cases.emplace_back(new test_bin_bcast(op, GGML_TYPE_F32, {1, 2, 2, 2}, {17, 1, 1, 1}, 1, false, false, GGML_TYPE_COUNT, true));+ }+ test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {2, 2, 2, 2}, {17, 1, 1, 1}, 1, false, false, GGML_TYPE_COUNT, true));+// single inplace tests, especially important for WebGPU backend since kernels for inplace vs. not are differenttest_cases.emplace_back(new test_bin_bcast(ggml_add_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16));test_cases.emplace_back(new test_bin_bcast(ggml_mul_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16));