LLM inference in C/C++
C++ ggml
hexagon: Op Fusion refactoring, optimizations and new fusions like `BINARY_NX` (back-to-back ADDs, etc) (#30843)
Max Krasnyansky643b0cdacb4c
diff --git a/ggml/src/ggml-hexagon/CMakeLists.txt b/ggml/src/ggml-hexagon/CMakeLists.txtindex 6ab520175b..9444f73213 100644--- a/ggml/src/ggml-hexagon/CMakeLists.txt+++ b/ggml/src/ggml-hexagon/CMakeLists.txt@@ -51,6 +51,13 @@ endif()set(TARGET_NAME ggml-hexagon)ggml_add_backend_library(${TARGET_NAME}ggml-hexagon.cpp+ htp-common.h+ htp-kparams.cpp+ htp-kparams.h+ htp-opnode.cpp+ htp-opnode.h+ htp-fusion.cpp+ htp-fusion.hhtp-repack.cpphtp-repack.hhtp-drv.cppdiff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex da6dba85e8..0821bce16b 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -26,10 +26,6 @@#include <cmath>#include <initializer_list>-#ifndef M_LOG2E-# define M_LOG2E 1.44269504088896340736-#endif-#ifdef _WIN32# define WIN32_LEAN_AND_MEAN# ifndef NOMINMAX@@ -59,25 +55,13 @@#include "ggml-impl.h"#include "ggml-quants.h"#include "htp-opnode.h"-#include "htp-ops.h"-#include "htp/matmul-ops.h"-#include "htp/binary-ops.h"-#include "htp/flash-attn-ops.h"-#include "htp/unary-ops.h"-#include "htp/get-rows-ops.h"-#include "htp/set-rows-ops.h"-#include "htp/softmax-ops.h"-#include "htp/pool-ops.h"-#include "htp/rope-ops.h"-#include "htp/ssm-conv.h"-#include "htp/gated-delta-net-ops.h"-#include "htp/argsort-ops.h"-#include "htp/concat-ops.h"-#include "htp/cpy-ops.h"-#include "htp/im2col-ops.h"+#include "htp/htp-ops.h"#include "htp_iface.h"#include "htp-drv.h"#include "htp-repack.h"+#include "htp-common.h"+#include "htp-kparams.h"+#include "htp-fusion.h"using intvec = std::vector<int>;using uintvec = std::vector<unsigned int>;@@ -85,8 +69,6 @@ using u32vec = std::vector<uint32_t>;#define GGML_HEXAGON_MAX_SESSIONS 16-#define GGML_HEXAGON_FENCE_SLOT_SIZE 128-struct ggml_hexagon_device_config {int physical_idx = 0;int virtual_idx = 0;@@ -99,17 +81,17 @@ struct ggml_hexagon_device_config {static ggml_hexagon_device_config opt_device_configs[GGML_HEXAGON_MAX_SESSIONS];-static int opt_arch = 0; // autodetect-static size_t opt_ndev = 1;-static size_t opt_nhvx = 0; // use all-static int opt_nhmx = 1; // when set, enable HMX; when 0, use HVX only-static size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for buffer mappings-static int opt_etm = 0;-static int opt_verbose = 0;-static int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu)-static bool opt_hostbuf = false;-static bool opt_dma64 = false;-static int opt_power_mode = HTP_POWER_MODE_BURST; // DCVS/HMX power profile requested at session start+int opt_arch = 0; // autodetect+size_t opt_ndev = 1;+size_t opt_nhvx = 0; // use all+int opt_nhmx = 1; // when set, enable HMX; when 0, use HVX only+size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for buffer mappings+int opt_etm = 0;+int opt_verbose = 0;+int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu)+bool opt_hostbuf = false;+bool opt_dma64 = false;+int opt_power_mode = HTP_POWER_MODE_BURST; // DCVS/HMX power profile requested at session startstatic const std::pair<const char *, int> htp_power_mode_names[] = {{ "low_power_saver", HTP_POWER_MODE_LOW_POWER_SAVER },@@ -145,49 +127,31 @@ static const char * htp_power_mode_to_str(int mode) {return "unknown";}-static size_t opt_mbuf_dyn = 512ul * 1024 * 1024; // max dynamic (compute) buffer size-static size_t opt_mbuf_static = 1ul * 1024 * 1024 * 1024; // max static (weight/KV) buffer size-static size_t opt_mbuf_total = 0; // total buffer space limit (0 = unconstrained)+size_t opt_mbuf_dyn = 512ul * 1024 * 1024; // max dynamic (compute) buffer size+size_t opt_mbuf_static = 1ul * 1024 * 1024 * 1024; // max static (weight/KV) buffer size+size_t opt_mbuf_total = 0; // total buffer space limit (0 = unconstrained)-static int opt_mm_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)-static int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)-static int opt_fa_head_split = 1; // 1 = partition flash_attn by KV heads in multicore (default on), 0 = token-based (original)-static int opt_gdn_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported)-static int opt_ar_select = 2; // 2 = fused ALLREDUCE+ADD (default), 1 = unfused ALLREDUCE, 0 = fallback to CPY+FENCE-static int opt_ar_scatter = 1; // 1 = reduce-scatter the fused ALLREDUCE+ADD (default), 0 = full reduction+int opt_mm_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)+int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)+int opt_fa_head_split = 1; // 1 = partition flash_attn by KV heads in multicore (default on), 0 = token-based (original)+int opt_gdn_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported)+int opt_ar_select = 2; // 2 = fused ALLREDUCE+ADD (default), 1 = unfused ALLREDUCE, 0 = fallback to CPY+FENCE+int opt_ar_scatter = 1; // 1 = reduce-scatter the fused ALLREDUCE+ADD (default), 0 = full reduction// Default PMU events, if profiling with PMU (mode=2) is enabled// See https://docs.qualcomm.com/doc/80-N2040-60/topic/pmu-events.html// https://docs.qualcomm.com/doc/80-N2040-61/topic/hvx-pmu-events.htmlstatic u32vec opt_pmu_evt { 0x3, 0x111, 0x100, 0x105, 0x240, 0x256, 0x7D, 0x8C };-static int opt_opbatch = 1280; // max number of ops in a batch-static int opt_opqueue = 8; // max number of pending batches-static int opt_optrace = 0; // trace buffer size per thread (0 means default)-static int opt_oppoll = 0; // polling for batch completions-static int opt_opfusion = 1; // enable/disable op fusion-static int opt_rt_repack = 1; // runtime repacking: 0 = disabled, 1 = quantized only (default), 2 = always--enum ggml_hexagon_fusion_flags {- GGML_HEXAGON_FUSE_ALLREDUCE_ADD = (1 << 1), // 2- GGML_HEXAGON_FUSE_RMS_NORM_MUL = (1 << 2), // 4- GGML_HEXAGON_FUSE_MUL_MAT_ADD = (1 << 3), // 8- GGML_HEXAGON_FUSE_MUL_MAT_NX = (1 << 4), // 16- GGML_HEXAGON_FUSE_MUL_MAT_ID_NX = (1 << 5), // 32- GGML_HEXAGON_FUSE_GDN_CPY = (1 << 6), // 64-};--static inline bool ggml_hexagon_is_fusion_enabled(int flag) {- if (opt_opfusion <= 0) return false;- if (opt_opfusion == 1) return true; // 1 enables all- return (opt_opfusion & flag) != 0;-}+int opt_opbatch = 2048; // max number of ops in a batch+int opt_opqueue = 8; // max number of pending batches+int opt_optrace = 0; // trace buffer size per thread (0 means default)+int opt_oppoll = 0; // polling for batch completions+int opt_opfusion = 1; // enable/disable op fusion+int opt_rt_repack = 1; // runtime repacking: 0 = disabled, 1 = quantized only (default), 2 = alwaysstatic std::regex* opt_opfilter = NULL; // regex of ops to not claim-#define HEX_VERBOSE(...) \- if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__)-static const char * status_to_str(uint32_t status) {switch (status) {case HTP_STATUS_OK:@@ -207,20 +171,18 @@ static const char * status_to_str(uint32_t status) {// ** debug helpers-static void ggml_hexagon_dump_op_exec(const std::string &sess_name, const htp_opnode & node, const uint32_t req_flags) {+static void ggml_hexagon_dump_op_exec(const char * sess_name, const htp_opnode & node, const uint32_t req_flags) {if (!opt_verbose) return;- htp_opformat fmt(node);- GGML_LOG_DEBUG("ggml-hex: %s execute-op %s|%s|%s|%s|%s|%s|%s|flags 0x%x\n", sess_name.c_str(),- node.op_name().c_str(), fmt.names, fmt.dims, fmt.types, fmt.strides, fmt.buffs, fmt.kparams, req_flags);+ GGML_LOG_DEBUG("ggml-hex: %s execute-op %s|flags 0x%x\n",+ sess_name, htp_format_op(node, HTP_OPFMT_EXEC).c_str(), req_flags);}-static void ggml_hexagon_dump_op_supp(const std::string &sess_name, const struct ggml_tensor * op, bool supp) {+static void ggml_hexagon_dump_op_supp(const char * sess_name, const struct ggml_tensor * op, bool supp) {if (!opt_verbose) return;- htp_opformat fmt(htp_opformat(htp_opnode(HTP_OP_INVALID, const_cast<ggml_tensor*>(op))));- GGML_LOG_DEBUG("ggml-hex: %s supports-op %s|%s|%s|%s|%s|%s|%s\n", sess_name.c_str(),- ggml_op_desc(op), fmt.names, fmt.dims, fmt.types, fmt.strides, fmt.buffs, supp ? "yes" : "no");+ GGML_LOG_DEBUG("ggml-hex: %s supports-op %s|%s\n",+ sess_name, htp_format_op(op, HTP_OPFMT_SUPP).c_str(), supp ? "yes" : "no");}static const char * htp_event_name(uint16_t id) {@@ -253,7 +215,7 @@ static const char * htp_event_name(uint16_t id) {}}-static void ggml_hexagon_dump_op_prof(const std::string &sess_name, const htp_opnode & node, const htp_prof_desc & pd) {+static void ggml_hexagon_dump_op_prof(const char * sess_name, const htp_opnode & node, const htp_prof_desc & pd) {if (!opt_profile) return;uint32_t op_usec = pd.usecs;@@ -267,13 +229,12 @@ static void ggml_hexagon_dump_op_prof(const std::string &sess_name, const htp_oppmu[0], pmu[1], pmu[2], pmu[3], pmu[4], pmu[5], pmu[6], pmu[7]);}- htp_opformat fmt(node);float mhz = op_usec > 0 ? (float) op_cycles / op_usec : 0.0f;- GGML_LOG_DEBUG("ggml-hex: %s profile-op %s|%s|%s|%s|%s|%s|usec %u cycles %u start %u mhz %.1f%s\n", sess_name.c_str(),- node.op_name().c_str(), fmt.names, fmt.dims, fmt.types, fmt.strides, fmt.kparams, op_usec, op_cycles, pd.cycles_start, mhz, pmu_str);+ GGML_LOG_DEBUG("ggml-hex: %s profile-op %s|usec %u cycles %u start %u mhz %.1f%s\n",+ sess_name, htp_format_op(node, HTP_OPFMT_PROF).c_str(), op_usec, op_cycles, pd.cycles_start, mhz, pmu_str);}-static void ggml_hexagon_dump_batch_prof(const std::string & sess_name, const htp_opbatch_rsp & rsp) {+static void ggml_hexagon_dump_batch_prof(const char * sess_name, const htp_opbatch_rsp & rsp) {uint64_t batch_cycles = rsp.cycles_stop - rsp.cycles_start;float batch_mhz = rsp.usecs > 0 ? (float) batch_cycles / rsp.usecs : 0.0f;@@ -286,10 +247,10 @@ static void ggml_hexagon_dump_batch_prof(const std::string & sess_name, const ht}GGML_LOG_DEBUG("ggml-hex: %s profile-op OPBATCH|----|n-ops %u|%s|----|----|usec %u cycles %llu start %llu mhz %.1f\n",- sess_name.c_str(), rsp.n_ops, evt_str, rsp.usecs, (unsigned long long) batch_cycles, (unsigned long long) rsp.cycles_start, batch_mhz);+ sess_name, rsp.n_ops, evt_str, rsp.usecs, (unsigned long long) batch_cycles, (unsigned long long) rsp.cycles_start, batch_mhz);}-static void ggml_hexagon_dump_trace_events(const std::string & sess_name, const htp_opbatch_rsp & rsp,+static void ggml_hexagon_dump_trace_events(const char * sess_name, const htp_opbatch_rsp & rsp,const htp_trace_desc * trace_events, uint32_t n_traces) {if (opt_profile == 3 && trace_events) {uint32_t valid_cnt[HTP_MAX_NTHREADS + 1] = {0};@@ -304,237 +265,14 @@ static void ggml_hexagon_dump_trace_events(const std::string & sess_name, constbool is_stop = (e.info & 0x8000) != 0;uint16_t info = e.info & 0x7FFF;GGML_LOG_DEBUG("ggml-hex: %s trace-evt %s: thread %u info %u %s %u\n",- sess_name.c_str(), htp_event_name(e.id), t, info, is_stop ? "stop" : "start", e.cycles);+ sess_name, htp_event_name(e.id), t, info, is_stop ? "stop" : "start", e.cycles);}}}}-enum ggml_hexagon_tensor_flags {- GGML_HEXAGON_TENSOR_TILED = (1 << 0),- GGML_HEXAGON_TENSOR_WEIGHT = (1 << 1),- GGML_HEXAGON_TENSOR_FENCE = (1 << 2),- GGML_HEXAGON_TENSOR_FUSEABLE = (1 << 3),-};--struct ggml_hexagon_session;--static void ggml_hexagon_precompute_matmul_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * dst,- struct htp_mm_kernel_params * kparams-);--static void ggml_hexagon_precompute_fused_matmul_add_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * src2,- const struct ggml_tensor * dst,- struct htp_mm_kernel_params * kparams-);--static void ggml_hexagon_precompute_unary_params(- const struct ggml_hexagon_session * sess,- uint32_t op,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * dst,- struct htp_unary_kernel_params * kparams-);--static bool ggml_hexagon_precompute_binary_params(- const struct ggml_hexagon_session * sess,- uint32_t op,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * dst,- struct htp_binary_kernel_params * kparams,- const struct ggml_tensor * src2 = nullptr-);--static void ggml_hexagon_precompute_get_rows_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * dst,- struct htp_get_rows_kernel_params * kparams-);--static void ggml_hexagon_precompute_set_rows_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * dst,- struct htp_set_rows_kernel_params * kparams-);--static void ggml_hexagon_precompute_softmax_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_softmax_kernel_params * kparams-);--static void ggml_hexagon_precompute_rope_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_rope_kernel_params * kparams-);--static void ggml_hexagon_precompute_ssm_conv_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * dst,- struct htp_ssm_conv_kernel_params * kparams-);--static void ggml_hexagon_precompute_gated_delta_net_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_gdn_kernel_params * kparams-);--static void ggml_hexagon_precompute_sort_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- bool is_top_k,- struct htp_sort_kernel_params * kparams-);--static void ggml_hexagon_precompute_pool_2d_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * dst,- struct htp_pool_2d_kernel_params * kparams,- bool is_pool_1d-);--static bool ggml_hexagon_precompute_concat_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_concat_kernel_params * kparams-);--static bool ggml_hexagon_precompute_cpy_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_copy_kernel_params * kparams-);--static bool ggml_hexagon_precompute_im2col_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_im2col_kernel_params * kparams-);--static bool ggml_hexagon_precompute_im2col_3d_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_im2col_kernel_params * kparams-);--static void ggml_hexagon_precompute_fused_mmnx_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * act,- int32_t n_weights,- struct htp_mm_kernel_params * kparams-);--static void ggml_hexagon_precompute_fused_mmidnx_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * act,- const struct ggml_tensor * dst,- int32_t n_weights,- struct htp_mm_kernel_params * kparams-);--static bool ggml_hexagon_precompute_allreduce_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * dst,- uint32_t rank,- uint32_t n_ranks,- bool has_add,- bool is_row_bcast,- bool is_shard_ok,- struct htp_allreduce_kernel_params * kparams-);--static bool ggml_hexagon_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride);-static bool ggml_hexagon_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst);-static ggml_tensor ggml_hexagon_tensor_collapse_rows(const struct ggml_tensor * t);--static bool mm_is_hmx_eligible(const ggml_tensor * t);-static htp_op_code op_remap_to_htp(const ggml_tensor * t);-static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams);-static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams);-static bool is_mergeable_mul_mat(const ggml_tensor * t);-static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor * n2);-static bool is_mergeable_mul_mat_id(const ggml_tensor * t);-static bool is_mergeable_mul_mat_id_pair(const ggml_tensor * n1, const ggml_tensor * n2);-// ** backend sessions-struct ggml_hexagon_tensor_extra {- std::vector<uint8_t> shadow_buf;- size_t shadow_size { 0 };- uint32_t flags { 0 };-};--static inline bool ggml_hexagon_tensor_is_fuseable(const struct ggml_tensor * t) {- if (!t->extra) return false;- auto extra = (const struct ggml_hexagon_tensor_extra *) t->extra;- return (extra->flags & GGML_HEXAGON_TENSOR_FUSEABLE) != 0;-}--static inline bool ggml_hexagon_can_rt_repack(const struct ggml_tensor * t) {- if (opt_rt_repack == 0 || !htp_is_tiled_weight_tensor(t)) {- return false;- }- if (t->buffer && ggml_backend_buft_is_host(t->buffer->buft)) {- return false;- }- return opt_rt_repack > 1 || ggml_is_quantized(t->type);-}--static inline bool ggml_hexagon_tensor_is_tiled(const struct ggml_tensor * t) {- const auto * extra = (const ggml_hexagon_tensor_extra *) t->extra;- if (extra && (extra->flags & GGML_HEXAGON_TENSOR_TILED)) {- return true;- }- return ggml_hexagon_can_rt_repack(t);-}--static inline bool ggml_hexagon_needs_repack(const struct ggml_tensor * t) {- const auto * extra = (const ggml_hexagon_tensor_extra *) t->extra;- if (extra && (extra->flags & GGML_HEXAGON_TENSOR_TILED)) {- return false;- }- return ggml_hexagon_can_rt_repack(t);-}--static inline bool ggml_hexagon_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) {- const uintptr_t a0 = (uintptr_t) a->data;- const uintptr_t b0 = (uintptr_t) b->data;- const uintptr_t a1 = a0 + ggml_nbytes(a);- const uintptr_t b1 = b0 + ggml_nbytes(b);-- return a0 < b1 && b0 < a1;-}--static inline bool ggml_hexagon_can_row_partition(const struct ggml_tensor * t) {- if (t->ne[1] > 1 && (t->nb[1] & 127) != 0) return false;- if (t->ne[2] > 1 && (t->nb[2] & 127) != 0) return false;- if (t->ne[3] > 1 && (t->nb[3] & 127) != 0) return false;- return true;-}--struct htp_opnode;-struct ggml_hexagon_opbatch;struct ggml_hexagon_opqueue;struct ggml_hexagon_shared_buffer;@@ -611,8 +349,13 @@ struct ggml_hexagon_session {void allocate(const ggml_hexagon_device_config & config) noexcept(false);void release() noexcept(true);- uint8_t * alloc_fence(uint32_t n_slots = 1);- void free_fence(void * ptr, uint32_t n_slots = 1);+ uint8_t * alloc_fence(uint32_t n_slots = 1);+ void free_fence(void * ptr, uint32_t n_slots = 1);+ ggml_tensor make_fence_tensor(const volatile void * slot, uint32_t n_slots = 1) const;+ uint32_t next_fence_seq() {+ if (++fence_seq == 0) fence_seq = 1;+ return fence_seq;+ }uint8_t * mdev_fence_slot = nullptr;std::unordered_map<uint64_t, volatile uint32_t *> cpy_fence_slots;@@ -646,6 +389,12 @@ struct ggml_hexagon_session {}};+const char * htp_session_name(const struct ggml_hexagon_session * sess) { return sess->c_name(); }+size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess) { return sess->vtcm_size; }+uint32_t htp_session_n_threads(const struct ggml_hexagon_session * sess) { return sess->n_threads; }+uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess) { return sess->n_hmx; }+uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess) { return sess->mdev.count; }+// ** backend buffersstruct ggml_backend_hexagon_device_context {@@ -712,7 +461,7 @@ struct ggml_hexagon_rpcmem_block {struct ggml_hexagon_shared_buffer {ggml_hexagon_session * sess;std::shared_ptr<ggml_hexagon_rpcmem_block> mem;- std::vector<ggml_hexagon_tensor_extra *> tensor_extra;+ std::vector<htp_tensor_extra *> tensor_extra;bool mapped;bool pinned;bool extended;@@ -824,7 +573,7 @@ struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer {ggml_hexagon_fence_buffer(ggml_hexagon_session * sess, ggml_backend_buffer_type_t buft, size_t size): ggml_hexagon_shared_buffer(sess, size, false /* pinned */),- slot_count(size / GGML_HEXAGON_FENCE_SLOT_SIZE),+ slot_count(size / HTP_FENCE_SLOT_SIZE),slot_head(0) {backend_buffer.buft = buft;backend_buffer.context = static_cast<ggml_hexagon_shared_buffer *>(this);@@ -837,21 +586,21 @@ struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer {if (n_slots == 1 && !free_slots.empty()) {uint32_t slot = free_slots.back();free_slots.pop_back();- ptr = base() + (size_t) slot * GGML_HEXAGON_FENCE_SLOT_SIZE;+ ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE;} else if (slot_head + n_slots <= slot_count) {uint32_t slot = slot_head;slot_head += n_slots;- ptr = base() + (size_t) slot * GGML_HEXAGON_FENCE_SLOT_SIZE;+ ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE;}if (ptr) {- memset(ptr, 0, (size_t) n_slots * GGML_HEXAGON_FENCE_SLOT_SIZE);+ memset(ptr, 0, (size_t) n_slots * HTP_FENCE_SLOT_SIZE);}return ptr;}void free_slot(void * ptr, uint32_t n_slots = 1) {if (!ptr) return;- uint32_t slot = ((uint8_t *) ptr - base()) / GGML_HEXAGON_FENCE_SLOT_SIZE;+ uint32_t slot = ((uint8_t *) ptr - base()) / HTP_FENCE_SLOT_SIZE;for (uint32_t i = 0; i < n_slots; i++) {free_slots.push_back(slot + i);}@@ -870,6 +619,26 @@ inline void ggml_hexagon_session::free_fence(void * ptr, uint32_t n_slots) {}}+inline ggml_tensor ggml_hexagon_session::make_fence_tensor(const volatile void * slot, uint32_t n_slots) const {+ static htp_tensor_extra fence_extra { {}, 0, HTP_TENSOR_FENCE };++ ggml_tensor t {};+ t.buffer = &this->fence_buf->backend_buffer;+ t.extra = &fence_extra;+ t.data = const_cast<void *>(slot);+ t.type = GGML_TYPE_I8;+ t.ne[0] = HTP_FENCE_SLOT_SIZE;+ t.ne[1] = (int64_t) n_slots;+ t.ne[2] = 1;+ t.ne[3] = 1;+ t.nb[0] = 1;+ t.nb[1] = HTP_FENCE_SLOT_SIZE;+ t.nb[2] = t.nb[1] * t.ne[1];+ t.nb[3] = t.nb[2];+ t.op = GGML_OP_NONE;+ return t;+}+static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) {auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(buffer->context);sbuf->sess->unclone_buffer(sbuf);@@ -888,7 +657,7 @@ static enum ggml_status ggml_backend_hexagon_buffer_init_tensor(ggml_backend_bufHEX_VERBOSE("ggml-hex: %s init-tensor %s : base %p data %p nbytes %zu\n", sess->c_name(),tensor->name, (void *) sbuf->base(), tensor->data, ggml_nbytes(tensor));- auto extra = new ggml_hexagon_tensor_extra();+ auto extra = new htp_tensor_extra();sbuf->tensor_extra.push_back(extra);tensor->extra = extra;@@ -901,21 +670,21 @@ static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer,const void * data,size_t offset,size_t size) {- auto extra = (ggml_hexagon_tensor_extra *) tensor->extra;+ auto extra = (htp_tensor_extra *) tensor->extra;auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;auto sess = sbuf->sess;if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {- extra->flags |= GGML_HEXAGON_TENSOR_WEIGHT;+ extra->flags |= HTP_TENSOR_WEIGHT;if (htp_is_tiled_weight_tensor(tensor)) {- extra->flags |= GGML_HEXAGON_TENSOR_TILED;+ extra->flags |= HTP_TENSOR_TILED;}}HEX_VERBOSE("ggml-hex: %s set-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n",sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags);- if ((extra->flags & GGML_HEXAGON_TENSOR_TILED) == 0) {+ if ((extra->flags & HTP_TENSOR_TILED) == 0) {memcpy((char *) tensor->data + offset, data, size);return;}@@ -944,14 +713,14 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,void * data,size_t offset,size_t size) {- auto extra = (ggml_hexagon_tensor_extra *) tensor->extra;+ auto extra = (htp_tensor_extra *) tensor->extra;auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;auto sess = sbuf->sess;HEX_VERBOSE("ggml-hex: %s get-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n",sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags);- if ((extra->flags & GGML_HEXAGON_TENSOR_TILED) == 0) {+ if ((extra->flags & HTP_TENSOR_TILED) == 0) {memcpy(data, (const char *) tensor->data + offset, size);return;}@@ -978,21 +747,21 @@ static void ggml_backend_hexagon_buffer_set_tensor_2d(ggml_backend_buffer_t buffsize_t n_copies,size_t stride_tensor,size_t stride_data) {- auto extra = (ggml_hexagon_tensor_extra *) tensor->extra;+ auto extra = (htp_tensor_extra *) tensor->extra;auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;auto sess = sbuf->sess;if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {- extra->flags |= GGML_HEXAGON_TENSOR_WEIGHT;+ extra->flags |= HTP_TENSOR_WEIGHT;if (htp_is_tiled_weight_tensor(tensor)) {- extra->flags |= GGML_HEXAGON_TENSOR_TILED;+ extra->flags |= HTP_TENSOR_TILED;}}HEX_VERBOSE("ggml-hex: %s set-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d flags 0x%x\n",sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, extra->flags);- if ((extra->flags & GGML_HEXAGON_TENSOR_TILED) == 0) {+ if ((extra->flags & HTP_TENSOR_TILED) == 0) {for (size_t i = 0; i < n_copies; i++) {memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size);}@@ -1023,14 +792,14 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buffsize_t n_copies,size_t stride_tensor,size_t stride_data) {- auto extra = (ggml_hexagon_tensor_extra *) tensor->extra;+ auto extra = (htp_tensor_extra *) tensor->extra;auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context;auto sess = sbuf->sess;HEX_VERBOSE("ggml-hex: %s get-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n",sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage);- if ((extra->flags & GGML_HEXAGON_TENSOR_TILED) == 0) {+ if ((extra->flags & HTP_TENSOR_TILED) == 0) {for (size_t i = 0; i < n_copies; i++) {memcpy((uint8_t *)data + i * stride_data, (const uint8_t *)tensor->data + offset + i * stride_tensor, size);}@@ -1435,7 +1204,6 @@ struct ggml_hexagon_opbatch {std::vector<htp_tensor> h_tens; // htp tensor descriptorsstd::vector<htp_op_desc> h_ops; // htp op descriptors- std::unordered_map<int, int> b_map; // buffer fd to indexstd::unordered_map<const ggml_tensor*, int> t_map; // tensor ptr to indexstd::unordered_multimap<void*, int> d_map; // tensor data to index@@ -1455,7 +1223,6 @@ struct ggml_hexagon_opbatch {n_ops = 0;b_vmem = 0;- b_map.clear();t_map.clear();d_map.clear();ops.resize(n_ops_max);@@ -1466,7 +1233,7 @@ struct ggml_hexagon_opbatch {n_bufs_max = HTP_OP_MAX_BUFS;n_ops_max = batch_size;- n_tens_max = std::min<size_t>(n_ops_max + n_ops_max * HTP_OP_MAX_INPUTS, HTP_OP_MAX_TENSORS);+ n_tens_max = std::min<size_t>(n_ops_max * 4 + HTP_OP_MAX_INPUTS + HTP_OP_MAX_OUTPUTS, HTP_OP_MAX_TENSORS);b_vmem_max = max_vmem;@@ -1476,7 +1243,6 @@ struct ggml_hexagon_opbatch {h_tens.resize(n_tens_max);h_ops.resize(n_ops_max);- b_map.reserve(n_bufs_max);t_map.reserve(n_tens_max);d_map.reserve(n_tens_max);@@ -1487,18 +1253,26 @@ struct ggml_hexagon_opbatch {}bool empty() const { return n_ops == 0; }+ bool full() const { return n_ops >= n_ops_max; }++ int find_buffer(int fd) const {+ for (unsigned int i = 0; i < n_bufs; i++) {+ if ((int) h_bufs[i].fd == fd) {+ return (int) i;+ }+ }+ return -1;+ }// add buffer and return its indexint add_buffer(ggml_hexagon_shared_buffer * sbuf) {// Lookup by fd- auto it = b_map.find(sbuf->fd());- if (it != b_map.end()) { return it->second; }+ int bi = find_buffer(sbuf->fd());+ if (bi >= 0) { return bi; }// Add new buffer to the batchGGML_ASSERT(n_bufs < HTP_OP_MAX_BUFS);- int bi = n_bufs++;-- b_map.insert({sbuf->fd(), bi});+ bi = n_bufs++;htp_buf_desc &b = h_bufs[bi];b.base = (uint64_t) sbuf->base();@@ -1515,62 +1289,16 @@ struct ggml_hexagon_opbatch {return bi;}- bool same_shape(const htp_tensor * h, const ggml_tensor * t) const {- auto extra = (ggml_hexagon_tensor_extra *) t->extra;-- const bool is_tiled = (extra->flags & GGML_HEXAGON_TENSOR_TILED) != 0;- const int64_t ne0_padded = hex_round_up(t->ne[0], 32);- const int64_t ne1_padded = hex_round_up(t->ne[1], 32);-- int64_t nb1 = is_tiled ? (int64_t) htp_tiled_row_size(t->type, ne0_padded) : t->nb[1];- int64_t nb2 = is_tiled ? nb1 * ne1_padded : t->nb[2];- int64_t nb3 = is_tiled ? nb2 * t->ne[2] : t->nb[3];- int64_t nb0 = (is_tiled && (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_BF16)) ? sizeof(ggml_fp16_t) : t->nb[0];- enum ggml_type expected_type = (is_tiled && (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : t->type;-- return (h->type == expected_type) &&- (h->ne[0] == t->ne[0]) && (h->ne[1] == t->ne[1]) && (h->ne[2] == t->ne[2]) && (h->ne[3] == t->ne[3]) &&- (h->nb[0] == nb0) && (h->nb[1] == nb1) && (h->nb[2] == nb2) && (h->nb[3] == nb3);- }-- // add tensor and return its index- int add_tensor(const ggml_tensor * t) {- auto extra = (ggml_hexagon_tensor_extra *) t->extra;- auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);-- // First lookup by tensor data- auto range = d_map.equal_range(t->data);- for (auto it = range.first; it != range.second; ++it) {- htp_tensor * h = &h_tens[it->second];- if (same_shape(h, t)) { return it->second; }- }-- // Lookup by tensor ptr- auto it = t_map.find(t);- if (it != t_map.end()) { return it->second; }-- // Add new tensor to the batch- int ti = n_tens++;- GGML_ASSERT(n_tens <= n_tens_max);-- t_map.insert({t, ti});- d_map.insert({t->data, ti});-- uint64_t t_offset = (uint8_t *) t->data - sbuf->base();- size_t t_size = ggml_nbytes(t);+ static void init_tensor_shape(htp_tensor & h, const ggml_tensor * t) {+ auto extra = (htp_tensor_extra *) t->extra;- htp_tensor &h = h_tens[ti];- h.bi = add_buffer(sbuf);- h.ti = ti;- h.data = t_offset;h.type = t->type;h.ne[0] = t->ne[0];h.ne[1] = t->ne[1];h.ne[2] = t->ne[2];h.ne[3] = t->ne[3];- const bool is_tiled = (extra->flags & GGML_HEXAGON_TENSOR_TILED) != 0;- if (is_tiled) {+ if (extra->flags & HTP_TENSOR_TILED) {if (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_BF16) {h.type = GGML_TYPE_F16;h.nb[0] = sizeof(ggml_fp16_t);@@ -1583,60 +1311,96 @@ struct ggml_hexagon_opbatch {h.nb[2] = h.nb[1] * ne1_padded;h.nb[3] = h.nb[2] * h.ne[2];h.size = h.nb[3] * h.ne[3];- t_size = h.size;} else {- h.size = t_size;- h.nb[0] = t->nb[0]; h.nb[1] = t->nb[1]; h.nb[2] = t->nb[2]; h.nb[3] = t->nb[3];+ h.nb[0] = t->nb[0];+ h.nb[1] = t->nb[1];+ h.nb[2] = t->nb[2];+ h.nb[3] = t->nb[3];+ h.size = ggml_nbytes(t);}- h.flags = 0;- if ((extra->flags & GGML_HEXAGON_TENSOR_WEIGHT) != 0) {- h.flags |= HTP_TENSOR_WEIGHT;- }- if ((extra->flags & GGML_HEXAGON_TENSOR_TILED) != 0) {- h.flags |= HTP_TENSOR_TILED;- }- if ((extra->flags & GGML_HEXAGON_TENSOR_FENCE) != 0) {- h.flags |= HTP_TENSOR_FENCE;+ h.flags = extra->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_TILED | HTP_TENSOR_FENCE);+ }++ static bool same_shape(const htp_tensor & a, const htp_tensor & b) {+ return (a.type == b.type) &&+ (a.ne[0] == b.ne[0]) && (a.ne[1] == b.ne[1]) && (a.ne[2] == b.ne[2]) && (a.ne[3] == b.ne[3]) &&+ (a.nb[0] == b.nb[0]) && (a.nb[1] == b.nb[1]) && (a.nb[2] == b.nb[2]) && (a.nb[3] == b.nb[3]);+ }++ // add tensor and return its index+ int add_tensor(const ggml_tensor * t) {+ // Lookup by tensor ptr+ auto it = t_map.find(t);+ if (it != t_map.end()) { return it->second; }++ htp_tensor shape {};+ init_tensor_shape(shape, t);++ // Lookup by tensor data and shape+ auto range = d_map.equal_range(t->data);+ for (auto dit = range.first; dit != range.second; ++dit) {+ if (same_shape(h_tens[dit->second], shape)) {+ t_map.insert({t, dit->second});+ return dit->second;+ }}+ auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);++ // Add new tensor to the batch+ int ti = n_tens++;+ GGML_ASSERT(n_tens <= n_tens_max);++ t_map.insert({t, ti});+ d_map.insert({t->data, ti});++ uint64_t t_offset = (uint8_t *) t->data - sbuf->base();++ htp_tensor & h = h_tens[ti];+ h = shape;+ h.bi = add_buffer(sbuf);+ h.ti = ti;+ h.data = t_offset;+HEX_VERBOSE("ggml-hex: %s add-tensor #%u %s : bi %d data %p offset %zu size %zu flags 0x%x : %zu:%zu:%zu:%zu\n", sess->c_name(),- ti, t->name, h.bi, (void*) t->data, (size_t) t_offset, t_size, h.flags,+ ti, t->name, h.bi, (void*) t->data, (size_t) t_offset, (size_t) h.size, h.flags,(size_t) h.ne[0], (size_t) h.ne[1], (size_t) h.ne[2], (size_t) h.ne[3]);return ti;}- bool fit_op(const htp_opnode & node) const {- if (n_ops >= n_ops_max) return false;-+ bool fit_tensors(const htp_opnode & node) const {// check how much extras we will needsize_t extra_bufs = 0;size_t extra_vmem = 0;size_t extra_tens = 0;+ const bool check_tens = (n_tens + node.inputs.size() + node.outputs.size()) > n_tens_max;+int seen_bufs[HTP_OP_MAX_BUFS];int n_seen_bufs = 0;- auto fit_tensor = [&](const ggml_tensor *t) {+ auto fit_tensor = [&](const ggml_tensor * t) {if (!t) return;- if (!t_map.count(t)) {+ if (check_tens) {+ if (t_map.count(t)) return;extra_tens++;+ }- auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);- int fd = sbuf->fd();- if (!b_map.count(fd)) {- for (int i = 0; i < n_seen_bufs; i++) {- if (seen_bufs[i] == fd) return;- }- if (n_seen_bufs < HTP_OP_MAX_BUFS) {- seen_bufs[n_seen_bufs++] = fd;- }- if (!sbuf->extended) {- extra_vmem += sbuf->size();- }- extra_bufs += 1;+ auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);+ int fd = sbuf->fd();+ if (find_buffer(fd) < 0) {+ for (int i = 0; i < n_seen_bufs; i++) {+ if (seen_bufs[i] == fd) return;}+ if (n_seen_bufs < HTP_OP_MAX_BUFS) {+ seen_bufs[n_seen_bufs++] = fd;+ }+ if (!sbuf->extended) {+ extra_vmem += sbuf->size();+ }+ extra_bufs += 1;}};@@ -1648,40 +1412,43 @@ struct ggml_hexagon_opbatch {}if ((extra_bufs + n_bufs) > n_bufs_max) return false;- if ((extra_tens + n_tens) > n_tens_max) return false;+ if (check_tens && (extra_tens + n_tens) > n_tens_max) return false;if ((extra_vmem + b_vmem) > b_vmem_max) return false;return true;}- // assumes that fit_op() was called first and returned true- void add_op(const htp_opnode & node) {- // Add new op-- unsigned int n = n_ops++;- GGML_ASSERT(n_ops <= n_ops_max);-- ops[n] = node;+ void sync_op_desc(unsigned int n) {+ const htp_opnode & node = ops[n];+ htp_op_desc & o = h_ops[n];- htp_op_desc &o = h_ops[n];- memcpy(o.params, node.node->op_params, sizeof(node.node->op_params));- memcpy(o.kernel_params, node.kernel_params, sizeof(o.kernel_params));+ memcpy(o.params, node.params, sizeof(o.params));+ memcpy(o.kernel_params, node.kernel_params, sizeof(o.kernel_params));o.opcode = node.opcode;- o.flags = 0;- ggml_hexagon_dump_op_exec(sess->c_name(), ops[n], o.flags);-- auto inputs = node.get_inputs();- for (unsigned int i=0; i < HTP_OP_MAX_INPUTS; i++) {- o.src[i] = (i < inputs.size() && inputs[i]) ? add_tensor(inputs[i]) : 0xffff;+ const auto & inputs = node.get_inputs();+ for (unsigned int i = 0; i < HTP_OP_MAX_INPUTS; i++) {+ o.src[i] = (i < inputs.size() && inputs[i]) ? add_tensor(inputs[i]) : 0xffff;}- auto outputs = node.get_outputs();- for (unsigned int i=0; i < HTP_OP_MAX_OUTPUTS; i++) {+ const auto & outputs = node.get_outputs();+ for (unsigned int i = 0; i < HTP_OP_MAX_OUTPUTS; i++) {o.dst[i] = (i < outputs.size() && outputs[i]) ? add_tensor(outputs[i]) : 0xffff;}}+ // caller must ensure the op fits+ void add_op(const htp_opnode & node) {+ unsigned int n = n_ops++;+ GGML_ASSERT(n_ops <= n_ops_max);++ ops[n] = node;+ h_ops[n].flags = 0;++ ggml_hexagon_dump_op_exec(sess->c_name(), ops[n], h_ops[n].flags);+ sync_op_desc(n);+ }+void sort_buffers() {if (n_bufs <= 1) return;@@ -1724,5184 +1491,866 @@ struct ggml_hexagon_opbatch {}}- bool try_fuse_common(std::initializer_list<const ggml_tensor *> tensors) const {- size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0;-- int seen_bufs[HTP_OP_MAX_BUFS];- int n_seen_bufs = 0;-- for (const auto * t : tensors) {- if (!t || t_map.count(t)) {- continue;- }- extra_tens++;- auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);- int fd = sbuf->fd();- if (!b_map.count(fd)) {- bool found = false;- for (int i = 0; i < n_seen_bufs; i++) {- if (seen_bufs[i] == fd) {- found = true;- break;- }- }- if (!found) {- if (n_seen_bufs < HTP_OP_MAX_BUFS) {- seen_bufs[n_seen_bufs++] = fd;- }- if (!sbuf->extended) {- extra_vmem += sbuf->size();- }- extra_bufs += 1;- }- }+ bool try_fuse(const htp_opnode & node) {+ if (n_ops == 0) {+ return false;}- if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) {+ htp_opnode & last_node = ops[n_ops - 1];+ if (!htp_fusion_try_fuse(sess, last_node, node)) {return false;}+ sync_op_desc(n_ops - 1);+ HEX_VERBOSE("ggml-hex: %s fused %s (N=%zu, #%u)\n",+ sess->c_name(), last_node.name().c_str(), last_node.fused.size() + 1, n_ops - 1);return true;}+};- bool try_fuse_common(const ggml_tensor * t1, const ggml_tensor * t2) const {- return try_fuse_common({t1, t2});- }-- bool try_fuse_allreduce_add(const htp_opnode & node) {- if (n_ops == 0 || opt_ar_select != 2) return false;- if (node.opcode != HTP_OP_ADD) return false;-- htp_opnode & last_node = ops[n_ops - 1];- if (last_node.opcode != HTP_OP_ALLREDUCE) return false;-- auto * ar_kparams = (struct htp_allreduce_kernel_params *) last_node.kernel_params;- const uint32_t rank = (uint32_t) ar_kparams->rank;- const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks;- const ggml_tensor * ar_local = last_node.inputs[rank];- const ggml_tensor * add_src0 = node.src0();- const ggml_tensor * add_src1 = node.src1();- const ggml_tensor * add_dst = node.dst();-- if (!ggml_hexagon_tensor_is_fuseable(ar_local)) return false;-- const ggml_tensor * res_tensor;- if (add_src0 == ar_local || add_src0->data == ar_local->data) {- res_tensor = add_src1;- } else if (add_src1 == ar_local || add_src1->data == ar_local->data) {- res_tensor = add_src0;- } else {- return false;- }+struct ggml_hexagon_registry {+ ggml_hexagon_registry(ggml_backend_reg_t reg);+ ~ggml_hexagon_registry();- if (ar_local->type != res_tensor->type) return false;+ ggml_backend_device devices[GGML_HEXAGON_MAX_SESSIONS];+};- const bool is_same_shape = (ar_local->ne[0] == res_tensor->ne[0] && ar_local->ne[1] == res_tensor->ne[1] &&- ar_local->ne[2] == res_tensor->ne[2] && ar_local->ne[3] == res_tensor->ne[3]);- const bool is_row_bcast = !is_same_shape && (ar_local->ne[0] == res_tensor->ne[0] && res_tensor->ne[1] == 1 &&- res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1);+struct ggml_hexagon_opqueue {+ // Shared buffer for storing batches+ ggml_hexagon_shared_buffer *shm_buf;+ size_t shm_blk_size;+ size_t depth;- if (!is_same_shape && !is_row_bcast) return false;+ using opvec = std::vector<htp_opnode>;- if (is_same_shape) {- if (ar_local->nb[1] != res_tensor->nb[1] || ar_local->nb[2] != res_tensor->nb[2] ||- ar_local->nb[3] != res_tensor->nb[3]) {- return false;- }- if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(res_tensor)) {- return false;- }- }- if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(add_dst)) {- return false;- }+ std::vector<opvec> op_cache; // per batch op cache- // scatter is only valid for in-place add within max outputs- const bool is_shard_ok = n_ranks <= HTP_OP_MAX_OUTPUTS && add_dst->data == ar_local->data;+ ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) : depth(depth) {+ size_t n_bufs = HTP_OP_MAX_BUFS;+ size_t n_ops = batch_size;+ size_t n_tensors = std::min<size_t>(n_ops * 4 + HTP_OP_MAX_INPUTS + HTP_OP_MAX_OUTPUTS, HTP_OP_MAX_TENSORS);- struct htp_allreduce_kernel_params new_kparams;- if (!ggml_hexagon_precompute_allreduce_params(- sess, add_dst, (uint32_t) ar_kparams->rank, (uint32_t) ar_kparams->n_ranks, true, is_row_bcast, is_shard_ok, &new_kparams- )) {- HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: solver failed\n", sess->c_name());- return false;+ size_t tr_size = 0;+ if (opt_profile == 3) {+ tr_size = (HTP_MAX_NTHREADS + 1) * opt_optrace * sizeof(htp_trace_desc);}- const bool scatter_ok = new_kparams.n_dsts > 1;- new_kparams.mode = scatter_ok ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL;+ shm_blk_size = sizeof(htp_buf_desc) * n_bufs ++ sizeof(htp_tensor) * n_tensors ++ sizeof(htp_op_desc) * n_ops ++ sizeof(htp_prof_desc) * n_ops ++ tr_size;- for (uint32_t r = 0; r < n_ranks; r++) {- const ggml_tensor * ar_src = last_node.inputs[r];- if (!ggml_hexagon_tensors_overlap(add_dst, ar_src)) {- continue;- }- // in-place aliasing is safe under reduce-scatter since each rank writes disjoint shards- if (scatter_ok && r == rank) {- continue;- }- HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: dst overlaps allreduce src %u\n", sess->c_name(), r);- return false;- }+ shm_buf = new ggml_hexagon_shared_buffer(sess, shm_blk_size * depth, true /* pinned */);- if (!try_fuse_common(res_tensor, add_dst)) {- return false;- }+ op_cache.resize(depth);- last_node.opcode = HTP_OP_ALLREDUCE_ADD;- last_node.name = "ALLREDUCE+ADD";- last_node.inputs.push_back(res_tensor);- last_node.outputs.clear();- last_node.outputs.push_back(add_dst);- last_node.fused.push_back(node.node);- memcpy(last_node.kernel_params, &new_kparams, sizeof(new_kparams));-- htp_op_desc & o = h_ops[n_ops - 1];- o.opcode = HTP_OP_ALLREDUCE_ADD;- memcpy(o.kernel_params, &new_kparams, sizeof(new_kparams));-- o.src[2 * n_ranks] = add_tensor(res_tensor);- if (new_kparams.mode == HTP_ALLREDUCE_SHARDED_FANOUT) {- // fan out shard to all per-rank partial buffers- GGML_ASSERT((uint32_t) new_kparams.n_dsts == n_ranks);- for (uint32_t d = 0; d < n_ranks; d++) {- GGML_ASSERT(o.src[d] != 0xffff);- o.dst[d] = o.src[d];- }- for (uint32_t d = n_ranks; d < HTP_OP_MAX_OUTPUTS; d++) {- o.dst[d] = 0xffff;- }- } else {- o.dst[0] = add_tensor(add_dst);- for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) {- o.dst[d] = 0xffff;- }+ if (opt_verbose) {+ GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n",+ sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size);}-- HEX_VERBOSE("ggml-hex: %s fused ALLREDUCE+ADD (#%u) mode=%d n_dsts=%d\n",- sess->c_name(), n_ops - 1, (int) new_kparams.mode, (int) new_kparams.n_dsts);- return true;}- bool try_fuse_rms_norm_mul(const htp_opnode & node) {- if (n_ops == 0) return false;- if (node.opcode != HTP_OP_MUL) return false;+ ~ggml_hexagon_opqueue() {+ delete shm_buf;+ }- htp_opnode & last_node = ops[n_ops - 1];- if (last_node.opcode != HTP_OP_RMS_NORM) return false;+ size_t shm_size() const { return shm_buf ? shm_buf->size() : 0; }- const ggml_tensor * mul_src0 = node.src0();- const ggml_tensor * mul_src1 = node.src1();- const ggml_tensor * rms_out = last_node.dst();+ // push new batch+ bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, const ggml_hexagon_opbatch* op_batch, uint64_t seq) {+ static_assert(sizeof(htp_opbatch_req) % 8 == 0, "sizeof(htp_opbatch_req) must be multiple of 8");+ static_assert(sizeof(htp_opbatch_rsp) % 8 == 0, "sizeof(htp_opbatch_rsp) must be multiple of 8");+ static_assert(sizeof(htp_buf_desc) % 8 == 0, "sizeof(htp_buf_desc) must be multiple of 8");+ static_assert(sizeof(htp_tensor) % 8 == 0, "sizeof(htp_tensor) must be multiple of 8");+ static_assert(sizeof(htp_op_desc) % 8 == 0, "sizeof(htp_op_desc) must be multiple of 8");+ static_assert(sizeof(htp_prof_desc) % 8 == 0, "sizeof(htp_prof_desc) must be multiple of 8");- if (!ggml_hexagon_tensor_is_fuseable(rms_out)) return false;+ if (seq - shm_buf->sess->batch_rsp_seq > depth) { return false; }- const ggml_tensor * weight;- if (mul_src0 == rms_out || mul_src0->data == rms_out->data) {- weight = mul_src1;- } else if (mul_src1 == rms_out || mul_src1->data == rms_out->data) {- weight = mul_src0;- } else {- return false;- }+ const uint32_t slot = (uint32_t) ((seq - 1) % depth);- const ggml_tensor * src0 = last_node.src0();+ req.seq = seq;+ req.n_bufs = op_batch->n_bufs;+ req.n_tensors = op_batch->n_tens;+ req.n_ops = op_batch->n_ops;- if (src0->ne[0] != weight->ne[0] || src0->ne[0] != node.dst()->ne[0]) {- return false;+ if (opt_profile) {+ op_cache[slot].assign(op_batch->ops.begin(), op_batch->ops.begin() + req.n_ops);}- const bool is_row_bcast = (weight->ne[1] == 1 && weight->ne[2] == 1 && weight->ne[3] == 1);- const bool is_same_shape = (src0->ne[0] == weight->ne[0] && src0->ne[1] == weight->ne[1] &&- src0->ne[2] == weight->ne[2] && src0->ne[3] == weight->ne[3]);- if (!is_row_bcast && !is_same_shape) return false;+ const size_t b_size = sizeof(htp_buf_desc) * req.n_bufs;+ const size_t t_size = sizeof(htp_tensor) * req.n_tensors;+ const size_t o_size = sizeof(htp_op_desc) * req.n_ops;+ const size_t p_size = sizeof(htp_prof_desc) * req.n_ops;- if (!ggml_are_same_shape(src0, node.dst())) {- return false;- }- if (ggml_is_contiguous(src0) != ggml_is_contiguous(node.dst())) {- return false;+ size_t tr_size = 0;+ if (opt_profile == 3) {+ req.n_traces = opt_optrace;+ tr_size = (HTP_MAX_NTHREADS + 1) * req.n_traces * sizeof(htp_trace_desc);+ } else {+ req.n_traces = 0;}- struct htp_unary_kernel_params new_kparams;- ggml_hexagon_precompute_unary_params(- sess, HTP_OP_RMS_NORM_MUL, src0, weight, node.dst(), &new_kparams- );-- if ((size_t) new_kparams.vtcm_size > sess->vtcm_size) {- HEX_VERBOSE("ggml-hex: %s skip RMS_NORM_MUL fusion: VTCM needed (%d) > budget (%zu)\n",- sess->c_name(), new_kparams.vtcm_size, sess->vtcm_size);- return false;- }+ dbuf.ptr = shm_buf->base() + ((size_t) slot * shm_blk_size);+ dbuf.fd = shm_buf->fd();+ dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;+ dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base();+ dbuf.size = b_size + t_size + o_size + p_size + tr_size;- if (!try_fuse_common(weight, node.dst())) {- return false;- }+ GGML_ASSERT(dbuf.size <= shm_blk_size);- last_node.opcode = HTP_OP_RMS_NORM_MUL;- last_node.name = "RMS_NORM+MUL";- last_node.inputs.clear();- last_node.inputs.push_back(src0);- last_node.inputs.push_back(weight);- last_node.outputs.clear();- last_node.outputs.push_back(node.dst());- last_node.fused.push_back(node.node);- memcpy(last_node.kernel_params, &new_kparams, sizeof(new_kparams));-- htp_op_desc & o = h_ops[n_ops - 1];- o.opcode = HTP_OP_RMS_NORM_MUL;- memcpy(o.kernel_params, &new_kparams, sizeof(new_kparams));-- o.src[0] = add_tensor(src0);- o.src[1] = add_tensor(weight);- for (uint32_t s = 2; s < HTP_OP_MAX_INPUTS; s++) {- o.src[s] = 0xffff;- }- o.dst[0] = add_tensor(node.dst());- for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) {- o.dst[d] = 0xffff;- }+ uint8_t * m_ptr = (uint8_t*) dbuf.ptr;+ uint8_t * b_ptr = m_ptr; m_ptr += b_size;+ uint8_t * t_ptr = m_ptr; m_ptr += t_size;+ uint8_t * o_ptr = m_ptr;- HEX_VERBOSE("ggml-hex: %s fused RMS_NORM+MUL (#%u)\n", sess->c_name(), n_ops - 1);- return true;- }-- bool try_fuse_mul_mat_add(const htp_opnode & node) {- if (n_ops == 0) return false;- if (node.opcode != HTP_OP_ADD) return false;-- htp_opnode & last_node = ops[n_ops - 1];- if (last_node.opcode != HTP_OP_MUL_MAT) return false;-- const ggml_tensor * add_src0 = node.src0();- const ggml_tensor * add_src1 = node.src1();- const ggml_tensor * mm_out = last_node.dst();-- if (!ggml_hexagon_tensor_is_fuseable(mm_out)) return false;-- const ggml_tensor * src2;- if (add_src0 == mm_out || add_src0->data == mm_out->data) {- src2 = add_src1;- } else if (add_src1 == mm_out || add_src1->data == mm_out->data) {- src2 = add_src0;- } else {- return false;- }-- const ggml_tensor * src0 = last_node.src0();- const ggml_tensor * src1 = last_node.src1();-- if (src2->type != GGML_TYPE_F32) return false;-- // the fused kernel writes the shape of the MUL_MAT output: the ADD must not broadcast it- if (!ggml_are_same_shape(mm_out, node.dst())) return false;-- const struct htp_mm_kernel_params * orig_kparams = (const struct htp_mm_kernel_params *) last_node.kernel_params;- struct htp_mm_kernel_params kparams;- ggml_hexagon_precompute_fused_matmul_add_params(sess, src0, src1, src2, node.dst(), &kparams);- if (kparams.kernel_type == HTP_MM_KERNEL_UNSUPPORTED) {- return false;- }-- if (orig_kparams->collapse != kparams.collapse) {- return false;- }-- const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3];- const bool can_fuse = (kparams.n_hmx > 0) || (src1_nrows == 1);- if (!can_fuse) return false;+ memcpy(b_ptr, (void *) op_batch->h_bufs.data(), b_size);+ memcpy(t_ptr, (void *) op_batch->h_tens.data(), t_size);+ memcpy(o_ptr, (void *) op_batch->h_ops.data(), o_size);- if ((size_t) kparams.vtcm_size > sess->vtcm_size) {- HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: VTCM needed (%d) > budget (%zu)\n",- sess->c_name(), kparams.vtcm_size, sess->vtcm_size);- return false;- }+ HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%llu : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n",+ shm_buf->sess->c_name(), (unsigned long long) req.seq, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem,+ b_size, t_size, o_size, (size_t) dbuf.size);- if (kparams.n_hmx > 0 && orig_kparams->n_hmx > 0) {- if (kparams.m_chunk < orig_kparams->m_chunk ||- kparams.n_chunk < orig_kparams->n_chunk ||- kparams.n_act_threads < orig_kparams->n_act_threads) {- HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, th %d->%d)\n",- sess->c_name(), orig_kparams->m_chunk, kparams.m_chunk,- orig_kparams->n_chunk, kparams.n_chunk,- orig_kparams->n_act_threads, kparams.n_act_threads);- return false;+ if (opt_verbose > 1) {+ htp_buf_desc *b = (htp_buf_desc*) b_ptr;+ for (unsigned int i=0; i < req.n_bufs; i++) {+ GGML_LOG_DEBUG("ggml-hex: %s htp-buf #%u : fd %d base %p size %zu\n", shm_buf->sess->c_name(), i,+ b[i].fd, (void *) b[i].base, (size_t) b[i].size);+ }+ htp_tensor *t = (htp_tensor*) t_ptr;+ for (unsigned int i=0; i < req.n_tensors; i++) {+ GGML_LOG_DEBUG("ggml-hex: %s htp-tensor #%u : bi %u offset %llu size %u : %zu:%zu:%zu:%zu\n",+ shm_buf->sess->c_name(), i, t[i].bi, (unsigned long long) t[i].data, t[i].size,+ (size_t) t[i].ne[0], (size_t) t[i].ne[1], (size_t) t[i].ne[2], (size_t) t[i].ne[3]);}}- if (!try_fuse_common(src2, node.dst())) {- return false;- }-- last_node.opcode = HTP_OP_MUL_MAT_ADD;- last_node.name = "MUL_MAT+ADD";- last_node.inputs.clear();- last_node.inputs.push_back(src0);- last_node.inputs.push_back(src1);- last_node.inputs.push_back(src2);- last_node.outputs.clear();- last_node.outputs.push_back(node.dst());- last_node.fused.push_back(node.node);- memcpy(last_node.kernel_params, &kparams, sizeof(kparams));-- htp_op_desc & o = h_ops[n_ops - 1];- o.opcode = HTP_OP_MUL_MAT_ADD;- memcpy(o.kernel_params, &kparams, sizeof(kparams));-- o.src[0] = add_tensor(src0);- o.src[1] = add_tensor(src1);- o.src[2] = add_tensor(src2);- for (uint32_t s = 3; s < HTP_OP_MAX_INPUTS; s++) {- o.src[s] = 0xffff;- }- o.dst[0] = add_tensor(node.dst());- for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) {- o.dst[d] = 0xffff;- }-- HEX_VERBOSE("ggml-hex: %s fused MUL_MAT+ADD (#%u)\n", sess->c_name(), n_ops - 1);return true;}- bool try_fuse_mul_mat_nx(const htp_opnode & node) {- if (n_ops == 0 || node.opcode != HTP_OP_MUL_MAT) return false;- if (!is_mergeable_mul_mat(node.node)) return false;-- const ggml_tensor * w_in = node.src0();- const ggml_tensor * x_in = node.src1();- const ggml_tensor * d_in = node.dst();+ void pop(htp_opbatch_rsp rsp, dspqueue_buffer dbuf) {+ const uint32_t slot = (uint32_t) ((rsp.seq - 1) % depth);- htp_opnode & last_node = ops[n_ops - 1];+ const size_t b_size = sizeof(htp_buf_desc) * rsp.n_bufs;+ const size_t t_size = sizeof(htp_tensor) * rsp.n_tensors;+ const size_t o_size = sizeof(htp_op_desc) * rsp.n_ops;+ const size_t p_size = sizeof(htp_prof_desc) * rsp.n_ops;- // Case 1: last_node is already MUL_MAT_NX- if (last_node.opcode == HTP_OP_MUL_MAT_NX) {- const uint32_t curr_n = (uint32_t) last_node.outputs.size();- if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 1 >= HTP_OP_MAX_INPUTS) {- return false;- }+ size_t tr_size = 0;+ uint32_t n_traces = 0;+ if (opt_profile == 3) {+ n_traces = opt_optrace;+ tr_size = (HTP_MAX_NTHREADS + 1) * n_traces * sizeof(htp_trace_desc);+ }- const ggml_tensor * w0 = last_node.inputs[0];- const ggml_tensor * x = last_node.inputs[curr_n];+ const size_t m_size = b_size + t_size + o_size + p_size + tr_size;+ GGML_ASSERT(m_size <= shm_blk_size);- if (x_in != x || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] ||- ggml_hexagon_tensor_is_tiled(w_in) != ggml_hexagon_tensor_is_tiled(w0)) {- return false;- }- if (!last_node.fused.empty() && (mm_is_hmx_eligible(last_node.fused[0]) != mm_is_hmx_eligible(node.node))) {- return false;- }+ HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%llu : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n",+ shm_buf->sess->c_name(), (unsigned long long) rsp.seq, rsp.n_bufs, rsp.n_tensors, rsp.n_ops,+ (size_t) dbuf.size, b_size, t_size, o_size);- const struct htp_mm_kernel_params * orig_kparams = (const struct htp_mm_kernel_params *) last_node.kernel_params;- const bool collapse = orig_kparams->collapse && ggml_hexagon_matmul_can_collapse(w_in, x, d_in);- if (orig_kparams->collapse && !collapse) {- return false;- }+ uint8_t * m_ptr = (uint8_t*) dbuf.ptr;+ uint8_t * p_ptr = m_ptr + (b_size + t_size + o_size);- struct htp_mm_kernel_params kparams;- if (collapse) {- const ggml_tensor x_collapsed = ggml_hexagon_tensor_collapse_rows(x);- ggml_hexagon_precompute_fused_mmnx_params(sess, w0, &x_collapsed, curr_n + 1, &kparams);- kparams.collapse = 1;- } else {- ggml_hexagon_precompute_fused_mmnx_params(sess, w0, x, curr_n + 1, &kparams);- }- if (!is_supported_mul_mat_nx_kernel(w0, &kparams)) {- return false;- }- if ((size_t) kparams.vtcm_size > sess->vtcm_size) {- HEX_VERBOSE("ggml-hex: %s skip NX fusion: VTCM needed (%d) > budget (%zu)\n",- sess->c_name(), kparams.vtcm_size, sess->vtcm_size);- return false;- }+ if (rsp.n_ops > 0 && opt_profile) {+ auto & ops = op_cache[slot];+ GGML_ASSERT(rsp.n_ops <= ops.size());- if (!try_fuse_common(w_in, d_in)) {- return false;+ const htp_prof_desc * pd = (const htp_prof_desc *) p_ptr;+ const htp_trace_desc * trace_events = nullptr;+ if (opt_profile == 3) {+ trace_events = (const htp_trace_desc *) (p_ptr + p_size);}- last_node.inputs[curr_n] = w_in;- last_node.inputs.push_back(x);- last_node.outputs.push_back(d_in);- last_node.fused.push_back(node.node);- memcpy(last_node.kernel_params, &kparams, sizeof(kparams));+ ggml_hexagon_dump_batch_prof(shm_buf->sess->c_name(), rsp);- htp_op_desc & o = h_ops[n_ops - 1];- memcpy(o.kernel_params, &kparams, sizeof(kparams));-- for (uint32_t s = 0; s <= curr_n + 1; s++) {- o.src[s] = add_tensor(last_node.inputs[s]);- }- for (uint32_t s = curr_n + 2; s < HTP_OP_MAX_INPUTS; s++) {- o.src[s] = 0xffff;- }- for (uint32_t d = 0; d <= curr_n; d++) {- o.dst[d] = add_tensor(last_node.outputs[d]);- }- for (uint32_t d = curr_n + 1; d < HTP_OP_MAX_OUTPUTS; d++) {- o.dst[d] = 0xffff;+ for (uint32_t i = 0; i < rsp.n_ops; i++) {+ ggml_hexagon_dump_op_prof(shm_buf->sess->c_name(), ops[i], pd[i]);}- HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_NX (N=%u, #%u)\n", sess->c_name(), curr_n + 1, n_ops - 1);- return true;+ ggml_hexagon_dump_trace_events(shm_buf->sess->c_name(), rsp, trace_events, n_traces);}+ }+};- // Case 2: last_node is single MUL_MAT- if (last_node.opcode == HTP_OP_MUL_MAT) {- if (!is_mergeable_mul_mat_pair(last_node.node, node.node)) {- return false;- }-- const ggml_tensor * w0 = last_node.src0();- const ggml_tensor * x = last_node.src1();- const ggml_tensor * w1 = node.src0();- const ggml_tensor * dst_0 = last_node.dst();- const ggml_tensor * dst_1 = node.dst();-- const struct htp_mm_kernel_params * orig_kparams = (const struct htp_mm_kernel_params *) last_node.kernel_params;- const bool collapse = orig_kparams->collapse && ggml_hexagon_matmul_can_collapse(w1, x, dst_1);- if (orig_kparams->collapse && !collapse) {- return false;- }+void ggml_hexagon_session::flush_peers() {+ auto vpeers = std::move(virt_peers);+ virt_peers.clear();+ for (auto * peer : vpeers) {+ peer->flush_sync();+ }- struct htp_mm_kernel_params kparams;- if (collapse) {- const ggml_tensor x_collapsed = ggml_hexagon_tensor_collapse_rows(x);- ggml_hexagon_precompute_fused_mmnx_params(sess, w0, &x_collapsed, 2, &kparams);- kparams.collapse = 1;- } else {- ggml_hexagon_precompute_fused_mmnx_params(sess, w0, x, 2, &kparams);- }- if (!is_supported_mul_mat_nx_kernel(w0, &kparams)) {- return false;- }- if ((size_t) kparams.vtcm_size > sess->vtcm_size) {- HEX_VERBOSE("ggml-hex: %s skip NX fusion: VTCM needed (%d) > budget (%zu)\n",- sess->c_name(), kparams.vtcm_size, sess->vtcm_size);- return false;- }+ auto ppeers = std::move(phys_peers);+ phys_peers.clear();+ for (auto * peer : ppeers) {+ peer->flush_async();+ }- if (!try_fuse_common(w1, node.dst())) {- return false;- }+ for (auto & sub : this->mdev.sessions) {+ sub->flush_peers();+ }+}- last_node.opcode = HTP_OP_MUL_MAT_NX;- last_node.name = "MUL_MAT_NX";- last_node.inputs.clear();- last_node.inputs.push_back(w0);- last_node.inputs.push_back(w1);- last_node.inputs.push_back(x);- last_node.outputs.clear();- last_node.outputs.push_back(dst_0);- last_node.outputs.push_back(dst_1);- last_node.fused.push_back(node.node);- memcpy(last_node.kernel_params, &kparams, sizeof(kparams));-- htp_op_desc & o = h_ops[n_ops - 1];- o.opcode = HTP_OP_MUL_MAT_NX;- memcpy(o.kernel_params, &kparams, sizeof(kparams));-- o.src[0] = add_tensor(w0);- o.src[1] = add_tensor(w1);- o.src[2] = add_tensor(x);- for (uint32_t s = 3; s < HTP_OP_MAX_INPUTS; s++) {- o.src[s] = 0xffff;- }- o.dst[0] = add_tensor(dst_0);- o.dst[1] = add_tensor(dst_1);- for (uint32_t d = 2; d < HTP_OP_MAX_OUTPUTS; d++) {- o.dst[d] = 0xffff;- }+void ggml_hexagon_session::flush_async() {+ flush_peers();+ flush_batch();+}- HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_NX (N=2, #%u)\n", sess->c_name(), n_ops - 1);- return true;+void ggml_hexagon_session::flush_pending(bool all) {+ for (auto & sub : this->mdev.sessions) {+ sub->flush_pending(all);+ if (sub->last_error > HTP_STATUS_OK) {+ this->last_error = sub->last_error.load();}-- return false;}- bool try_fuse_mul_mat_id_nx(const htp_opnode & node) {- if (n_ops == 0 || node.opcode != HTP_OP_MUL_MAT_ID) return false;- if (!is_mergeable_mul_mat_id(node.node)) return false;-- const ggml_tensor * w_in = node.src0();- const ggml_tensor * x_in = node.src1();- const ggml_tensor * ids_in = node.node->src[2];- const ggml_tensor * d_in = node.dst();-- htp_opnode & last_node = ops[n_ops - 1];-- // Case 1: last_node is already MUL_MAT_ID_NX- if (last_node.opcode == HTP_OP_MUL_MAT_ID_NX) {- const uint32_t curr_n = (uint32_t) last_node.outputs.size();- if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 2 >= HTP_OP_MAX_INPUTS) {- return false;- }-- const ggml_tensor * w0 = last_node.inputs[0];- const ggml_tensor * x = last_node.inputs[curr_n];- const ggml_tensor * ids = last_node.inputs[curr_n + 1];-- if (x_in != x || ids_in != ids || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] || w_in->ne[2] != w0->ne[2] ||- ggml_hexagon_tensor_is_tiled(w_in) != ggml_hexagon_tensor_is_tiled(w0)) {- return false;- }- if (!last_node.fused.empty() && (mm_is_hmx_eligible(last_node.fused[0]) != mm_is_hmx_eligible(node.node))) {- return false;- }-- struct htp_mm_kernel_params kparams;- ggml_hexagon_precompute_fused_mmidnx_params(sess, w0, x, d_in, curr_n + 1, &kparams);- if (!is_supported_mul_mat_id_nx_kernel(w0, &kparams)) {- return false;- }- if ((size_t) kparams.vtcm_size > sess->vtcm_size) {- HEX_VERBOSE("ggml-hex: %s skip ID NX fusion: VTCM needed (%d) > budget (%zu)\n",- sess->c_name(), kparams.vtcm_size, sess->vtcm_size);- return false;- }+ while (this->batch_rsp_seq < this->batch_req_seq) {+ struct htp_opbatch_rsp rsp;+ uint32_t rsp_size;+ uint32_t flags;- if (!try_fuse_common(w_in, d_in)) {- return false;- }+ struct dspqueue_buffer dbuf;+ uint32_t n_dbufs;- last_node.inputs[curr_n] = w_in;- last_node.inputs[curr_n + 1] = x;- last_node.inputs.push_back(ids);- last_node.outputs.push_back(d_in);- last_node.fused.push_back(node.node);- memcpy(last_node.kernel_params, &kparams, sizeof(kparams));+ // Read response packet from queue+ const uint32_t timeo = opt_oppoll ? 0 : DSPQUEUE_TIMEOUT;- htp_op_desc & o = h_ops[n_ops - 1];- memcpy(o.kernel_params, &kparams, sizeof(kparams));+ int err = dspqueue_read(this->queue, &flags, 1, &n_dbufs, &dbuf, sizeof(rsp), &rsp_size, (uint8_t *) &rsp, timeo);+ if (err == AEE_EEXPIRED || err == AEE_EWOULDBLOCK) {+ continue;+ }- for (uint32_t s = 0; s <= curr_n + 2; s++) {- o.src[s] = add_tensor(last_node.inputs[s]);- }- for (uint32_t s = curr_n + 3; s < HTP_OP_MAX_INPUTS; s++) {- o.src[s] = 0xffff;- }- for (uint32_t d = 0; d <= curr_n; d++) {- o.dst[d] = add_tensor(last_node.outputs[d]);- }- for (uint32_t d = curr_n + 1; d < HTP_OP_MAX_OUTPUTS; d++) {- o.dst[d] = 0xffff;- }+ if (err != 0) {+ GGML_ABORT("ggml-hex: dspqueue_read failed: 0x%08x\n", (unsigned) err);+ }- HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_ID_NX (N=%u, #%u)\n", sess->c_name(), curr_n + 1, n_ops - 1);- return true;+ // Basic sanity checks+ if (rsp_size != sizeof(rsp) || n_dbufs != 1) {+ GGML_ABORT("ggml-hex: %s dspcall : bad response : size %u dspbufs %u\n", this->c_name(), rsp_size, n_dbufs);}- // Case 2: last_node is single MUL_MAT_ID- if (last_node.opcode == HTP_OP_MUL_MAT_ID) {- if (!is_mergeable_mul_mat_id_pair(last_node.node, node.node)) {- return false;+ if (rsp.status > HTP_STATUS_OK) {+ GGML_LOG_ERROR("ggml-hex: %s dspcall : dsp-rsp %s\n", this->c_name(), status_to_str(rsp.status));+ this->last_error = rsp.status;+ for (auto & sub : this->mdev.sessions) {+ sub->last_error = rsp.status;}+ }- const ggml_tensor * w0 = last_node.src0();- const ggml_tensor * x = last_node.src1();- const ggml_tensor * ids = last_node.node->src[2];- const ggml_tensor * w1 = node.src0();-- struct htp_mm_kernel_params kparams;- ggml_hexagon_precompute_fused_mmidnx_params(sess, w0, x, node.dst(), 2, &kparams);- if (!is_supported_mul_mat_id_nx_kernel(w0, &kparams)) {- return false;- }- if ((size_t) kparams.vtcm_size > sess->vtcm_size) {- HEX_VERBOSE("ggml-hex: %s skip ID NX fusion: VTCM needed (%d) > budget (%zu)\n",- sess->c_name(), kparams.vtcm_size, sess->vtcm_size);- return false;- }+ op_queue->pop(rsp, dbuf);- if (!try_fuse_common(w1, node.dst())) {- return false;- }+ GGML_ASSERT(rsp.seq == this->batch_rsp_seq + 1);+ this->batch_rsp_seq = rsp.seq;- const ggml_tensor * dst_0 = last_node.dst();- const ggml_tensor * dst_1 = node.dst();-- last_node.opcode = HTP_OP_MUL_MAT_ID_NX;- last_node.name = "MUL_MAT_ID_NX";- last_node.inputs.clear();- last_node.inputs.push_back(w0);- last_node.inputs.push_back(w1);- last_node.inputs.push_back(x);- last_node.inputs.push_back(ids);- last_node.outputs.clear();- last_node.outputs.push_back(dst_0);- last_node.outputs.push_back(dst_1);- last_node.fused.push_back(node.node);- memcpy(last_node.kernel_params, &kparams, sizeof(kparams));-- htp_op_desc & o = h_ops[n_ops - 1];- o.opcode = HTP_OP_MUL_MAT_ID_NX;- memcpy(o.kernel_params, &kparams, sizeof(kparams));-- o.src[0] = add_tensor(w0);- o.src[1] = add_tensor(w1);- o.src[2] = add_tensor(x);- o.src[3] = add_tensor(ids);- for (uint32_t s = 4; s < HTP_OP_MAX_INPUTS; s++) {- o.src[s] = 0xffff;- }- o.dst[0] = add_tensor(dst_0);- o.dst[1] = add_tensor(dst_1);- for (uint32_t d = 2; d < HTP_OP_MAX_OUTPUTS; d++) {- o.dst[d] = 0xffff;- }+ if (!all) break;+ }+}- HEX_VERBOSE("ggml-hex: %s fused MUL_MAT_ID_NX (N=2, #%u)\n", sess->c_name(), n_ops - 1);- return true;- }+void ggml_hexagon_session::flush_sync(bool all) {+ flush_async();+ flush_pending(all);+}- return false;+void ggml_hexagon_session::start_batch() {+ if (this->mdev.count > 1) {+ enqueue_mdev_group();}+}- bool try_fuse_gdn_cpy(const htp_opnode & node) {- if (n_ops == 0 || node.opcode != HTP_OP_CPY) return false;-- htp_opnode & last_node = ops[n_ops - 1];- if (last_node.opcode != HTP_OP_GATED_DELTA_NET) return false;- if (last_node.outputs.size() != 1) return false;+void ggml_hexagon_session::flush_batch(size_t min_ops) {+ if (op_batch->n_ops < min_ops) { return; }- const ggml_tensor * gdn_out = last_node.dst();- const ggml_tensor * cpy_node = node.node;- const ggml_tensor * cpy_src = node.src0();- const ggml_tensor * cpy_dst = node.dst();+ op_batch->sort_buffers();- if (!cpy_src || !cpy_dst || !cpy_dst->data) return false;- if (gdn_out->type != GGML_TYPE_F32 || cpy_src->type != GGML_TYPE_F32 || cpy_dst->type != GGML_TYPE_F32) return false;- if ((gdn_out->flags & GGML_TENSOR_FLAG_OUTPUT) || (cpy_node->flags & GGML_TENSOR_FLAG_OUTPUT)) return false;+ htp_opbatch_req req {};+ dspqueue_buffer dbuf{};- const ggml_tensor * v = last_node.node->src[2];- if (!v) return false;+ const uint64_t seq = ++this->batch_req_seq;- const int64_t S_v = v->ne[0];- const int64_t H = v->ne[1];- const int64_t n_tokens = v->ne[2];- const int64_t n_seqs = v->ne[3];- const int64_t K = ggml_get_op_params_i32(last_node.node, 0);- const size_t tail_off = (size_t) S_v * H * n_tokens * n_seqs * sizeof(float);+ op_batch->update_mdev_group(this->mdev.idx);- const int64_t D = S_v * S_v * H;- const int64_t n_written = std::min<int64_t>(n_tokens, K);+ if (!op_queue->push(req, dbuf, op_batch, seq)) {+ flush_pending(false);+ op_queue->push(req, dbuf, op_batch, seq);+ }- if (cpy_src->op != GGML_OP_VIEW || (cpy_src->view_src != gdn_out && cpy_src->view_src->data != gdn_out->data) ||- cpy_src->view_offs != tail_off || !ggml_is_contiguous(cpy_src)) {- return false;- }+ for (auto & sub : this->mdev.sessions) {+ htp_opbatch_req sub_req {};+ dspqueue_buffer sub_dbuf{};- if (cpy_dst->ne[0] != D || cpy_dst->ne[1] != n_seqs || cpy_dst->nb[0] != sizeof(float)) {- return false;- }- if (n_seqs > 1 && cpy_dst->nb[1] != (size_t) D * sizeof(float)) {- return false;- }- if (n_written > 1) {- if (cpy_dst->ne[2] != n_written || cpy_dst->nb[2] != (size_t) D * n_seqs * sizeof(float)) {- return false;- }- }+ sub->batch_req_seq = seq;+ op_batch->update_mdev_group(sub->mdev.idx);- if (!try_fuse_common({cpy_dst})) {- return false;+ if (!sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq)) {+ sub->flush_pending(false);+ sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq);}- last_node.name += "+CPY";- last_node.outputs.push_back(cpy_dst);- last_node.fused.push_back(node.node);+ HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", sub->c_name(), sub_dbuf.ptr, sub_dbuf.size);- htp_op_desc & o = h_ops[n_ops - 1];- o.dst[1] = add_tensor(cpy_dst);- for (uint32_t d = 2; d < HTP_OP_MAX_OUTPUTS; d++) {- o.dst[d] = 0xffff;+ int err = dspqueue_write(sub->queue, 0, 1, &sub_dbuf, sizeof(sub_req), (const uint8_t*) &sub_req, DSPQUEUE_TIMEOUT);+ if (err != 0) {+ GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", sub->c_name(), (unsigned) err);}-- HEX_VERBOSE("ggml-hex: %s fused GATED_DELTA_NET+CPY (#%u)\n", sess->c_name(), n_ops - 1);- return true;}- bool try_fuse(const htp_opnode & node) {- if (!opt_opfusion) return false;- if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_ALLREDUCE_ADD) && try_fuse_allreduce_add(node)) return true;- if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_RMS_NORM_MUL) && try_fuse_rms_norm_mul(node)) return true;- if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_ADD) && try_fuse_mul_mat_add(node)) return true;- if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_NX) && try_fuse_mul_mat_nx(node)) return true;- if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_MUL_MAT_ID_NX) && try_fuse_mul_mat_id_nx(node)) return true;- if (ggml_hexagon_is_fusion_enabled(GGML_HEXAGON_FUSE_GDN_CPY) && try_fuse_gdn_cpy(node)) return true;- return false;+ HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", this->c_name(), dbuf.ptr, dbuf.size);++ int err = dspqueue_write(this->queue, 0, 1, &dbuf, sizeof(req), (const uint8_t*) &req, DSPQUEUE_TIMEOUT);+ if (err != 0) {+ GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", this->c_name(), (unsigned) err);}-};-struct ggml_hexagon_registry {- ggml_hexagon_registry(ggml_backend_reg_t reg);- ~ggml_hexagon_registry();-- ggml_backend_device devices[GGML_HEXAGON_MAX_SESSIONS];-};--struct ggml_hexagon_opqueue {- // Shared buffer for storing batches- ggml_hexagon_shared_buffer *shm_buf;- size_t shm_blk_size;- size_t depth;-- using opvec = std::vector<htp_opnode>;-- std::vector<opvec> op_cache; // per batch op cache- std::vector<uint64_t> start_usec; // per batch start time-- ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) : depth(depth) {- size_t n_bufs = HTP_OP_MAX_BUFS;- size_t n_ops = batch_size;- size_t n_tensors = n_ops * HTP_OP_MAX_OUTPUTS + n_ops * HTP_OP_MAX_INPUTS;-- size_t tr_size = 0;- if (opt_profile == 3) {- tr_size = (HTP_MAX_NTHREADS + 1) * opt_optrace * sizeof(htp_trace_desc);- }-- shm_blk_size = sizeof(htp_buf_desc) * n_bufs +- sizeof(htp_tensor) * n_tensors +- sizeof(htp_op_desc) * n_ops +- sizeof(htp_prof_desc) * n_ops +- tr_size;-- shm_buf = new ggml_hexagon_shared_buffer(sess, shm_blk_size * depth, true /* pinned */);-- op_cache.resize(depth);- start_usec.resize(depth, 0);-- if (opt_verbose) {- GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n",- sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size);- }- }-- ~ggml_hexagon_opqueue() {- delete shm_buf;- }-- size_t shm_size() const { return shm_buf ? shm_buf->size() : 0; }-- // push new batch- bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, const ggml_hexagon_opbatch* op_batch, uint64_t seq) {- static_assert(sizeof(htp_opbatch_req) % 8 == 0, "sizeof(htp_opbatch_req) must be multiple of 8");- static_assert(sizeof(htp_opbatch_rsp) % 8 == 0, "sizeof(htp_opbatch_rsp) must be multiple of 8");- static_assert(sizeof(htp_buf_desc) % 8 == 0, "sizeof(htp_buf_desc) must be multiple of 8");- static_assert(sizeof(htp_tensor) % 8 == 0, "sizeof(htp_tensor) must be multiple of 8");- static_assert(sizeof(htp_op_desc) % 8 == 0, "sizeof(htp_op_desc) must be multiple of 8");- static_assert(sizeof(htp_prof_desc) % 8 == 0, "sizeof(htp_prof_desc) must be multiple of 8");-- if (seq - shm_buf->sess->batch_rsp_seq > depth) { return false; }-- const uint32_t slot = (uint32_t) ((seq - 1) % depth);-- req.seq = seq;- req.n_bufs = op_batch->n_bufs;- req.n_tensors = op_batch->n_tens;- req.n_ops = op_batch->n_ops;-- op_cache[slot] = op_batch->ops;- start_usec[slot] = ggml_time_us();-- const size_t b_size = sizeof(htp_buf_desc) * req.n_bufs;- const size_t t_size = sizeof(htp_tensor) * req.n_tensors;- const size_t o_size = sizeof(htp_op_desc) * req.n_ops;- const size_t p_size = sizeof(htp_prof_desc) * req.n_ops;-- size_t tr_size = 0;- if (opt_profile == 3) {- req.n_traces = opt_optrace;- tr_size = (HTP_MAX_NTHREADS + 1) * req.n_traces * sizeof(htp_trace_desc);- } else {- req.n_traces = 0;- }-- dbuf.ptr = shm_buf->base() + ((size_t) slot * shm_blk_size);- dbuf.fd = shm_buf->fd();- dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;- dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base();- dbuf.size = b_size + t_size + o_size + p_size + tr_size;-- GGML_ASSERT(dbuf.size <= shm_blk_size);-- uint8_t * m_ptr = (uint8_t*) dbuf.ptr;- uint8_t * b_ptr = m_ptr; m_ptr += b_size;- uint8_t * t_ptr = m_ptr; m_ptr += t_size;- uint8_t * o_ptr = m_ptr;-- memcpy(b_ptr, (void *) op_batch->h_bufs.data(), b_size);- memcpy(t_ptr, (void *) op_batch->h_tens.data(), t_size);- memcpy(o_ptr, (void *) op_batch->h_ops.data(), o_size);-- HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%llu : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n",- shm_buf->sess->c_name(), (unsigned long long) req.seq, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem,- b_size, t_size, o_size, (size_t) dbuf.size);-- if (opt_verbose > 1) {- htp_buf_desc *b = (htp_buf_desc*) b_ptr;- for (unsigned int i=0; i < req.n_bufs; i++) {- GGML_LOG_DEBUG("ggml-hex: %s htp-buf #%u : fd %d base %p size %zu\n", shm_buf->sess->c_name(), i,- b[i].fd, (void *) b[i].base, (size_t) b[i].size);- }- htp_tensor *t = (htp_tensor*) t_ptr;- for (unsigned int i=0; i < req.n_tensors; i++) {- GGML_LOG_DEBUG("ggml-hex: %s htp-tensor #%u : bi %u offset %llu size %u : %zu:%zu:%zu:%zu\n",- shm_buf->sess->c_name(), i, t[i].bi, (unsigned long long) t[i].data, t[i].size,- (size_t) t[i].ne[0], (size_t) t[i].ne[1], (size_t) t[i].ne[2], (size_t) t[i].ne[3]);- }- }-- return true;- }-- void pop(htp_opbatch_rsp rsp, dspqueue_buffer dbuf) {- const uint32_t slot = (uint32_t) ((rsp.seq - 1) % depth);-- const size_t b_size = sizeof(htp_buf_desc) * rsp.n_bufs;- const size_t t_size = sizeof(htp_tensor) * rsp.n_tensors;- const size_t o_size = sizeof(htp_op_desc) * rsp.n_ops;- const size_t p_size = sizeof(htp_prof_desc) * rsp.n_ops;-- size_t tr_size = 0;- uint32_t n_traces = 0;- if (opt_profile == 3) {- n_traces = opt_optrace;- tr_size = (HTP_MAX_NTHREADS + 1) * n_traces * sizeof(htp_trace_desc);- }-- const size_t m_size = b_size + t_size + o_size + p_size + tr_size;- GGML_ASSERT(m_size <= shm_blk_size);-- HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%llu : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n",- shm_buf->sess->c_name(), (unsigned long long) rsp.seq, rsp.n_bufs, rsp.n_tensors, rsp.n_ops,- (size_t) dbuf.size, b_size, t_size, o_size);-- uint8_t * m_ptr = (uint8_t*) dbuf.ptr;- uint8_t * p_ptr = m_ptr + (b_size + t_size + o_size);-- if (rsp.n_ops > 0) {- auto & ops = op_cache[slot];- GGML_ASSERT(rsp.n_ops <= ops.size());-- const htp_prof_desc * pd = (const htp_prof_desc *) p_ptr;- const htp_trace_desc * trace_events = nullptr;- if (opt_profile == 3) {- trace_events = (const htp_trace_desc *) (p_ptr + p_size);- }-- if (opt_profile) {- ggml_hexagon_dump_batch_prof(shm_buf->sess->name, rsp);- }-- for (uint32_t i = 0; i < rsp.n_ops; i++) {- if (opt_profile) {- ggml_hexagon_dump_op_prof(shm_buf->sess->name, ops[i], pd[i]);- }- }-- if (opt_profile) {- ggml_hexagon_dump_trace_events(shm_buf->sess->name, rsp, trace_events, n_traces);- }- }- }-};--void ggml_hexagon_session::flush_peers() {- auto vpeers = std::move(virt_peers);- virt_peers.clear();- for (auto * peer : vpeers) {- peer->flush_sync();- }-- auto ppeers = std::move(phys_peers);- phys_peers.clear();- for (auto * peer : ppeers) {- peer->flush_async();- }-- for (auto & sub : this->mdev.sessions) {- sub->flush_peers();- }-}--void ggml_hexagon_session::flush_async() {- flush_peers();- flush_batch();-}--void ggml_hexagon_session::flush_pending(bool all) {- for (auto & sub : this->mdev.sessions) {- sub->flush_pending(all);- if (sub->last_error > HTP_STATUS_OK) {- this->last_error = sub->last_error.load();- }- }-- while (this->batch_rsp_seq < this->batch_req_seq) {- struct htp_opbatch_rsp rsp;- uint32_t rsp_size;- uint32_t flags;-- struct dspqueue_buffer dbuf;- uint32_t n_dbufs;-- // Read response packet from queue- const uint32_t timeo = opt_oppoll ? 0 : DSPQUEUE_TIMEOUT;-- int err = dspqueue_read(this->queue, &flags, 1, &n_dbufs, &dbuf, sizeof(rsp), &rsp_size, (uint8_t *) &rsp, timeo);- if (err == AEE_EEXPIRED || err == AEE_EWOULDBLOCK) {- continue;- }-- if (err != 0) {- GGML_ABORT("ggml-hex: dspqueue_read failed: 0x%08x\n", (unsigned) err);- }-- // Basic sanity checks- if (rsp_size != sizeof(rsp) || n_dbufs != 1) {- GGML_ABORT("ggml-hex: %s dspcall : bad response : size %u dspbufs %u\n", this->c_name(), rsp_size, n_dbufs);- }-- if (rsp.status > HTP_STATUS_OK) {- GGML_LOG_ERROR("ggml-hex: %s dspcall : dsp-rsp %s\n", this->c_name(), status_to_str(rsp.status));- this->last_error = rsp.status;- for (auto & sub : this->mdev.sessions) {- sub->last_error = rsp.status;- }- }-- op_queue->pop(rsp, dbuf);-- GGML_ASSERT(rsp.seq == this->batch_rsp_seq + 1);- this->batch_rsp_seq = rsp.seq;-- if (!all) break;- }-}--void ggml_hexagon_session::flush_sync(bool all) {- flush_async();- flush_pending(all);-}--void ggml_hexagon_session::start_batch() {- if (this->mdev.count > 1) {- enqueue_mdev_group();- }-}--void ggml_hexagon_session::flush_batch(size_t min_ops) {- if (op_batch->n_ops < min_ops) { return; }-- op_batch->sort_buffers();-- htp_opbatch_req req {};- dspqueue_buffer dbuf{};-- const uint64_t seq = ++this->batch_req_seq;-- op_batch->update_mdev_group(this->mdev.idx);-- if (!op_queue->push(req, dbuf, op_batch, seq)) {- flush_pending(false);- op_queue->push(req, dbuf, op_batch, seq);- }-- for (auto & sub : this->mdev.sessions) {- htp_opbatch_req sub_req {};- dspqueue_buffer sub_dbuf{};-- sub->batch_req_seq = seq;- op_batch->update_mdev_group(sub->mdev.idx);-- if (!sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq)) {- sub->flush_pending(false);- sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq);- }-- HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", sub->c_name(), sub_dbuf.ptr, sub_dbuf.size);-- int err = dspqueue_write(sub->queue, 0, 1, &sub_dbuf, sizeof(sub_req), (const uint8_t*) &sub_req, DSPQUEUE_TIMEOUT);- if (err != 0) {- GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", sub->c_name(), (unsigned) err);- }- }-- HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", this->c_name(), dbuf.ptr, dbuf.size);-- int err = dspqueue_write(this->queue, 0, 1, &dbuf, sizeof(req), (const uint8_t*) &req, DSPQUEUE_TIMEOUT);- if (err != 0) {- GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", this->c_name(), (unsigned) err);- }-- op_batch->reset();-}--void ggml_hexagon_session::enqueue_op(const htp_opnode & node) {- auto clone_tensor_buffer = [this](const ggml_tensor * t) {- auto sbuf = this->mmap_tensor(t);- if (!sbuf) return;- if (sbuf->sess != this) {- this->clone_buffer(sbuf);- }- for (auto & sub : this->mdev.sessions) {- sub->clone_buffer(sbuf);- }- };-- for (auto t : node.get_inputs()) {- clone_tensor_buffer(t);- }- for (auto t : node.get_outputs()) {- clone_tensor_buffer(t);- }-- if (opt_opfusion && op_batch->try_fuse(node)) {- return;- }-- if (!op_batch->fit_op(node)) {- flush_async();- }-- if (op_batch->empty()) {- start_batch();- }-- if (!op_batch->fit_op(node)) {- GGML_ABORT("ggml-hex: %s op does not fit into empty batch (vmem/tensor/buffer limit exceeded)\n",- c_name());- }-- op_batch->add_op(node);-}--void ggml_hexagon_session::enqueue_mdev_group() {- htp_opnode group_node(HTP_OP_MDEV_GROUP);-- uint8_t * fence_slot = this->mdev_fence_slot;-- static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE };- ggml_tensor dummy_t {};- dummy_t.buffer = &this->fence_buf->backend_buffer;- dummy_t.extra = &fence_extra;- dummy_t.data = (void *) fence_slot;- dummy_t.type = GGML_TYPE_I8;- dummy_t.ne[0] = HTP_FENCE_SLOT_SIZE;- dummy_t.ne[1] = (int64_t) this->mdev.count;- dummy_t.ne[2] = 1;- dummy_t.ne[3] = 1;- dummy_t.nb[0] = 1;- dummy_t.nb[1] = HTP_FENCE_SLOT_SIZE;- dummy_t.nb[2] = dummy_t.nb[1] * dummy_t.ne[1];- dummy_t.nb[3] = dummy_t.nb[2];- dummy_t.op = GGML_OP_NONE;- dummy_t.op_params[0] = (int32_t) this->mdev.idx;-- ggml_tensor * node = group_node.add_dummy(dummy_t);- node->src[0] = node;- group_node.init(node);- group_node.outputs.clear();- group_node.name = "MDEV_GROUP";-- for (auto & sub : this->mdev.sessions) {- sub->clone_buffer(this->fence_buf);- }-- op_batch->add_op(group_node);-}--void ggml_hexagon_session::enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor, uint32_t fence_seq) {- const bool with_fence = sync_tensor != nullptr;- htp_opnode cpy_node(with_fence ? HTP_OP_CPY_FENCE : HTP_OP_CPY);-- ggml_tensor* node = cpy_node.add_dummy(*dst);- node->op = GGML_OP_CPY;- node->src[0] = const_cast<ggml_tensor *>(src);- node->src[1] = with_fence ? cpy_node.add_dummy(*sync_tensor) : nullptr;- if (with_fence) {- node->op_params[0] = (int32_t) fence_seq;- }-- cpy_node.init(node);- if (with_fence) {- cpy_node.name = "CPY+FENCE";- }- const bool ok = ggml_hexagon_precompute_cpy_params(this, node, (struct htp_copy_kernel_params *) cpy_node.kernel_params);- const auto * kparams = (const struct htp_copy_kernel_params *) cpy_node.kernel_params;- if (ok && !with_fence && kparams->total_elems == 0) {- return;- }- this->enqueue_op(cpy_node);-}--void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq, bool wait) {- htp_opnode sync_node(HTP_OP_FENCE);-- ggml_tensor* node = sync_node.add_dummy(*sync_tensor);- node->op = GGML_OP_NONE;- node->src[0] = node;- node->op_params[0] = (int32_t) fence_seq;- node->op_params[1] = wait ? 0 : 1;-- sync_node.init(node);- sync_node.name = wait ? "FENCE_WAIT" : "FENCE_SIGNAL";- this->enqueue_op(sync_node);-}--static bool ggml_hexagon_precompute_allreduce_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * dst,- uint32_t rank,- uint32_t n_ranks,- bool has_add,- bool is_row_bcast,- bool is_shard_ok,- struct htp_allreduce_kernel_params * kparams-) {- memset(kparams, 0, sizeof(*kparams));- kparams->rank = (int32_t) rank;- kparams->n_ranks = (int32_t) n_ranks;- kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0;-- const uint32_t nelem = (uint32_t) ggml_nelements(dst);- const uint32_t elem_size = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);- const bool is_contiguous = ggml_is_contiguous(dst);-- const uint32_t ne0 = (uint32_t) dst->ne[0];- const uint32_t ne1 = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);- kparams->ne0 = (int32_t) ne0;- kparams->ne1 = (int32_t) ne1;-- const bool use_1d = is_contiguous && !(has_add && is_row_bcast && ne1 > 1);-- if (has_add) {- // sharded reduce-scatter for contiguous in-place add- if (opt_ar_scatter && is_shard_ok && use_1d && !is_row_bcast) {- const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128);- const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem);- const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem);- kparams->n_dsts = (int32_t) n_ranks;- kparams->rank_elem_start = (int32_t) rank_elem_start;- kparams->rank_nelem = (int32_t) (rank_elem_end - rank_elem_start);- } else if (use_1d) {- kparams->n_dsts = 1;- kparams->rank_elem_start = 0;- kparams->rank_nelem = (int32_t) nelem;- } else {- kparams->n_dsts = 1;- kparams->rank_elem_start = 0;- kparams->rank_nelem = (int32_t) ne1;- }- } else {- kparams->n_dsts = (int32_t) n_ranks;- if (use_1d) {- const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128);- const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem);- const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem);- const uint32_t rank_nelem = rank_elem_end - rank_elem_start;- kparams->rank_elem_start = (int32_t) rank_elem_start;- kparams->rank_nelem = (int32_t) rank_nelem;- } else {- const uint32_t rank_chunk_rows = (ne1 + n_ranks - 1) / n_ranks;- const uint32_t rank_r0 = (std::min)(rank * rank_chunk_rows, ne1);- const uint32_t rank_r1 = (std::min)(rank_r0 + rank_chunk_rows, ne1);- const uint32_t rank_nrows = rank_r1 - rank_r0;- kparams->rank_elem_start = (int32_t) rank_r0;- kparams->rank_nelem = (int32_t) rank_nrows;- }- }-- kparams->mode = (kparams->n_dsts > 1) ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL;-- if (use_1d) {- const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem;- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nelem / 128));- kparams->n_threads = n_threads;- const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast);-- uint32_t block_elems = 65536;- if (block_elems > rank_nelem / n_threads && rank_nelem / n_threads > 128) {- block_elems = hex_round_up(rank_nelem / (n_threads * 2), 128);- }- block_elems = (std::max)(128u, block_elems);-- kparams->block_elems = block_elems;- kparams->vtcm_size_per_thread = 2 * block_elems * elem_size;- kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;-- while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_elems > 128) {- const size_t max_bytes_per_buf = sess->vtcm_size / (n_vtcm_buffers * 2);- block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128);- if (block_elems < 128) break;- kparams->block_elems = block_elems;- kparams->vtcm_size_per_thread = 2 * block_elems * elem_size;- kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;- }-- if (sess->vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) {- HEX_VERBOSE("ggml-hex: %s allreduce 1D solver failed to fit VTCM (%d > %zu)\n",- sess->c_name(), kparams->vtcm_size, sess->vtcm_size);- return false;- }-- kparams->elems_per_thread = hex_round_up((rank_nelem + n_threads - 1) / n_threads, block_elems);- kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_1D;- return true;- } else {- const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem;- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nrows));- kparams->n_threads = n_threads;- const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast);-- const uint32_t row_bytes = ne0 * elem_size;- const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128);- kparams->row_size_aligned = row_size_aligned;-- const uint32_t nrows_per_thread = (rank_nrows + n_threads - 1) / n_threads;- uint32_t block_rows = (std::min)(128u, nrows_per_thread);- block_rows = (std::max)(1u, block_rows);- kparams->block_elems = block_rows;-- kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned);- kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;-- while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_rows > 1) {- const size_t max_rows_per_buf = sess->vtcm_size / (n_vtcm_buffers * 2 * row_size_aligned);- block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf);- kparams->block_elems = block_rows;- kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned);- kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;- if (max_rows_per_buf == 0) break;- }-- if (sess->vtcm_size < (size_t) kparams->vtcm_size || block_rows < 1) {- HEX_VERBOSE("ggml-hex: %s allreduce 2D solver failed to fit VTCM (%d > %zu)\n",- sess->c_name(), kparams->vtcm_size, sess->vtcm_size);- return false;- }-- kparams->elems_per_thread = nrows_per_thread;- kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_2D;- return true;- }-}--void ggml_hexagon_session::enqueue_allreduce(- const ggml_tensor * dst,- const std::vector<const ggml_tensor *> & src_tensors,- const std::vector<const ggml_tensor *> & sync_tensors,- uint32_t rank,- uint32_t n_ranks,- uint32_t fence_seq_entry,- uint32_t fence_seq_exit-) {- htp_opnode ar_node(HTP_OP_ALLREDUCE);-- ggml_tensor* node = ar_node.add_dummy(*dst);- node->op = GGML_OP_NONE;- node->op_params[0] = (int32_t) fence_seq_entry;- node->op_params[1] = (int32_t) fence_seq_exit;-- ar_node.init(node);-- ar_node.inputs.clear();- for (size_t i = 0; i < src_tensors.size(); i++) {- ar_node.inputs.push_back(src_tensors[i]);- }- for (size_t i = 0; i < sync_tensors.size(); i++) {- ar_node.inputs.push_back(ar_node.add_dummy(*sync_tensors[i]));- }-- ar_node.outputs.clear();- for (size_t i = 0; i < src_tensors.size(); i++) {- ar_node.outputs.push_back(src_tensors[i]);- }-- ggml_hexagon_precompute_allreduce_params(- this, dst, rank, n_ranks, false, false, /*is_shard_ok=*/ false,- (struct htp_allreduce_kernel_params *) ar_node.kernel_params);-- ar_node.name = "ALLREDUCE";- this->enqueue_op(ar_node);-}--ggml_hexagon_shared_buffer * ggml_hexagon_session::mmap_tensor(const ggml_tensor * t) {- if (!t) return nullptr;-- auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);- if (!sbuf->mapped) {- const bool is_weight = ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS;- const bool extended = opt_dma64 && is_weight;- sbuf->mmap(extended);- }- return sbuf;-}--bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf)-{- GGML_ASSERT(sbuf && sbuf->mem);- if (sbuf->sess == this) return true;-- auto mem = sbuf->mem;- int fd = mem->fd;-- GGML_ASSERT(fd >= 0);-- if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return true;-- GGML_ASSERT(sbuf->mapped);-- HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(),- sbuf->c_name(), sbuf->base(), sbuf->size(), fd);-- auto clone = std::make_unique<ggml_hexagon_shared_buffer>(this, *sbuf);- try {- clone->mmap(sbuf->extended);- } catch (const std::exception & exc) {- GGML_LOG_ERROR("ggml-hex: %s lazy mapping of buffer context failed: %s\n", this->c_name(), exc.what());- return false;- }-- this->cloned_buffers[fd] = std::move(clone);- mem->mapped_clones.insert(this);- return true;-}--void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbuf) {- GGML_ASSERT(sbuf && sbuf->mem);-- auto mem = sbuf->mem;- int fd = mem->fd;-- GGML_ASSERT(fd >= 0);-- auto it = this->cloned_buffers.find(fd);- if (it != this->cloned_buffers.end()) {- auto clone = std::move(it->second);- this->cloned_buffers.erase(it);- }- mem->mapped_clones.erase(this);-}--void ggml_hexagon_session::unclone_buffer(const ggml_hexagon_shared_buffer * sbuf) {- GGML_ASSERT(sbuf && sbuf->mem);-- auto mem = sbuf->mem;- std::vector<ggml_hexagon_session *> sessions(mem->mapped_clones.begin(), mem->mapped_clones.end());-- for (auto * sess : sessions) {- sess->release_buffer(sbuf);- }-}--static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) {- // Allocate a bunch pinned buffers till failure.- // This is kind of expensive but handy for figuring out exactly how much we can mmap on a specific device.- // Typically we're going to allocate all/most of these buffers anyway for the model weights.-- std::vector<ggml_hexagon_shared_buffer *> sbufs;-- const size_t MiB = 1024 * 1024;- const size_t GiB = MiB * 1024;-- size_t vmem = 0;- size_t step = 256u * MiB;-- try {- sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB;- sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB;- sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB;-- while (1) {- sbufs.push_back(new ggml_hexagon_shared_buffer(sess, step, true));- vmem += step;- }- } catch (...) { }-- for (auto b : sbufs) { delete b; }-- return vmem - step; // backoff to account for overhead from internal mappings-}--void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) noexcept(false) {- int phys_idx = config.physical_idx;- int virt_idx = config.virtual_idx;-- this->valid_session = false;- this->valid_handle = false;- this->valid_queue = false;- this->valid_iface = false;-- this->name = config.name;- this->phys_idx = phys_idx;- this->virt_idx = virt_idx;- this->domain_id = config.domain_id;- this->session_id = 0;- this->batch_req_seq = 0;- this->batch_rsp_seq = 0;- this->last_error = HTP_STATUS_OK;-- GGML_LOG_DEBUG("ggml-hex: %s allocating new session : domain %u phys-idx %u virt-idx %u\n", this->name.c_str(), this->domain_id, phys_idx, virt_idx);-- if (config.domain_id < 0 || config.domain_name.empty()) {- GGML_LOG_ERROR("ggml-hex: %s: invalid physical CDSP core %d\n", config.name.c_str(), config.physical_idx);- throw std::runtime_error("ggml-hex: invalid physical CDSP core");- }-- const std::string & dom_name = config.domain_name;-- // Create new session if virtual_idx > 0- if (virt_idx > 0) {- struct remote_rpc_reserve_new_session n {};- n.domain_name_len = dom_name.size();- n.domain_name = const_cast<char *>(dom_name.c_str());- n.session_name = const_cast<char *>(this->name.c_str());- n.session_name_len = this->name.size();- n.session_id = virt_idx;-- int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n));- if (err != AEE_SUCCESS) {- GGML_LOG_ERROR("ggml-hex: %s failed to reserve new session (physical %d, virtual %d) : error 0x%x\n",- this->c_name(), phys_idx, virt_idx, err);- throw std::runtime_error("ggml-hex: remote_session_control(new-sess) failed (see log for details)");- }-- // Save the IDs- this->session_id = n.session_id;- this->domain_id = n.effective_domain_id;- this->valid_session = true;- } else {- struct remote_rpc_effective_domain_id eff {};- eff.domain_name = const_cast<char *>(dom_name.c_str());- eff.domain_name_len = dom_name.size();- eff.session_id = 0;-- int err = remote_session_control(FASTRPC_GET_EFFECTIVE_DOMAIN_ID, (void *) &eff, sizeof(eff));- if (err == AEE_SUCCESS) {- this->domain_id = eff.effective_domain_id;- } else {- GGML_LOG_DEBUG("ggml-hex: %s FASTRPC_GET_EFFECTIVE_DOMAIN_ID returned 0x%x, using domain_id %d\n",- this->name.c_str(), err, this->domain_id);- }- }-- // Enable unsigned modules- {- struct remote_rpc_control_unsigned_module u;- u.domain = this->domain_id;- u.enable = 1;- int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u));- if (err != AEE_SUCCESS) {- GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err);- throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)");- }- }-- char session_uri[256];- {- char htp_uri[256];- snprintf(htp_uri, sizeof(htp_uri), "file:///libggml-htp-v%u.so?htp_iface_skel_handle_invoke&_modver=1.0", opt_arch);-- struct remote_rpc_get_uri u = {};- u.session_id = this->session_id;- u.domain_name = const_cast<char *>(dom_name.c_str());- u.domain_name_len = dom_name.size();- u.module_uri = const_cast<char *>(htp_uri);- u.module_uri_len = strlen(htp_uri);- u.uri = session_uri;- u.uri_len = sizeof(session_uri);-- int err = remote_session_control(FASTRPC_GET_URI, (void *) &u, sizeof(u));- if (err != AEE_SUCCESS) {- snprintf(session_uri, sizeof(session_uri), "%s&_dom=%s&_session=%u",- htp_uri, dom_name.c_str(), this->session_id);-- GGML_LOG_WARN("ggml-hex: %s failed to get URI (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n",- this->c_name(), phys_idx, virt_idx, err, session_uri);- }- }-- // Open session- int err = htp_iface_open(session_uri, &this->handle);- if (err != AEE_SUCCESS) {- GGML_LOG_ERROR("ggml-hex: %s failed to open session : uri %s error 0x%x\n", this->c_name(), session_uri, err);- throw std::runtime_error("ggml-hex: failed to open session (see log for details)");- }-- this->valid_handle = true;-- // Query HW info and resolve session options- {- unsigned int hw_n_threads = 0;- unsigned int hw_n_hvx = 0;- unsigned int hw_n_hmx = 0;- unsigned long long hw_vtcm_size = 0;- int hw_err = htp_iface_hwinfo(this->handle, &hw_n_threads, &hw_n_hvx, &hw_n_hmx, &hw_vtcm_size);- if (hw_err == 0) {- const uint32_t max_n_threads = (std::min)((uint32_t) HTP_MAX_NTHREADS, (uint32_t) hw_n_threads);- this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_threads) : max_n_threads;- this->n_hvx = this->n_threads;- this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0;- this->vtcm_size = (uint64_t)hw_vtcm_size;- GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n",- this->c_name(), this->n_threads, this->n_hvx, this->n_hmx,- (unsigned long long)(this->vtcm_size / (1024 * 1024)));- } else {- GGML_LOG_WARN("ggml-hex: %s failed to query hwinfo (0x%x), using defaults\n", this->c_name(), hw_err);- const uint32_t default_n_threads = (std::min)(8u, (uint32_t) HTP_MAX_NTHREADS);- this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads;- this->n_hvx = this->n_threads;- this->n_hmx = (opt_nhmx != 0) ? 1 : 0;- this->vtcm_size = 8 * 1024 * 1024;- }- }-- // Enable FastRPC QoS mode- {- struct remote_rpc_control_latency l;- l.enable = 1;-- int err = remote_handle64_control(this->handle, DSPRPC_CONTROL_LATENCY, (void *) &l, sizeof(l));- if (err != 0) {- GGML_LOG_WARN("ggml-hex: failed to enable fastrpc QOS mode: 0x%08x\n", (unsigned) err);- }- }-- GGML_LOG_INFO("ggml-hex: %s new session : session-id %d domain-id %d uri %s handle 0x%lx\n", this->c_name(),- this->session_id, this->domain_id, session_uri, (unsigned long) this->handle);-- const size_t req_q_size = (sizeof(htp_opbatch_req) * opt_opqueue * 2) + 1024;- const size_t rsp_q_size = (sizeof(htp_opbatch_rsp) * opt_opqueue * 2) + 1024;-- // Now let's setup the DSP queue- err = dspqueue_create(this->domain_id,- 0, // Flags- req_q_size, // Request queue size (in bytes)- rsp_q_size, // Response queue size (in bytes)- nullptr, // Read packet callback (we handle reads explicitly)- nullptr, // Error callback (we handle errors during reads)- (void *) this, // Callback context- &queue);- if (err != 0) {- GGML_LOG_ERROR("ggml-hex: %s dspqueue_create failed: 0x%08x\n", this->name.c_str(), (unsigned) err);- throw std::runtime_error("ggml-hex: failed to create dspqueue (see log for details)");- }-- this->valid_queue = true;-- // Export queue for use on the DSP- err = dspqueue_export(queue, &this->queue_id);- if (err != 0) {- GGML_LOG_ERROR("ggml-hex: dspqueue_export failed: 0x%08x\n", (unsigned) err);- throw std::runtime_error("ggml-hex: dspqueue export failed (see log for details)");- }-- if (opt_etm) {- err = htp_iface_etm(this->handle, 1);- if (err != 0) {- GGML_LOG_ERROR("ggml-hex: failed to enable ETM tracing: 0x%08x\n", (unsigned) err);- }- }-- // Allocate buffers and state for op batching- this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue);-- this->fence_buf = new ggml_hexagon_fence_buffer(this, &dev_ctx->fence_buffer_type, 64 * 1024);- if (this->mdev.count > 1) {- this->mdev_fence_slot = this->alloc_fence(this->mdev.count);- }-- if (!opt_vmem) {- opt_vmem = ggml_hexagon_measure_max_vmem(this);- GGML_LOG_INFO("ggml-hex: %s measured max vmem %zu\n", this->c_name(), opt_vmem);- }- const size_t shm_size = this->op_queue->shm_size();- this->max_vmem = (opt_vmem > shm_size) ? (opt_vmem - shm_size) : opt_vmem;-- this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem);-- // Start dspqueue/opbatch processing- err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem, (uint32_t) opt_power_mode);- if (err != 0) {- GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err);- throw std::runtime_error("ggml-hex: iface start failed (see log for details)");- }- this->valid_iface = true;- GGML_LOG_INFO("ggml-hex: %s power mode: %s\n", this->c_name(), htp_power_mode_to_str(opt_power_mode));-- if (opt_profile) {- htp_iface_pmu_conf pmu_conf{};- std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events);-- err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf);- if (err != 0) {- GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err);- }- }-}--void ggml_hexagon_session::release() noexcept(true) {- GGML_LOG_INFO("ggml-hex: releasing session: %s\n", this->name.c_str());-- this->mdev.sessions.clear();-- int err;-- if (this->valid_iface) {- // Stop dspqueue/opbatch processing- err = htp_iface_stop(this->handle);- if (err != 0) {- GGML_ABORT("ggml-hex: htp_iface_stop failed: 0x%08x\n", (unsigned) err);- }- }-- delete this->op_batch;- delete this->op_queue;- for (auto & it : this->cpy_fence_slots) {- free_fence((void *) it.second, 1);- }- this->cpy_fence_slots.clear();-- if (this->fence_buf) {- unclone_buffer(this->fence_buf);- delete this->fence_buf;- this->fence_buf = nullptr;- }- while (!this->cloned_buffers.empty()) {- release_buffer(this->cloned_buffers.begin()->second.get());- }-- if (opt_etm) {- err = htp_iface_etm(this->handle, 0);- if (err != 0) {- GGML_LOG_ERROR("ggml-hex: warn : failed to disable ETM tracing: 0x%08x\n", (unsigned) err);- }- }-- if (opt_profile) {- htp_iface_pmu_conf pmu_conf{};- err = htp_iface_profiler(this->handle, 0, &pmu_conf);- if (err != 0) {- GGML_LOG_ERROR("ggml-hex: warn : failed to disable profiling: 0x%08x\n", (unsigned) err);- }- }-- if (this->valid_queue) {- err = dspqueue_close(queue);- if (err != 0) {- GGML_ABORT("ggml-hex: dspqueue_close failed: 0x%08x\n", (unsigned) err);- }- }-- if (this->valid_handle) {- htp_iface_close(this->handle);- }-}--ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev, uint32_t mdev_idx, uint32_t mdev_count) noexcept(false) {- this->dev = dev;- this->dev_ctx = static_cast<ggml_backend_hexagon_device_context *>(dev->context);- this->mdev.idx = mdev_idx;- this->mdev.count = mdev_count > 0 ? mdev_count : (uint32_t) (1 + config.mdev_group.size());- op_batch = nullptr;- op_queue = nullptr;- fence_buf = nullptr;- fence_seq = ((uintptr_t)this) & 0xFFFF;-- try {- allocate(config);- if (this->mdev.idx == 0 && !config.mdev_group.empty()) {- for (size_t i = 0; i < config.mdev_group.size(); i++) {- this->mdev.sessions.push_back(std::make_unique<ggml_hexagon_session>(- config.mdev_group[i], this->dev, (uint32_t) (i + 1), this->mdev.count));- }- }- } catch (const std::exception & exc) {- release();- throw;- }-}--ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) {- release();-}--// ** backend interface--static bool ggml_hexagon_flash_attn_is_hmx_eligible(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * q,- const struct ggml_tensor * k,- const struct ggml_tensor * v,- const struct ggml_tensor * sinks-) {- if (sess->n_hmx == 0) {- return false;- }-- if (opt_fa_select < 2) {- return false;- }-- if ((k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0) ||- (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_Q8_0)) {- return false;- }-- const uint32_t DK = q->ne[0];- const uint32_t DV = v->ne[0];-- // Head dims that are not multiples of 64 are handled by internally padding to- // DK_pad/DV_pad = round_up(.,64) and zero-filling the tail lanes.- if (DK % 8 != 0 || DV % 8 != 0) {- return false;- }-- GGML_UNUSED(sinks);-- // Explicit force mode- if (opt_fa_select > 2) {- return true;- }-- const uint32_t M = q->ne[1];-- // Prefill or batched decode- if (M > 1) {- return true;- }-- const uint32_t n_head = q->ne[2];- const uint32_t n_kv_heads = k->ne[2];- const uint32_t G = n_kv_heads > 0 ? n_head / n_kv_heads : 1;- const uint32_t S = k->ne[1];- const uint32_t total_threads = sess->n_threads * sess->mdev.count;-- // Compute-bound head dim (for DK == 256 decode, HVX is faster when KV heads span all threads)- if (DK > 256 || (DK == 256 && (DV > 256 || n_kv_heads < total_threads))) {- return true;- }-- // Tile alignment for 32-row HMX tiles- const bool is_tile_aligned = (G > 0 && (32 % G == 0));- if (!is_tile_aligned) {- if (sess->n_threads >= 6) {- return false;- }- return S >= 1024;- }-- // Context depth crossover- uint32_t s_cross = 512;- if (DK == 256) {- s_cross = 4096;- } else if (DK <= 64) {- s_cross = (sess->n_threads >= 8) ? 2048 : ((sess->n_threads >= 6) ? 768 : 512);- } else {- s_cross = (sess->n_threads >= 8) ? 1024 : 512;- }-- return S >= s_cross;-}--static bool ggml_hexagon_precompute_flash_attn_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_fa_kernel_params * kparams-) {- if (opt_fa_select < 1) {- return false;- }-- memset(kparams, 0, sizeof(*kparams));-- const struct ggml_tensor * q = op->src[0];- const struct ggml_tensor * k = op->src[1];- const struct ggml_tensor * v = op->src[2];- const struct ggml_tensor * mask = op->src[3];-- const uint32_t neq0 = q->ne[0]; // head_dim (DK)- const uint32_t neq1 = q->ne[1]; // n_tokens- const uint32_t neq2 = q->ne[2]; // n_heads-- const uint32_t nek1 = k->ne[1]; // kv_len-- const uint32_t nev0 = v->ne[0]; // head_dim (DV)-- const uint32_t DK = neq0;- const uint32_t DV = nev0;-- const uint32_t n_kv_heads = k->ne[2];- const uint32_t G = neq2 / n_kv_heads;-- float scale = 1.0f;- float max_bias = 0.0f;- float logit_softcap = 0.0f;- memcpy(&scale, &op->op_params[0], sizeof(float));- memcpy(&max_bias, &op->op_params[1], sizeof(float));- memcpy(&logit_softcap, &op->op_params[2], sizeof(float));-- if (logit_softcap != 0.0f) {- scale /= logit_softcap;- }-- kparams->scale = scale;- kparams->max_bias = max_bias;- kparams->logit_softcap = logit_softcap;-- kparams->head_split = (opt_fa_head_split != 0) ? 1 : 0;- kparams->flags = 0;- kparams->G = G;-- const uint32_t n_head = q->ne[2];- kparams->n_head_log2 = 1u << (uint32_t) std::floor(std::log2(n_head));- kparams->m0 = std::pow(2.0f, -(max_bias) / kparams->n_head_log2);- kparams->m1 = std::pow(2.0f, -(max_bias / 2.0f) / kparams->n_head_log2);-- // Check HMX eligibility- const struct ggml_tensor * sinks = op->src[4];- if (ggml_hexagon_flash_attn_is_hmx_eligible(sess, q, k, v, sinks)) {- // HMX tiles head_dim in units of 64; when DK/DV are not 64-aligned the kernel- // operates on padded dims with zero-filled tail lanes. VTCM budget and chunk-size- // are sized for the padded tiles.- const uint32_t DK_pad = hex_round_up(DK, 64);- const uint32_t DV_pad = hex_round_up(DV, 64);- size_t Br = 0, Bc = 0;- int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, sess->vtcm_size, sess->n_threads, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head);- if (ret == 0) {- kparams->kernel_type = HTP_FA_KERNEL_HMX;- if (logit_softcap == 0.0f) {- kparams->scale = scale * (float) M_LOG2E;- kparams->logit_softcap = 0.0f;- } else {- kparams->scale = scale;- kparams->logit_softcap = logit_softcap * (float) M_LOG2E;- }- kparams->Br = Br;- kparams->Bc = Bc;- kparams->n_kv_blocks = (nek1 + Bc - 1) / Bc;- kparams->n_threads = (kparams->n_kv_blocks >= 3 && sess->n_threads >= 2) ? sess->n_threads : 1;-- kparams->u.hmx.g_br = hex_align_up(G * Br, 32);- kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && sess->n_threads >= 2) ? 1 : 0;- kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK_pad, DV_pad, Br, Bc, kparams->n_threads, kparams->u.hmx.pipeline != 0, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head);-- const size_t row_vec_bytes = hex_align_up(Bc * sizeof(uint16_t), 256);- kparams->u.hmx.row_buf_stride = row_vec_bytes / 128; // HVX vector is 128 bytes-- const size_t m_line_bytes = hex_align_up(Bc * sizeof(uint16_t), 128);- kparams->u.hmx.mask_buf_row_stride = m_line_bytes / sizeof(uint16_t);- kparams->u.hmx.mask_broadcast = (mask != nullptr && mask->ne[2] == 1) ? 1 : 0;- kparams->u.hmx.div_G = init_fastdiv_values(G);- if (mask) {- kparams->src3_div2 = init_fastdiv_values(mask->ne[2]);- kparams->src3_div3 = init_fastdiv_values(mask->ne[3]);- }-- kparams->qrows = 0;- kparams->qrows_per_thread = 0;- return true;- }- }-- // Fallback to HVX- kparams->kernel_type = HTP_FA_KERNEL_HVX;- kparams->Br = 1;- kparams->Bc = 64; // FLASH_ATTN_BLOCK_SIZE- kparams->n_kv_blocks = (k->ne[1] + 64 - 1) / 64;- kparams->n_threads = sess->n_threads;-- const size_t size_q_row_padded = hex_round_up(q->ne[0] * ((q->type == GGML_TYPE_F32) ? 4 : 2), 128);- const size_t size_k_row_padded = hex_round_up(k->ne[0] * 2, 128);- const size_t size_v_row_padded = hex_round_up(v->ne[0] * 2, 128);-- kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, (q->type == GGML_TYPE_F32), mask != nullptr, sinks != nullptr, n_head, sess->n_threads);-- kparams->u.hvx.size_q_row_padded = size_q_row_padded;- kparams->u.hvx.size_k_row_padded = size_k_row_padded;- kparams->u.hvx.size_v_row_padded = size_v_row_padded;- kparams->u.hvx.src0_div21 = init_fastdiv_values(q->ne[2] * q->ne[1]);- kparams->u.hvx.src0_div1 = init_fastdiv_values(q->ne[1]);- kparams->broadcast_rk2 = init_fastdiv_values(q->ne[2]/k->ne[2]);- kparams->broadcast_rk3 = init_fastdiv_values(q->ne[3]/k->ne[3]);- kparams->broadcast_rv2 = init_fastdiv_values(q->ne[2]/v->ne[2]);- kparams->broadcast_rv3 = init_fastdiv_values(q->ne[3]/v->ne[3]);- if (mask) {- kparams->src3_div2 = init_fastdiv_values(mask->ne[2]);- kparams->src3_div3 = init_fastdiv_values(mask->ne[3]);- }-- kparams->qrows = q->ne[1] * q->ne[2] * q->ne[3];- kparams->qrows_per_thread = (kparams->qrows + sess->n_threads - 1) / sess->n_threads;-- return true;-}--static bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * src1 = op->src[1];- const struct ggml_tensor * src2 = op->src[2];- const struct ggml_tensor * src3 = op->src[3];- const struct ggml_tensor * src4 = op->src[4];- const struct ggml_tensor * dst = op;-- // Check for F16/Q8_0 support- if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) ||- (src1->type != GGML_TYPE_F16 && src1->type != GGML_TYPE_Q8_0) ||- (src2->type != GGML_TYPE_F16 && src2->type != GGML_TYPE_Q8_0)) {- return false;- }-- if (src3 && src3->type != GGML_TYPE_F16) { // mask- return false;- }-- if (src4 && src4->type != GGML_TYPE_F32) { // sinks- return false;- }-- // For now we support F32 or F16 output as htp backend often converts output on the fly if needed,- // but the op implementation writes to F16 or F32.- // Let's assume dst can be F32 or F16.- if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) {- return false;- }-- struct htp_fa_kernel_params kparams;- if (!ggml_hexagon_precompute_flash_attn_params(sess, op, &kparams)) {- return false;- }-- if ((size_t) kparams.vtcm_size > sess->vtcm_size) {- HEX_VERBOSE("ggml-hex: skip flash_attn_ext because VTCM needed (%d) > budget (%zu)\n",- kparams.vtcm_size, sess->vtcm_size);- return false;- }-- return true;-}--static bool ggml_hexagon_supported_gated_delta_net(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- if (opt_gdn_select < 1) {- return false;- }-- const struct ggml_tensor * q = op->src[0];- const struct ggml_tensor * k = op->src[1];- const struct ggml_tensor * v = op->src[2];- const struct ggml_tensor * g = op->src[3];- const struct ggml_tensor * beta = op->src[4];- const struct ggml_tensor * state = op->src[5];- const struct ggml_tensor * dst = op;-- if (q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F32 || v->type != GGML_TYPE_F32 ||- g->type != GGML_TYPE_F32 || beta->type != GGML_TYPE_F32 || state->type != GGML_TYPE_F32 ||- dst->type != GGML_TYPE_F32) {- return false;- }-- if (!ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) || !ggml_is_contiguous_rows(v) ||- !ggml_is_contiguous(g) || !ggml_is_contiguous(beta) || !ggml_is_contiguous(state) ||- !ggml_is_contiguous(dst)) {- return false;- }-- const int64_t S_v = v->ne[0];- const int64_t H = v->ne[1];- const int64_t n_tokens = v->ne[2];- const int64_t n_seqs = v->ne[3];- const int64_t K = ggml_get_op_params_i32(op, 0);-- if (S_v <= 0 || S_v > 128 || H <= 0 || n_tokens <= 0 || n_seqs <= 0) {- return false;- }- if (q->ne[0] != S_v || k->ne[0] != S_v || q->ne[1] <= 0 || k->ne[1] <= 0 ||- q->ne[2] != n_tokens || k->ne[2] != n_tokens || q->ne[3] <= 0 || k->ne[3] <= 0 ||- (n_seqs % q->ne[3]) != 0 || (n_seqs % k->ne[3]) != 0) {- return false;- }- if ((g->ne[0] != 1 && g->ne[0] != S_v) || beta->ne[0] != 1) {- return false;- }- // state holds s0 only [S_v, S_v, H, n_seqs]; K is op param 0.- if (ggml_nelements(state) != S_v * S_v * H * n_seqs) {- return false;- }- if (dst->ne[0] != S_v * H || dst->ne[1] != n_tokens * n_seqs + S_v * n_seqs * K) {- return false;- }-- const uint32_t total_rows = (uint32_t) (H * n_seqs);- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, total_rows);-- const bool can_use_hmx = (opt_gdn_select >= 2) &&- (sess->n_hmx > 0) &&- (S_v % 64 == 0) &&- (n_tokens >= HTP_GDN_MIN_TOKENS) &&- (g->ne[0] == 1) &&- (K == 1);-- if (can_use_hmx) {- struct htp_gdn_hmx_vtcm_layout layout;- uint32_t n_heads_batch = 0;- if (!htp_gdn_hmx_solve_layout(&layout, (uint32_t) S_v, HTP_GDN_CHUNK_SIZE, total_rows, sess->vtcm_size, n_threads, true, &n_heads_batch)) {- return false;- }- } else {- struct htp_gdn_vtcm_layout layout;- htp_gdn_vtcm_layout_build(&layout, (uint32_t) S_v, n_threads);- if (layout.total_bytes > sess->vtcm_size) {- return false;- }- }-- return true;-}--static bool ggml_hexagon_matmul_is_hmx_eligible(- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * dst,- bool is_matmul_id,- bool is_batched-) {- GGML_UNUSED(dst);-- if (src1->type != GGML_TYPE_F32 && (src1->type != GGML_TYPE_F16 || is_matmul_id)) {- return false;- }-- const int ne00 = src0->ne[0];- const int ne01 = src0->ne[1];- const int ne11 = src1->ne[1];- const int ne12 = src1->ne[2];- const int wtype = src0->type;-- // HMX weight tiles accept non-32-aligned N for non-matmul_id.- if (ne01 % HTP_TENSOR_TILE_N_ROWS != 0 && is_matmul_id) {- return false;- }-- // HMX supports F16, F32, and repack quantized types.- if (!htp_is_hmx_weight_type((ggml_type) wtype)) {- return false;- }-- // HMX paths require K aligned to HMX tile cols (32).- if (ne00 % HTP_TENSOR_TILE_N_COLS != 0) {- return false;- }-- // Quantized and tiled HMX kernels only handle flat 2D matmul (or matmul_id wrapping flat 2D matmuls).- const bool is_tiled = ggml_hexagon_tensor_is_tiled(src0);- if (ggml_is_quantized((ggml_type) wtype) && !is_tiled) {- return false;- }- if (!is_matmul_id && is_batched && is_tiled) {- return false;- }-- // HMX assumes contiguous row-major layout.- if (src0->nb[0] > src0->nb[1] || src1->nb[0] > src1->nb[1]) {- return false;- }-- // M alignment: Use HMX when M > HTP_MM_HMX_MIN_NROWS.- // For MUL_MAT_ID, src1 shape is [K, n_expert_used, n_tokens, 1], so n_tokens is ne12.- const int m = is_matmul_id ? ne12 : ne11;- if (m <= HTP_MM_HMX_MIN_NROWS) {- return false;- }-- return true;-- GGML_UNUSED(dst);-}--static bool ggml_hexagon_precompute_hmx_mm_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * act,- const struct ggml_tensor * dst,- int wtype,- int ne00_padded,- int ne01_padded,- int ne02,- int ne11,- int ne12,- int ne11_padded,- bool is_matmul_id,- bool is_batched,- size_t src2_size,- size_t vtcm_budget,- struct htp_mm_kernel_params * kparams-) {- const bool is_tiled = ggml_hexagon_tensor_is_tiled(src0);- const int aligned_tile_size = is_tiled ? htp_tensor_get_tile_padded_size(wtype) : 0;- const int n_threads = (int)sess->n_threads;- const int ne10 = act->ne[0];-- int m_for_solver = ne11;- int m_for_solver_padded = ne11_padded;- // matmul_id partitions by expert; regular matmul partitions M rows (ne11) across devices- if (!is_matmul_id && sess->mdev.count > 1 && ((uint32_t) ne11 >= sess->mdev.count)) {- // when dst is null, padded dims are used for estimate which are 128-byte aligned- const bool dst_row_split = dst ? ggml_hexagon_can_row_partition(dst) : true;- const bool act_row_split = ggml_hexagon_can_row_partition(act);- if (dst_row_split && act_row_split) {- m_for_solver = (ne11 + (int) sess->mdev.count - 1) / (int) sess->mdev.count;- m_for_solver_padded = hex_round_up(std::max(m_for_solver, 32), 32);- }- }- const bool pipeline = is_matmul_id ? false : htp_mm_hmx_pipeline(m_for_solver);-- const bool is_batched_val = is_matmul_id ? false : is_batched;- const int group_size = (ne02 > 0 ? ne12 / ne02 : 1);-- size_t m_chunk = 0;- size_t n_chunk = 0;- size_t vtcm_size = 0;- bool use_grouped = false;- int act_threads_selected = 0;-- if (is_batched_val && wtype == GGML_TYPE_F16 && !is_tiled && group_size > 1) {- // Try grouped path first- if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, m_for_solver, group_size, n_threads, pipeline, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) {- use_grouped = true;- }- }-- if (!use_grouped) {- // Fallback to simple 2D path (group_size = 1)- int m_id_rows = 0;- if (dst && is_matmul_id) {- const int n_experts = ne02 > 0 ? ne02 : 1;- const size_t total_expert_rows = (size_t) dst->ne[1] * dst->ne[2];- int m_per_expert = (int) ((total_expert_rows + n_experts - 1) / n_experts);- if (sess->mdev.count > 1 && ggml_hexagon_can_row_partition(dst)) {- m_per_expert = (m_per_expert + (int) sess->mdev.count - 1) / (int) sess->mdev.count;- }- m_id_rows = hex_round_up(std::max(m_per_expert, 32), 32);- }- const uint32_t cost_m = is_matmul_id ? (uint32_t) m_id_rows : (uint32_t) m_for_solver;- if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, m_for_solver_padded, cost_m, n_threads, pipeline, is_matmul_id, aligned_tile_size, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) {- return false;- }- }-- kparams->n_hmx = 1;- kparams->pipeline = pipeline ? 1 : 0;- kparams->m_chunk = m_chunk;- kparams->n_chunk = n_chunk;- kparams->n_threads = n_threads;- kparams->n_act_threads = act_threads_selected;- kparams->tile_size = is_tiled ? htp_tensor_get_tile_size(wtype) : 0;- kparams->aligned_tile_size = aligned_tile_size;- kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);- kparams->vtcm_size = vtcm_size;- kparams->vtcm_src0_size = 0;- kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected);- kparams->div_ne00_padded = init_fastdiv_values(ne00_padded);- kparams->vtcm_act_size = 0;- kparams->vtcm_bias_size = (int32_t) src2_size;- kparams->vtcm_dst_size = 0;-- if (is_batched && !is_matmul_id) {- kparams->kernel_type = HTP_MM_KERNEL_HMX_BATCHED;- } else {- kparams->kernel_type = HTP_MM_KERNEL_HMX_2D;- }- return true;-}--static void ggml_hexagon_precompute_hvx_mm_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * dst,- int wtype,- bool is_tiled,- int ne02,- int ne03,- int ne10,- int ne11,- int ne12,- int ne13,- bool is_matmul_id,- const size_t src2_row_size,- size_t vtcm_budget,- struct htp_mm_kernel_params * kparams-) {- if (opt_mm_select < 1) {- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;- return;- }-- kparams->n_hmx = 0;- kparams->n_threads = sess->n_threads;-- GGML_UNUSED(ne02);- GGML_UNUSED(ne03);-- const int src1_nrows = ne11 * ne12 * ne13;-- if (is_tiled) {- // Tiled HVX (quantized weights or tiled F16/F32/BF16)- kparams->tile_size = htp_tensor_get_tile_size(wtype);- kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);-- const bool k_align = (ne10 % htp_tensor_get_tile_k(wtype) == 0);-- if (is_matmul_id) {- kparams->kernel_type = (src1_nrows < (int) sess->n_threads)- ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW;- kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);-- struct htp_mm_hvx_vtcm_layout L;- uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;- uint32_t best_n_prefetch = 2;- for (uint32_t d = max_prefetch; d >= 2; d /= 2) {- htp_mm_hvx_vtcm_layout_build(- &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads,- 0, src0->nb[1], kparams->act_row_size, 0, d, true, false- );- if (L.total_bytes <= vtcm_budget) {- best_n_prefetch = d;- break;- }- }- if (L.total_bytes > vtcm_budget) {- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;- return;- }- kparams->n_prefetch = best_n_prefetch;- kparams->vtcm_size = L.total_bytes;- kparams->vtcm_src0_size = L.src0_bytes;- kparams->vtcm_act_size = L.act_bytes;- kparams->vtcm_bias_size = 0;- kparams->vtcm_dst_size = L.dst_bytes;- return;- } else {- bool try_tiled = (k_align && opt_mm_select >= 1);- if (try_tiled) {- kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);- if (src1_nrows < (int) sess->n_threads) {- kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_BLOCK;- } else {- kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW;- }-- struct htp_mm_hvx_vtcm_layout L;- uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;- uint32_t best_n_prefetch = 2;- for (uint32_t d = max_prefetch; d >= 2; d /= 2) {- htp_mm_hvx_vtcm_layout_build(- &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads,- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, d, false, false- );- if (L.total_bytes <= vtcm_budget) {- best_n_prefetch = d;- break;- }- }-- uint32_t m_chunk = 0;- if (htp_mm_hvx_solve_vtcm_params(- kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads,- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, best_n_prefetch, vtcm_budget,- &L, &m_chunk)) {- kparams->n_prefetch = best_n_prefetch;- kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0;- kparams->vtcm_size = L.total_bytes;- kparams->vtcm_src0_size = L.src0_bytes;- kparams->vtcm_act_size = L.act_bytes;- kparams->vtcm_bias_size = L.bias_bytes;- kparams->vtcm_dst_size = L.dst_bytes;- return;- }- }-- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;- return;- }- } else if (is_matmul_id) {- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;- return;- } else if (wtype == GGML_TYPE_F16) {- // Flat F16 HVX- struct htp_mm_hvx_vtcm_layout L;- uint32_t m_chunk = 0;- if (htp_mm_hvx_solve_vtcm_params(- HTP_MM_KERNEL_HVX_F16_F16_VTCM, wtype, ne10, src1_nrows, sess->n_threads,- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget,- &L, &m_chunk)) {- kparams->kernel_type = HTP_MM_KERNEL_HVX_F16_F16_VTCM;- kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0;- kparams->act_row_size = hex_round_up(ne10 * 2, 128);- kparams->vtcm_size = L.total_bytes;- kparams->vtcm_src0_size = L.src0_bytes;- kparams->vtcm_act_size = L.act_bytes;- kparams->vtcm_bias_size = L.bias_bytes;- kparams->vtcm_dst_size = L.dst_bytes;- kparams->n_prefetch = 16;- return;- }-- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;- return;- } else if (wtype == GGML_TYPE_F32) {- // Flat F32 HVX- struct htp_mm_hvx_vtcm_layout L;- uint32_t m_chunk = 0;- if (htp_mm_hvx_solve_vtcm_params(- HTP_MM_KERNEL_HVX_F32_F32_VTCM, wtype, ne10, src1_nrows, sess->n_threads,- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget,- &L, &m_chunk)) {- kparams->kernel_type = HTP_MM_KERNEL_HVX_F32_F32_VTCM;- kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0;- kparams->act_row_size = hex_round_up(ne10 * 4, 128);- kparams->vtcm_size = L.total_bytes;- kparams->vtcm_src0_size = L.src0_bytes;- kparams->vtcm_act_size = L.act_bytes;- kparams->vtcm_bias_size = L.bias_bytes;- kparams->vtcm_dst_size = L.dst_bytes;- kparams->n_prefetch = 16;- return;- }-- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;- return;- } else if (wtype == GGML_TYPE_BF16) {- // Flat BF16 HVX- struct htp_mm_hvx_vtcm_layout L;- uint32_t m_chunk = 0;- if (htp_mm_hvx_solve_vtcm_params(- HTP_MM_KERNEL_HVX_BF16_F32_VTCM, wtype, ne10, src1_nrows, sess->n_threads,- dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget,- &L, &m_chunk)) {- kparams->kernel_type = HTP_MM_KERNEL_HVX_BF16_F32_VTCM;- kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0;- kparams->act_row_size = hex_round_up(ne10 * 4, 128);- kparams->vtcm_size = L.total_bytes;- kparams->vtcm_src0_size = L.src0_bytes;- kparams->vtcm_act_size = L.act_bytes;- kparams->vtcm_bias_size = L.bias_bytes;- kparams->vtcm_dst_size = L.dst_bytes;- kparams->n_prefetch = 16;- return;- }-- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;- return;- } else {- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;- return;- }-}--static void ggml_hexagon_precompute_matmul_params_impl(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * dst,- const size_t src2_row_size,- const size_t src2_size,- struct htp_mm_kernel_params * kparams-) {- memset(kparams, 0, sizeof(*kparams));-- const int ne00 = src0->ne[0];- const int ne01 = src0->ne[1];- const int ne02 = src0->ne[2];- const int ne03 = src0->ne[3];-- const int ne10 = src1->ne[0];- const int ne11 = src1->ne[1];- const int ne12 = src1->ne[2];- const int ne13 = src1->ne[3];-- const bool is_tiled = ggml_hexagon_tensor_is_tiled(src0);- const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;- const int ne00_padded = is_tiled ? hex_round_up(ne00, htp_tensor_get_tile_k(wtype)) : ne00;- const int ne11_padded = hex_round_up(ne11, 32);- // VTCM has to hold whole 32-row weight tiles, so size for the rounded-up N- // even when the tensor itself is ragged.- const int ne01_tiled = hex_round_up(ne01, 32);-- const bool is_matmul_id = (dst->op == GGML_OP_MUL_MAT_ID);- const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1);-- const size_t vtcm_budget = sess->vtcm_size;-- // Check HMX eligibility and try precomputing HMX parameters- bool hmx_enabled = (sess->n_hmx > 0) && (opt_mm_select >= 2);- if (hmx_enabled && ggml_hexagon_matmul_is_hmx_eligible(src0, src1, dst, is_matmul_id, is_batched)) {- if (ggml_hexagon_precompute_hmx_mm_params(sess, src0, src1, dst, wtype, ne00_padded, ne01_tiled, ne02, ne11, ne12, ne11_padded, is_matmul_id, is_batched, src2_size, vtcm_budget, kparams)) {- goto finalize;- }- }-- // Fallback to HVX parameter computation- ggml_hexagon_precompute_hvx_mm_params(sess, src0, src1, dst, wtype, is_tiled, ne02, ne03, ne10, ne11, ne12, ne13, is_matmul_id, src2_row_size, vtcm_budget, kparams);--finalize:- kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11);- kparams->div_ne1 = init_fastdiv_values(ne11);- kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1);- kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1);- kparams->div_ne12 = init_fastdiv_values(ne12);-}--// The rows of dims 1..3 can be walked with one stride (size-1 dims skipped); returns that stride-static bool ggml_hexagon_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride) {- size_t s = 0, next = 0;- for (int i = 1; i < GGML_MAX_DIMS; i++) {- if (ne[i] == 1) continue;- if (s == 0) { s = nb[i]; next = s * ne[i]; continue; }- if (nb[i] != next) return false;- next *= ne[i];- }- *stride = s ? s : nb[1];- return true;-}--// A 2D weight applied to a batched activation whose rows are evenly strided is the same matmul over ne11 * ne12 * ne13 rows-static bool ggml_hexagon_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst) {- size_t s1, sd;- return (dst->op == GGML_OP_MUL_MAT || dst->op == GGML_OP_ADD) &&- src0->ne[2] == 1 && src0->ne[3] == 1 && src1->ne[2] * src1->ne[3] > 1 &&- src1->nb[0] == ggml_type_size(src1->type) && ggml_hexagon_rows_stride(src1->ne, src1->nb, &s1) &&- dst->nb[0] == ggml_type_size(dst->type) && ggml_hexagon_rows_stride(dst->ne, dst->nb, &sd);-}--static bool ggml_hexagon_matmul_add_can_collapse(- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * src2,- const struct ggml_tensor * dst-) {- if (!ggml_hexagon_matmul_can_collapse(src0, src1, dst)) {- return false;- }- if (!src2) {- return true;- }- const int64_t src2_nrows = src2->ne[1] * src2->ne[2] * src2->ne[3];- if (src2_nrows == 1) {- return true;- }- size_t s2;- return src2->nb[0] == ggml_type_size(src2->type) &&- src2->ne[0] == dst->ne[0] &&- src2->ne[1] == dst->ne[1] &&- src2->ne[2] == dst->ne[2] &&- src2->ne[3] == dst->ne[3] &&- ggml_hexagon_rows_stride(src2->ne, src2->nb, &s2);-}--static ggml_tensor ggml_hexagon_tensor_collapse_rows(const struct ggml_tensor * t) {- size_t stride = 0;- ggml_hexagon_rows_stride(t->ne, t->nb, &stride);- ggml_tensor c = *t;- c.ne[1] = t->ne[1] * t->ne[2] * t->ne[3];- c.ne[2] = 1;- c.ne[3] = 1;- c.nb[1] = stride;- c.nb[2] = c.nb[1] * c.ne[1];- c.nb[3] = c.nb[2];- return c;-}--static void ggml_hexagon_precompute_matmul_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * dst,- struct htp_mm_kernel_params * kparams-) {- if (ggml_hexagon_matmul_can_collapse(src0, src1, dst)) {- const ggml_tensor src1_collapsed = ggml_hexagon_tensor_collapse_rows(src1);- const ggml_tensor dst_collapsed = ggml_hexagon_tensor_collapse_rows(dst);- ggml_hexagon_precompute_matmul_params_impl(sess, src0, &src1_collapsed, &dst_collapsed, 0, 0, kparams);- kparams->collapse = 1;- return;- }- ggml_hexagon_precompute_matmul_params_impl(sess, src0, src1, dst, 0, 0, kparams);-}--static void ggml_hexagon_precompute_fused_matmul_add_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * src2,- const struct ggml_tensor * dst,- struct htp_mm_kernel_params * kparams-) {- if (ggml_hexagon_matmul_add_can_collapse(src0, src1, src2, dst)) {- const ggml_tensor src1_collapsed = ggml_hexagon_tensor_collapse_rows(src1);- const ggml_tensor dst_collapsed = ggml_hexagon_tensor_collapse_rows(dst);- const ggml_tensor src2_collapsed = (src2 && (src2->ne[1] * src2->ne[2] * src2->ne[3] > 1))- ? ggml_hexagon_tensor_collapse_rows(src2)- : (src2 ? *src2 : ggml_tensor{});- const struct ggml_tensor * p_src2 = src2 ? &src2_collapsed : nullptr;- const size_t src2_size = p_src2 ? hex_round_up(ggml_nbytes(p_src2), 128) : 0;- ggml_hexagon_precompute_matmul_params_impl(sess, src0, &src1_collapsed, &dst_collapsed, p_src2 ? p_src2->nb[1] : 0, src2_size, kparams);- kparams->collapse = 1;- return;- }- const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0;- ggml_hexagon_precompute_matmul_params_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, kparams);-}--static bool ggml_hexagon_precompute_binary_params(- const struct ggml_hexagon_session * sess,- uint32_t op,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * dst,- struct htp_binary_kernel_params * kparams,- const struct ggml_tensor * src2-) {- memset(kparams, 0, sizeof(*kparams));-- const size_t elem_size = ggml_type_size(src0->type);- const size_t src0_row_size = src0->ne[0] * elem_size;- const size_t src1_row_size = src1->ne[0] * elem_size;- const size_t dst_row_size = dst->ne[0] * elem_size;-- const size_t src0_row_size_aligned = hex_round_up(src0_row_size, 128);- const size_t src1_row_size_aligned = hex_round_up(src1_row_size, 128);- const size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128);-- if (!htp_binary_strides_safe(src0, src1, dst, elem_size)) {- return false;- }-- const bool is_add_id = op == HTP_OP_ADD_ID;- const bool is_scalar = !is_add_id && src1->ne[0] == 1;- const bool is_row_bcast = !is_add_id && !is_scalar &&- src1->ne[0] == src0->ne[0] &&- (src0->ne[1] > 1 || src0->ne[2] > 1 || src0->ne[3] > 1) &&- src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1;- const bool is_same_shape = !is_add_id && !is_scalar &&- src1->ne[0] == src0->ne[0] &&- src1->ne[1] == src0->ne[1] &&- (src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) &&- (src1->ne[3] == src0->ne[3] || src1->ne[3] == 1);- const bool is_complex = !is_add_id && !is_scalar && !is_same_shape && !is_row_bcast && (src1->ne[0] == src0->ne[0]);- const bool is_contig = ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst);- const bool is_scalar_broadcast = !is_add_id && (ggml_nelements(src1) == 1);-- if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) {- const uint32_t total_elems = (uint32_t) ggml_nelements(src0);- const uint32_t n_threads = sess->n_threads;- const uint32_t max_chunk_elems = 32768 / elem_size;- const uint32_t min_chunk_elems = 256;- const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * n_threads) - 1) / (2 * n_threads), 32);- const uint32_t chunk_size = (std::min)(max_chunk_elems, (std::max)(target_chunk_elems, min_chunk_elems));- const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128);-- kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED;- kparams->n_threads = n_threads;- kparams->rows_per_buffer = 1;- kparams->src0_row_size_aligned = src0_row_size_aligned;- kparams->src1_row_size_aligned = is_scalar_broadcast ? 0 : src1_row_size_aligned;- kparams->dst_row_size_aligned = dst_row_size_aligned;- kparams->src1_size = 0;- kparams->chunk_size = chunk_size;- kparams->chunk_bytes = chunk_bytes;- kparams->is_scalar = is_scalar_broadcast ? 1 : 0;-- struct htp_binary_vtcm_layout L;- htp_binary_vtcm_layout_build(&L, kparams, sess->vtcm_size);- if (L.total_bytes == 0 || L.total_bytes > sess->vtcm_size) {- return false;- }-- kparams->vtcm_size = L.total_bytes;- return true;- }-- enum htp_binary_kernel_type kernel_type;- size_t src1_size = 0;- size_t src2_size = 0;-- if (is_add_id) {- GGML_ASSERT(src2 != nullptr);- kernel_type = HTP_BINARY_KERNEL_ADD_ID;- src1_size = hex_round_up(src1->ne[1] * src1_row_size_aligned, 128);- src2_size = hex_round_up(ggml_nelements(src2) * sizeof(int32_t), 128);- if (src1_size + src2_size > sess->vtcm_size) {- return false;- }- } else if (is_row_bcast) {- kernel_type = HTP_BINARY_KERNEL_ROW_BCAST;- src1_size = src1_row_size_aligned;- } else if (is_scalar) {- src1_size = hex_round_up(ggml_nelements(src1) * elem_size, 128);- if (src1_size > sess->vtcm_size) {- return false;- }- kernel_type = HTP_BINARY_KERNEL_SCALAR;- kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;- } else if (is_same_shape) {- kernel_type = HTP_BINARY_KERNEL_SAME_SHAPE;- } else if (is_complex) {- src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128);- if (src1_size > sess->vtcm_size) {- return false;- }- kernel_type = HTP_BINARY_KERNEL_COMPLEX;- kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;- } else {- src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128);- if (src1_size > sess->vtcm_size) {- return false;- }- kernel_type = HTP_BINARY_KERNEL_REPEAT;- kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;- }-- kparams->kernel_type = kernel_type;- kparams->n_threads = sess->n_threads;- kparams->src0_row_size_aligned = src0_row_size_aligned;- kparams->src1_row_size_aligned = src1_row_size_aligned;- kparams->dst_row_size_aligned = dst_row_size_aligned;- kparams->src1_size = src1_size;- kparams->src2_size = src2_size;-- const bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]);- const bool src1_contig_dim1 = (src1->nb[2] == src1->ne[1] * src1->nb[1]);- const bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]);-- const bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]);- const bool src1_contig_dim2 = (src1->nb[3] == src1->ne[2] * src1->nb[2]);- const bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]);-- bool split_at_ne01 = false;- if (src0->ne[2] > 1) {- if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {- split_at_ne01 = (src1->ne[2] != src0->ne[2]) || !src0_contig_dim1 || !src1_contig_dim1 || !dst_contig_dim1;- } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {- split_at_ne01 = !src0_contig_dim1 || !dst_contig_dim1;- } else {- split_at_ne01 = true;- }- }- kparams->split_at_ne01 = split_at_ne01 ? 1 : 0;-- bool split_at_ne02 = false;- if (src0->ne[3] > 1) {- if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {- split_at_ne02 = (src1->ne[3] != src0->ne[3]) || !src0_contig_dim2 || !src1_contig_dim2 || !dst_contig_dim2;- } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {- split_at_ne02 = !src0_contig_dim2 || !dst_contig_dim2;- } else {- split_at_ne02 = true;- }- }- kparams->split_at_ne02 = split_at_ne02 ? 1 : 0;-- struct htp_binary_vtcm_layout L;- htp_binary_vtcm_layout_build(&L, kparams, sess->vtcm_size);- if (L.rows_per_buffer == 0 || L.total_bytes > sess->vtcm_size) {- if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) {- const uint32_t total_elems = (uint32_t) ggml_nelements(src0);- const uint32_t n_threads = sess->n_threads;- const uint32_t max_chunk_elems = 32768 / elem_size;- const uint32_t min_chunk_elems = 256;- const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * n_threads) - 1) / (2 * n_threads), 32);- const uint32_t chunk_size = (std::min)(max_chunk_elems, (std::max)(target_chunk_elems, min_chunk_elems));- const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128);-- kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED;- kparams->n_threads = n_threads;- kparams->rows_per_buffer = 1;- kparams->src1_row_size_aligned = is_scalar_broadcast ? 0 : src1_row_size_aligned;- kparams->src1_size = 0;- kparams->chunk_size = chunk_size;- kparams->chunk_bytes = chunk_bytes;- kparams->is_scalar = is_scalar_broadcast ? 1 : 0;-- htp_binary_vtcm_layout_build(&L, kparams, sess->vtcm_size);- if (L.total_bytes == 0 || L.total_bytes > sess->vtcm_size) {- return false;- }-- kparams->vtcm_size = L.total_bytes;- return true;- }- return false;- }-- kparams->rows_per_buffer = L.rows_per_buffer;- kparams->vtcm_size = L.total_bytes;-- return true;-}--static void ggml_hexagon_precompute_unary_params(- const struct ggml_hexagon_session * sess,- uint32_t op,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * dst,- struct htp_unary_kernel_params * kparams-) {- memset(kparams, 0, sizeof(*kparams));-- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];- const uint32_t n_threads = (std::min)((uint32_t)sess->n_threads, src0_nrows);-- kparams->n_threads = n_threads;-- const size_t elem_size = ggml_type_size(src0->type);-- const size_t src0_data_row_size = src0->ne[0] * elem_size;- const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type);-- const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);- const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128);-- kparams->src0_row_size_aligned = src0_row_size_aligned;- kparams->dst_row_size_aligned = dst_row_size_aligned;-- size_t src1_data_row_size = 0;- size_t src1_row_size_aligned = 0;- bool broadcast_weight = false;-- if (op == HTP_OP_RMS_NORM_MUL) {- GGML_ASSERT(src1 != nullptr);- src1_data_row_size = src1->ne[0] * ggml_type_size(src1->type);- src1_row_size_aligned = hex_round_up(src1_data_row_size, 128);- broadcast_weight = (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1);- }-- kparams->src1_row_size_aligned = src1_row_size_aligned;- kparams->broadcast_weight = broadcast_weight;-- struct htp_unary_vtcm_layout L;- uint32_t col_tile = 0;- uint32_t vtcm_row_per_thread = 0;-- htp_unary_vtcm_layout_build(&L, op, src0->ne[0], dst->ne[0],- op == HTP_OP_RMS_NORM_MUL ? src1->ne[0] : 0,- broadcast_weight, n_threads, sess->vtcm_size, elem_size,- &col_tile, &vtcm_row_per_thread);-- kparams->col_tile = col_tile;- kparams->vtcm_row_per_thread = vtcm_row_per_thread;- kparams->vtcm_size = L.total_bytes;-- kparams->vtcm_src0_size_per_thread = L.src0_bytes;- kparams->vtcm_src1_size_per_thread = L.src1_bytes;- kparams->vtcm_dst_size_per_thread = L.dst_bytes;-- kparams->vtcm_src0_size = L.src0_bytes * n_threads;- kparams->vtcm_src1_size = L.src1_bytes * n_threads;- kparams->vtcm_dst_size = L.dst_bytes * n_threads;-- kparams->block = col_tile ? 0 : ((L.src0_bytes / 2) / src0_row_size_aligned);-- const uint32_t tiles_per_row = col_tile > 0 ? (src0->ne[0] + col_tile - 1) / col_tile : 1;- kparams->div_ne01 = init_fastdiv_values(src0->ne[1]);- kparams->div_ne02 = init_fastdiv_values(src0->ne[2]);- kparams->div_ne012 = init_fastdiv_values(src0->ne[1] * src0->ne[2]);- kparams->div_tpr = init_fastdiv_values(tiles_per_row);-}--static void ggml_hexagon_precompute_get_rows_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * dst,- struct htp_get_rows_kernel_params * kparams-) {- memset(kparams, 0, sizeof(*kparams));-- const uint32_t ne00 = src0->ne[0];- const uint32_t ne02 = src0->ne[2];- const uint32_t ne03 = src0->ne[3];-- const uint32_t ne10 = src1->ne[0];- const uint32_t ne11 = src1->ne[1];- const uint32_t ne12 = src1->ne[2];- const uint32_t nr = ne10 * ne11 * ne12;-- const bool tiled = ggml_hexagon_tensor_is_tiled(src0);- const int wtype = (tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;-- if (tiled) {- kparams->kernel_type = HTP_GET_ROWS_KERNEL_TILED;- } else if (src0->type == dst->type) {- kparams->kernel_type = HTP_GET_ROWS_KERNEL_SAMETYPE;- } else {- kparams->kernel_type = HTP_GET_ROWS_KERNEL_FLAT;- }-- const uint32_t total_tasks = nr;-- kparams->n_threads = (std::min)((uint32_t)sess->n_threads, total_tasks);-- struct htp_get_rows_vtcm_layout vtcm_layout = {};- while (kparams->n_threads > 0) {- htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, kparams->n_threads);- if (vtcm_layout.total_bytes <= sess->vtcm_size) {- break;- }- --kparams->n_threads;- }-- if (kparams->n_threads == 0 && total_tasks > 0) {- htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, 1);- }-- kparams->vtcm_size = (total_tasks == 0) ? 0 : vtcm_layout.total_bytes;- kparams->tasks_per_thread = kparams->n_threads > 0 ? (total_tasks + kparams->n_threads - 1) / kparams->n_threads : 0;- kparams->total_tasks = total_tasks;-- kparams->div_ne10 = ne10 > 0 ? init_fastdiv_values(ne10) : fastdiv_values{0, 0};- kparams->div_ne10_ne11 = (ne10 * ne11) > 0 ? init_fastdiv_values(ne10 * ne11) : fastdiv_values{0, 0};- kparams->div_ne02 = ne02 > 0 ? init_fastdiv_values(ne02) : fastdiv_values{0, 0};- kparams->div_ne03 = ne03 > 0 ? init_fastdiv_values(ne03) : fastdiv_values{0, 0};-}--static void ggml_hexagon_precompute_set_rows_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0, // values- const struct ggml_tensor * src1, // indices- const struct ggml_tensor * dst, // destination- struct htp_set_rows_kernel_params * kparams-) {- memset(kparams, 0, sizeof(*kparams));-- const uint32_t nr = src0->ne[1];-- kparams->n_threads = (std::min)((uint32_t)sess->n_threads, nr);- kparams->tasks_per_thread = (nr + kparams->n_threads - 1) / kparams->n_threads;- kparams->total_tasks = nr;-- kparams->div_ne11 = init_fastdiv_values(src1->ne[1]);- kparams->div_ne12 = init_fastdiv_values(src1->ne[2]);- kparams->div_tasks_per_thread = init_fastdiv_values(kparams->tasks_per_thread);- kparams->div_ne02 = init_fastdiv_values(src0->ne[2]);-- struct htp_set_rows_vtcm_layout vtcm_layout;- htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_threads);- kparams->vtcm_size = vtcm_layout.total_bytes;-}--static void ggml_hexagon_precompute_softmax_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_softmax_kernel_params * kparams-) {- memset(kparams, 0, sizeof(*kparams));-- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * src1 = op->src[1];-- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows);-- float scale = 1.0f;- float max_bias = 0.0f;- memcpy(&scale, &op->op_params[0], sizeof(float));- memcpy(&max_bias, &op->op_params[1], sizeof(float));-- kparams->scale = scale;- kparams->max_bias = max_bias;-- const uint32_t n_head = src0->ne[2];- const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));- kparams->n_head = n_head;- kparams->n_head_log2 = n_head_log2;-- if (max_bias > 0.0f && n_head_log2 > 0) {- kparams->m0 = powf(2.0f, -(max_bias) / n_head_log2);- kparams->m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);- } else {- kparams->m0 = 1.0f;- kparams->m1 = 1.0f;- }-- kparams->use_src1 = (src1 != nullptr) ? 1 : 0;- kparams->use_f16 = (src1 != nullptr && src1->type == GGML_TYPE_F16) ? 1 : 0;-- const uint32_t ne00 = src0->ne[0];- const uint32_t ne10 = src1 ? src1->ne[0] : 1;-- struct htp_softmax_vtcm_layout layout;- htp_softmax_vtcm_layout_build(&layout, ne00, ne10, kparams->use_src1 != 0, kparams->use_f16 != 0, n_threads);-- kparams->n_threads = n_threads;- kparams->src0_nrows = src0_nrows;- kparams->src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;- kparams->vtcm_size = (uint32_t) layout.total_bytes;- kparams->vtcm_src0_size_per_thread = (uint32_t) layout.src0_bytes_per_thread;- kparams->vtcm_src1_size_per_thread = (uint32_t) layout.src1_bytes_per_thread;- kparams->vtcm_dst_size_per_thread = (uint32_t) layout.dst_bytes_per_thread;- kparams->src0_row_size_aligned = (uint32_t) layout.src0_spad_half_size;- kparams->src1_row_size_aligned = (uint32_t) layout.src1_spad_half_size;- kparams->dst_row_size_aligned = (uint32_t) layout.dst_spad_half_size;- kparams->src0_spad_half_size = (uint32_t) layout.src0_spad_half_size;- kparams->src1_spad_half_size = (uint32_t) layout.src1_spad_half_size;- kparams->dst_spad_half_size = (uint32_t) layout.dst_spad_half_size;- if (!kparams->use_src1) {- kparams->kernel_id = HTP_SOFTMAX_KERNEL_NOMASK;- } else if (kparams->use_f16) {- kparams->kernel_id = HTP_SOFTMAX_KERNEL_MASK_F16;- } else {- kparams->kernel_id = HTP_SOFTMAX_KERNEL_MASK_F32;- }-- if (src0->ne[1] > 0) kparams->div_ne01 = init_fastdiv_values(src0->ne[1]);- if (src0->ne[2] > 0) kparams->div_ne02 = init_fastdiv_values(src0->ne[2]);- const uint32_t ne12 = src1 ? src1->ne[2] : 1;- const uint32_t ne13 = src1 ? src1->ne[3] : 1;- if (ne12 > 0) kparams->div_ne12 = init_fastdiv_values(ne12);- if (ne13 > 0) kparams->div_ne13 = init_fastdiv_values(ne13);-}--static void ggml_hexagon_precompute_rope_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_rope_kernel_params * kparams-) {- memset(kparams, 0, sizeof(*kparams));-- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * src2 = op->src[2];- const struct ggml_tensor * dst = op;-- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows);- const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0;-- struct htp_rope_vtcm_layout layout;- htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads, n_freq_factors);-- kparams->n_threads = n_threads;- kparams->src0_nrows = src0_nrows;- kparams->src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;- kparams->vtcm_size = (uint32_t) layout.total_bytes;- kparams->spad_per_thread = (uint32_t) layout.bytes_per_thread;- kparams->theta_cache_offset = (uint32_t) layout.theta_cache_size_aligned;- kparams->src0_row_size_aligned = (uint32_t) layout.src0_row_size_aligned;- kparams->freq_factors_offset = (uint32_t) (layout.bytes_per_thread * n_threads);- kparams->freq_factors_size = (uint32_t) layout.freq_factors_size_aligned;-- if (src0_nrows > 0) {- kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]);- kparams->div_ne1 = init_fastdiv_values(dst->ne[1]);- }-}--static void ggml_hexagon_precompute_ssm_conv_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * src1,- const struct ggml_tensor * dst,- struct htp_ssm_conv_kernel_params * kparams-) {- memset(kparams, 0, sizeof(*kparams));-- const uint32_t d_conv = (uint32_t) src1->ne[0];- const uint32_t d_inner = (uint32_t) src0->ne[1];- const uint32_t n_t = (uint32_t) dst->ne[1];- const uint32_t n_s = (uint32_t) dst->ne[2];- const uint32_t ncs = (uint32_t) src0->ne[0];-- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (d_inner + 31) / 32);-- kparams->n_threads = n_threads;- kparams->d_conv = d_conv;- kparams->d_inner = d_inner;- kparams->n_t = n_t;- kparams->n_s = n_s;-- const uint32_t raw_rpt = (d_inner + n_threads - 1) / n_threads;- const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, 32);-- const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128) + 128;- const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128);- const uint32_t vtcm_src1_per_thread = src1_raw_bytes + src1_T_bytes;-- uint32_t vtcm_src0_per_thread = 0;- uint32_t vtcm_dst_per_thread = 0;-- if (n_t == 1) {- kparams->d_inner_tile = d_inner_per_thread;-- const uint32_t src0_tile_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128);- const uint32_t src0_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128);- vtcm_src0_per_thread = 2 * src0_tile_raw_bytes + src0_T_bytes;-- const uint32_t dst_tile_bytes = hex_round_up(d_inner_per_thread * sizeof(float), 128);- vtcm_dst_per_thread = 2 * dst_tile_bytes;- } else {- const size_t vtcm_budget = (sess->vtcm_size > 0 ? sess->vtcm_size / n_threads : (1024 * 1024));-- // the kernel double-buffers the raw src0 tile and the dst tile, and transposes- // one 32-channel block at a time- const uint32_t src0_block_T = hex_round_up(ncs * 32 * sizeof(float), 128);- const size_t fixed_bytes = vtcm_src1_per_thread + src0_block_T;- const size_t avail_for_tiles = vtcm_budget > fixed_bytes ? vtcm_budget - fixed_bytes : (128 * 1024);-- uint32_t target_max_tile = hex_round_up((d_inner_per_thread + 3) / 4, 32);- target_max_tile = (std::max)(target_max_tile, 32u);- target_max_tile = (std::min)(target_max_tile, 128u);-- uint32_t d_inner_tile = (uint32_t)(avail_for_tiles / (2 * (ncs + n_t) * sizeof(float)));- d_inner_tile = (d_inner_tile / 32) * 32;- if (d_inner_tile == 0) {- d_inner_tile = 32;- }- if (d_inner_tile > target_max_tile) {- d_inner_tile = target_max_tile;- }- if (d_inner_tile > d_inner_per_thread) {- d_inner_tile = d_inner_per_thread;- }- kparams->d_inner_tile = d_inner_tile;-- const uint32_t src0_tile_raw = hex_round_up(d_inner_tile * ncs * sizeof(float), 128);- vtcm_src0_per_thread = 2 * src0_tile_raw + src0_block_T;-- vtcm_dst_per_thread = 2 * hex_round_up(d_inner_tile * n_t * sizeof(float), 128);- }-- kparams->vtcm_src0_size_per_thread = vtcm_src0_per_thread;- kparams->vtcm_src1_size_per_thread = vtcm_src1_per_thread;- kparams->vtcm_dst_size_per_thread = vtcm_dst_per_thread;-- kparams->vtcm_src0_size = vtcm_src0_per_thread * n_threads;- kparams->vtcm_src1_size = vtcm_src1_per_thread * n_threads;- kparams->vtcm_dst_size = vtcm_dst_per_thread * n_threads;- kparams->vtcm_size = kparams->vtcm_src0_size + kparams->vtcm_src1_size + kparams->vtcm_dst_size;-}--static void ggml_hexagon_precompute_gated_delta_net_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_gdn_kernel_params * kparams-) {- memset(kparams, 0, sizeof(*kparams));-- const struct ggml_tensor * q = op->src[0];- const struct ggml_tensor * k = op->src[1];- const struct ggml_tensor * v = op->src[2];- const struct ggml_tensor * g = op->src[3];- const struct ggml_tensor * state = op->src[5];-- const uint32_t S_v = (uint32_t) v->ne[0];- const uint32_t H = (uint32_t) v->ne[1];- const uint32_t n_tokens = (uint32_t) v->ne[2];- const uint32_t n_seqs = (uint32_t) v->ne[3];- const uint32_t K = (uint32_t) ggml_get_op_params_i32(op, 0);-- const uint32_t rq3 = (uint32_t) (n_seqs / q->ne[3]);- const uint32_t rk3 = (uint32_t) (n_seqs / k->ne[3]);- const uint32_t total_rows = H * n_seqs;- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, total_rows);-- const bool can_use_hmx = (opt_gdn_select >= 2) &&- (sess->n_hmx > 0) &&- (S_v % 64 == 0) &&- (n_tokens >= HTP_GDN_MIN_TOKENS) &&- (g->ne[0] == 1) &&- (K == 1);-- struct htp_gdn_hmx_vtcm_layout hmx_layout;- struct htp_gdn_vtcm_layout hvx_layout;- uint32_t n_heads_batch = 1;-- if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout, S_v, HTP_GDN_CHUNK_SIZE, total_rows, sess->vtcm_size, n_threads, true, &n_heads_batch)) {- kparams->kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED;- kparams->pipeline = hmx_layout.pipeline ? 1 : 0;- kparams->chunk_size = HTP_GDN_CHUNK_SIZE;- kparams->n_chunks = (n_tokens + HTP_GDN_CHUNK_SIZE - 1) / HTP_GDN_CHUNK_SIZE;- kparams->n_heads_batch = (uint16_t) n_heads_batch;- kparams->vtcm_size = (uint32_t) hmx_layout.total_bytes;- kparams->state_aligned = (uint32_t) hmx_layout.state_f32_bytes;- kparams->vtcm_per_thread = (uint32_t) (hmx_layout.total_bytes / (n_threads > 0 ? n_threads : 1));- } else {- htp_gdn_vtcm_layout_build(&hvx_layout, S_v, n_threads);- kparams->kernel_type = HTP_GDN_KERNEL_HVX_RECURRENT;- kparams->pipeline = 0;- kparams->n_heads_batch = 1;- kparams->state_aligned = (uint32_t) hvx_layout.state_aligned;- kparams->vtcm_per_thread = (uint32_t) hvx_layout.bytes_per_thread;- kparams->vtcm_size = (uint32_t) hvx_layout.total_bytes;- }-- kparams->n_threads = n_threads;- kparams->S_v = S_v;- kparams->H = H;- kparams->n_tokens = n_tokens;- kparams->n_seqs = n_seqs;- kparams->K = K;- kparams->total_rows = total_rows;- kparams->rows_per_thread = (total_rows + kparams->n_threads - 1) / kparams->n_threads;- kparams->kda = (g->ne[0] == S_v) ? 1 : 0;- kparams->state_seq_stride = (uint32_t) (state->nb[3] / sizeof(float));- kparams->state_size_per_snap = S_v * S_v * H * n_seqs;- kparams->scale = 1.0f / sqrtf((float) S_v);-- if (H > 0) kparams->div_H = init_fastdiv_values(H);- if (q->ne[1] > 0) kparams->div_q1 = init_fastdiv_values((uint32_t) q->ne[1]);- if (k->ne[1] > 0) kparams->div_k1 = init_fastdiv_values((uint32_t) k->ne[1]);- if (rq3 > 0) kparams->div_rq3 = init_fastdiv_values(rq3);- if (rk3 > 0) kparams->div_rk3 = init_fastdiv_values(rk3);- if (kparams->n_threads > 0) kparams->div_n_threads = init_fastdiv_values(kparams->n_threads);-}--static void ggml_hexagon_precompute_sort_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- bool is_top_k,- struct htp_sort_kernel_params * kparams-) {- memset(kparams, 0, sizeof(*kparams));-- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * dst = op;-- const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];- const uint32_t ne00 = src0->ne[0];- const uint32_t k = dst->ne[0];-- int32_t order = GGML_SORT_ORDER_DESC;- if (!is_top_k) {- order = ((const int32_t *) op->op_params)[0];- }-- const uint32_t n_threads_max = sess->n_threads > 0 ? sess->n_threads : 4;- const size_t vtcm_budget = sess->vtcm_size > 0 ? sess->vtcm_size : (8 * 1024 * 1024);-- struct htp_sort_vtcm_layout layout;- bool ok = htp_sort_solve_layout(&layout, ne00, total_rows, k, n_threads_max, vtcm_budget, is_top_k);- GGML_ASSERT(ok);-- kparams->n_threads = (int32_t) layout.n_threads;- kparams->total_rows = (int32_t) total_rows;- kparams->row_start = 0;- kparams->row_end = (int32_t) total_rows;- kparams->ne00 = (int32_t) ne00;- kparams->k = (int32_t) k;- kparams->order = order;- kparams->is_top_k = is_top_k ? 1 : 0;- kparams->use_dma = 1;- kparams->chunk_elems = (int32_t) layout.chunk_elems;- kparams->n_chunks = (int32_t) layout.n_chunks;- kparams->vtcm_size = (int32_t) layout.total_bytes;- kparams->phase1_slot_size = (int32_t) layout.phase1_slot_size;- kparams->merge_values_off = (int32_t) layout.merge_values_off;- kparams->merge_indices_off = (int32_t) layout.merge_indices_off;- kparams->merge_elems = (int32_t) layout.merge_elems;- kparams->n_slots = (int32_t) layout.n_slots;-}--static void ggml_hexagon_pool_interior_range(- uint32_t src_x, uint32_t dst_x, uint32_t kernel_x, uint32_t stride_x, int32_t pad_x,- uint32_t * ox_lo, uint32_t * ox_hi) {- const uint32_t lo_raw = ((uint32_t) pad_x + stride_x - 1) / stride_x;- *ox_lo = (lo_raw < dst_x) ? lo_raw : dst_x;-- const int32_t numer_hi = (int32_t) src_x - (int32_t) kernel_x + pad_x;- if (numer_hi < 0) {- *ox_hi = 0;- } else {- const uint32_t hi_raw = (uint32_t) numer_hi / stride_x + 1;- *ox_hi = (hi_raw < dst_x) ? hi_raw : dst_x;- }- if (*ox_hi < *ox_lo) {- *ox_hi = *ox_lo;- }-}--static bool ggml_hexagon_pool_shape_fits(- const struct ggml_tensor * src0,- const struct ggml_tensor * dst-) {- return ggml_nbytes(src0) <= UINT32_MAX - 256 && ggml_nbytes(dst) <= UINT32_MAX - 256;-}--static void ggml_hexagon_precompute_pool_2d_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0,- const struct ggml_tensor * dst,- struct htp_pool_2d_kernel_params * kparams,- bool is_pool_1d-) {- memset(kparams, 0, sizeof(*kparams));-- const uint32_t src_x = (uint32_t) src0->ne[0];- const uint32_t src_y = is_pool_1d ? 1 : (uint32_t) src0->ne[1];- const uint32_t dst_x = (uint32_t) dst->ne[0];- const uint32_t dst_y = is_pool_1d ? 1 : (uint32_t) dst->ne[1];- const uint32_t src_plane_bytes = src_x * src_y * sizeof(float);- const uint32_t dst_plane_bytes = dst_x * dst_y * sizeof(float);- const uint32_t planes = (uint32_t) (is_pool_1d- ? (uint64_t) src0->ne[1] * (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3]- : (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3]);- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, planes > 0 ? planes : 1);-- struct htp_pool_vtcm_layout layout;- const bool ok = htp_pool_solve_layout(&layout, src_x, src_y, dst_x, dst_y, n_threads, sess->vtcm_size);- GGML_ASSERT(ok);-- kparams->src_x = src_x;- kparams->src_y = src_y;- kparams->dst_x = dst_x;- kparams->dst_y = dst_y;- kparams->kernel_x = (uint32_t) ggml_get_op_params_i32(dst, 1);- kparams->kernel_y = is_pool_1d ? 1 : (uint32_t) ggml_get_op_params_i32(dst, 2);- kparams->stride_x = (uint32_t) ggml_get_op_params_i32(dst, is_pool_1d ? 2 : 3);- kparams->stride_y = is_pool_1d ? 1 : (uint32_t) ggml_get_op_params_i32(dst, 4);- kparams->pad_x = ggml_get_op_params_i32(dst, is_pool_1d ? 3 : 5);- kparams->pad_y = is_pool_1d ? 0 : ggml_get_op_params_i32(dst, 6);- kparams->src_plane_bytes = src_plane_bytes;- kparams->dst_plane_bytes = dst_plane_bytes;- kparams->src_plane_bytes_aligned = (uint32_t) layout.src_spad_half_size;- kparams->dst_plane_bytes_aligned = (uint32_t) layout.dst_spad_half_size;- kparams->n_threads = n_threads;- kparams->planes = planes;- kparams->pool_op = (uint32_t) ggml_get_op_params_i32(dst, 0);- // Fast HVX path requires exact tiling and a supported kernel width.- kparams->fast_path = (kparams->pad_x == 0 && kparams->pad_y == 0 &&- kparams->stride_x == kparams->kernel_x && kparams->stride_y == kparams->kernel_y &&- (kparams->kernel_x == 1 || kparams->kernel_x == 2)) ? 1 : 0;- kparams->global_path = (kparams->pad_x == 0 && kparams->pad_y == 0 &&- kparams->stride_x == kparams->kernel_x &&- kparams->stride_y == kparams->kernel_y &&- kparams->kernel_x == kparams->src_x &&- kparams->kernel_y == kparams->src_y &&- kparams->dst_x == 1 && kparams->dst_y == 1) ? 1 : 0;- kparams->block_path = (kparams->pad_x == 0 && kparams->pad_y == 0 &&- kparams->kernel_y == 1 && kparams->stride_y == 1 &&- kparams->stride_x == kparams->kernel_x &&- kparams->kernel_x >= 32) ? 1 : 0;- kparams->avg_divide_count = (is_pool_1d && kparams->pool_op == GGML_OP_POOL_AVG) ? 1 : 0;- ggml_hexagon_pool_interior_range(kparams->src_x, kparams->dst_x, kparams->kernel_x,- kparams->stride_x, kparams->pad_x,- &kparams->ox_lo, &kparams->ox_hi);-- const bool narrow_ok = (uint64_t) kparams->dst_x * kparams->kernel_x <= 32;- kparams->narrow_path = (kparams->fast_path && narrow_ok) ? 1 : 0;- kparams->inv_kernel_area = 1.0f / (float) (kparams->kernel_x * kparams->kernel_y);-}--static bool ggml_hexagon_precompute_concat_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_concat_kernel_params * kparams-) {- memset(kparams, 0, sizeof(*kparams));- kparams->kernel_type = HTP_CONCAT_KERNEL_UNSUPPORTED;-- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * src1 = op->src[1];- const struct ggml_tensor * dst = op;-- if (!src0 || !src1 || !dst) {- return false;- }-- int dim = ((const int32_t *) op->op_params)[0];- if (dim < 0 || dim >= GGML_MAX_DIMS) {- return false;- }- kparams->dim = dim;-- if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_I32) {- return false;- }- if (src0->type != dst->type || src1->type != dst->type) {- return false;- }-- const uint32_t type_size = ggml_type_size(dst->type);-- for (int d = 0; d < GGML_MAX_DIMS; d++) {- const int64_t ne_d = (d == dim) ? src0->ne[d] + src1->ne[d] : src0->ne[d];- if (dst->ne[d] != ne_d || (d != dim && src1->ne[d] != dst->ne[d])) {- return false;- }- }-- const uint32_t max_gather_ne0 = 128 / type_size;- const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);- const bool b_dense = (src1->nb[0] == type_size);- const bool gather_strides_ok =- src0->nb[1] >= (size_t) src0->ne[0] * type_size &&- (b_dense ? src1->nb[1] >= (size_t) src1->ne[0] * type_size- : src1->nb[0] >= (size_t) src1->ne[1] * type_size);-- if (dim == 0 &&- (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16) &&- dst->ne[0] <= max_gather_ne0 &&- dst->nb[0] == type_size && dst->nb[1] == (size_t) dst->ne[0] * type_size &&- src0->nb[0] == type_size &&- (src1->nb[0] == type_size || src1->nb[1] == type_size) &&- gather_strides_ok &&- htp_gather_rows_tensor_outer_contiguous(dst) &&- htp_gather_rows_tensor_outer_contiguous(src0) &&- htp_gather_rows_tensor_outer_contiguous(src1)) {-- if (htp_gather_rows_solve_layout(&kparams->u.gather,- (uint32_t) src0->ne[0], (uint32_t) src0->nb[1],- (uint32_t) src1->ne[0], (uint32_t) src1->nb[1], b_dense,- type_size, total_rows,- sess->n_threads, sess->mdev.count, sess->vtcm_size)) {- kparams->kernel_type = HTP_CONCAT_KERNEL_GATHER_ROWS;- kparams->n_threads = (uint8_t) sess->n_threads;- kparams->vtcm_size = kparams->u.gather.slice * sess->n_threads;- return true;- }- }-- const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size);-- if (dma_strides_ok) {- kparams->kernel_type = HTP_CONCAT_KERNEL_REGULAR;- kparams->n_threads = 1;- return true;- }-- const bool is_src1_transposed = (src1->nb[0] > src1->nb[1]);- const bool is_src0_transposed = (src0->nb[0] > src0->nb[1]);- const bool transposed_rows_ok = (src0->nb[0] == type_size && src1->nb[1] == type_size && dst->nb[0] == type_size);-- if (dim == 0 && is_src1_transposed && !is_src0_transposed && transposed_rows_ok &&- (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16)) {-- const uint32_t n_threads = sess->n_threads > 0 ? (uint32_t) sess->n_threads : 8;- struct htp_concat_transposed_vtcm_layout layout;- htp_concat_transposed_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src1->ne[0], type_size, n_threads);-- if (sess->vtcm_size > 0 && layout.total_bytes > sess->vtcm_size) {- return false;- }-- kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED;- kparams->n_threads = n_threads;- kparams->vtcm_size = layout.total_bytes;- kparams->u.transposed.spad0_size_per_thread = layout.src0_spad_size_per_thread;- kparams->u.transposed.spad1_size_per_thread = layout.src1_spad_size_per_thread;- return true;- }-- return false;-}--static bool ggml_hexagon_precompute_cpy_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_copy_kernel_params * kparams-) {- memset(kparams, 0, sizeof(*kparams));- kparams->kernel_type = HTP_COPY_KERNEL_UNSUPPORTED;-- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * dst = op;-- if (!src0 || !dst) {- return false;- }-- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_I32) {- return false;- }- if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_I32) {- return false;- }-- const int64_t nelem_src = ggml_nelements(src0);- const int64_t nelem_dst = ggml_nelements(dst);- if (nelem_src != nelem_dst || nelem_src < 0) {- return false;- }-- const uint32_t src_type_size = ggml_type_size(src0->type);- const uint32_t dst_type_size = ggml_type_size(dst->type);-- kparams->src0_type_size = (uint8_t) src_type_size;- kparams->dst_type_size = (uint8_t) dst_type_size;- kparams->total_elems = (uint32_t) nelem_src;-- if (nelem_src == 0) {- kparams->kernel_type = HTP_COPY_KERNEL_1D_CONTIG;- return true;- }-- if (nelem_src == 1) {- if (src0->type == dst->type) {- kparams->kernel_type = HTP_COPY_KERNEL_SCALAR;- return true;- }- if ((src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) ||- (src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32) ||- (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) ||- (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32)) {- kparams->kernel_type = HTP_COPY_KERNEL_SCALAR;- return true;- }- return false;- }-- const bool sametype = (src0->type == dst->type);-- bool same_extents = true;- for (int d = 0; d < GGML_MAX_DIMS; d++) {- if (src0->ne[d] != dst->ne[d]) {- same_extents = false;- break;- }- }-- const bool transposed = (src0->nb[0] > src0->nb[1]) || (dst->nb[0] > dst->nb[1]) ||- (src0->nb[0] != src_type_size) || (dst->nb[0] != dst_type_size) ||- (src0->nb[1] < (size_t) src0->ne[0] * src_type_size) || (dst->nb[1] < (size_t) dst->ne[0] * dst_type_size);- const bool sameshape = same_extents && !transposed;-- const bool src_is_contiguous = ggml_is_contiguous(src0);- const bool dst_is_contiguous = ggml_is_contiguous(dst);-- if (sametype) {- if (src_is_contiguous && dst_is_contiguous) {- kparams->kernel_type = HTP_COPY_KERNEL_1D_CONTIG;- return true;- }-- const uint32_t max_gather_ne0 = 128 / src_type_size;- const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);- if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) &&- dst_is_contiguous && !src_is_contiguous &&- src0->nb[0] == src_type_size && src0->ne[0] <= max_gather_ne0 &&- src0->nb[1] >= (size_t) src0->ne[0] * src_type_size &&- total_rows > 1 &&- htp_gather_rows_tensor_outer_contiguous(src0)) {-- if (htp_gather_rows_solve_layout(&kparams->u.gather,- (uint32_t) src0->ne[0], (uint32_t) src0->nb[1],- 0, 0, false,- src_type_size, total_rows,- sess->n_threads, sess->mdev.count, sess->vtcm_size)) {- kparams->kernel_type = HTP_COPY_KERNEL_GATHER_ROWS;- kparams->n_threads = (uint8_t) sess->n_threads;- kparams->vtcm_size = kparams->u.gather.slice * sess->n_threads;- return true;- }- }-- if (sameshape) {- kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE;- kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);- return true;- }-- kparams->kernel_type = HTP_COPY_KERNEL_RESHAPE;- kparams->n_threads = sess->n_threads > 0 ? (uint8_t) sess->n_threads : 4;- kparams->u.reshape.div_ne0 = init_fastdiv_values((uint32_t) dst->ne[0]);- kparams->u.reshape.div_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[1] * dst->ne[0]));- kparams->u.reshape.div_ne2_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[2] * dst->ne[1] * dst->ne[0]));- kparams->u.reshape.div_ne00 = init_fastdiv_values((uint32_t) src0->ne[0]);- kparams->u.reshape.div_ne01_ne00 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[0]));- kparams->u.reshape.div_ne02_ne01_ne00 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1] * src0->ne[0]));- return true;- }-- if (!sameshape) {- return false;- }-- const bool valid_conversion = (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) ||- (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) ||- (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) ||- (src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32);- if (!valid_conversion) {- return false;- }-- const uint32_t n_threads = sess->n_threads > 0 ? (uint32_t) sess->n_threads : 4;- struct htp_copy_convert_vtcm_layout layout;- htp_copy_convert_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1], src_type_size, dst_type_size, n_threads);-- if (sess->vtcm_size > 0 && layout.total_bytes > sess->vtcm_size) {- return false;- }-- kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_CONVERT;- kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);- kparams->n_threads = (uint8_t) n_threads;- kparams->vtcm_size = layout.total_bytes;- kparams->u.convert.src0_buf_size = layout.src0_buf_size;- kparams->u.convert.dst_buf_size = layout.dst_buf_size;- kparams->u.convert.spad0_size_per_thread = layout.spad0_size_per_thread;- kparams->u.convert.spad1_size_per_thread = layout.spad1_size_per_thread;- kparams->u.convert.src0_row_stride = layout.src0_row_stride;- kparams->u.convert.dst_row_stride = layout.dst_row_stride;- kparams->u.convert.blk_rows = layout.blk_rows;- kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);- kparams->u.convert.div_ne02_ne01 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1]));-- return true;-}--static void ggml_hexagon_precompute_fused_mmnx_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0, // W0- const struct ggml_tensor * act, // x- int32_t n_weights,- struct htp_mm_kernel_params * kparams-) {- memset(kparams, 0, sizeof(*kparams));- kparams->n_threads = sess->n_threads;-- const int ne00 = src0->ne[0];- const int ne01 = src0->ne[1];- const int ne02 = src0->ne[2];- const int ne03 = src0->ne[3];-- const int ne10 = act->ne[0];- const int ne11 = act->ne[1];- const int ne12 = act->ne[2];- const int ne13 = act->ne[3];-- const bool is_tiled = ggml_hexagon_tensor_is_tiled(src0);- const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;- const int ne00_padded = is_tiled ? hex_round_up(ne00, 32) : ne00;- const int ne11_padded = hex_round_up(ne11, 32);- const int ne01_tiled = hex_round_up(ne01, 32);-- const size_t vtcm_budget = sess->vtcm_size;- const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1);-- bool hmx_enabled = (sess->n_hmx > 0) && (opt_mm_select >= 2);- if (hmx_enabled && ggml_hexagon_matmul_is_hmx_eligible(src0, act, nullptr, false, is_batched)) {- if (ggml_hexagon_precompute_hmx_mm_params(sess, src0, act, nullptr, wtype, ne00_padded, ne01_tiled, ne02, ne11, ne12, ne11_padded, false, is_batched, 0, vtcm_budget, kparams)) {- kparams->n_weights = n_weights;- goto finalize;- }- }-- if (!is_tiled || ne10 % 32 != 0) {- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;- return;- }-- {- const int act_nrows = ne11 * ne12 * ne13;- const size_t act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);- const size_t src0_row_size = src0->nb[1];-- const uint32_t max_prefetch = (act_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;- uint32_t best_n_prefetch = 2;- for (uint32_t d = max_prefetch; d >= 2; d /= 2) {- struct htp_mm_hvx_vtcm_layout L;- htp_mm_hvx_vtcm_layout_build(- &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, sess->n_threads,- 0, src0_row_size, act_row_size, 0, d, false, true- );- if (L.total_bytes <= sess->vtcm_size) {- best_n_prefetch = d;- break;- }- }-- struct htp_mm_hvx_vtcm_layout L;- bool try_tiled = (opt_mm_select >= 1);-- // Test tiled first- htp_mm_hvx_vtcm_layout_build(- &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, sess->n_threads,- 0, src0_row_size, act_row_size, 0, best_n_prefetch, false, true- );-- if (try_tiled && L.total_bytes <= sess->vtcm_size) {- kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW;- kparams->tile_size = htp_tensor_get_tile_size(wtype);- kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);- kparams->act_row_size = act_row_size;- kparams->vtcm_src0_size = L.src0_bytes;- kparams->vtcm_act_size = L.act_bytes;- kparams->vtcm_bias_size = 0;- kparams->vtcm_dst_size = L.dst_bytes;- kparams->vtcm_size = L.total_bytes;- kparams->n_prefetch = best_n_prefetch;- kparams->n_weights = n_weights;- } else {- kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;- return;- }- }--finalize:- kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11);- kparams->div_ne1 = init_fastdiv_values(ne11);- kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1);- kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1);- kparams->div_ne12 = init_fastdiv_values(ne12);-}--static void ggml_hexagon_precompute_fused_mmidnx_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * src0, // W0- const struct ggml_tensor * act, // x- const struct ggml_tensor * dst, // dst0- int32_t n_weights,- struct htp_mm_kernel_params * kparams-) {- ggml_hexagon_precompute_matmul_params_impl(sess, src0, act, dst, 0, 0, kparams);- kparams->n_weights = n_weights;-}--static bool ggml_hexagon_op_mul_mat_use_fwht(const struct ggml_tensor * dst) {- if (dst->op != GGML_OP_MUL_MAT || ggml_get_op_params_i32(dst, 1) != GGML_HINT_SRC0_IS_HADAMARD) {- return false;- }-- const struct ggml_tensor * src0 = dst->src[0];- const struct ggml_tensor * src1 = dst->src[1];-- const int64_t n = src1->ne[0];- if (n < 32 || (n & (n - 1)) != 0) {- return false;- }-- return src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 &&- (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) &&- ggml_is_contiguous_rows(src1) && ggml_is_contiguous(dst) &&- ggml_are_same_shape(src1, dst);-}--static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) {- const struct ggml_tensor * src0 = dst->src[0];- const struct ggml_tensor * src1 = dst->src[1];-- if (ggml_get_op_params_i32(dst, 1) == GGML_HINT_SRC0_IS_HADAMARD) {- if (!ggml_hexagon_op_mul_mat_use_fwht(dst)) {- return false;- }- struct htp_unary_kernel_params ukparams;- ggml_hexagon_precompute_unary_params(sess, HTP_OP_HADAMARD, src1, nullptr, dst, &ukparams);- return ukparams.vtcm_row_per_thread > 0 && (size_t) ukparams.vtcm_size <= sess->vtcm_size;- }-- if (dst->type != GGML_TYPE_F32) {- return false;- }-- if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) {- return false;- }-- switch (src0->type) {- case GGML_TYPE_Q1_0:- case GGML_TYPE_Q2_0:- case GGML_TYPE_Q4_0:- case GGML_TYPE_Q4_1:- case GGML_TYPE_Q8_0:- case GGML_TYPE_IQ4_NL:- case GGML_TYPE_MXFP4:- case GGML_TYPE_Q4_K:- case GGML_TYPE_Q5_K:- case GGML_TYPE_Q6_K:- case GGML_TYPE_Q3_K:- case GGML_TYPE_Q2_K: {- if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) {- return false;- }-- const int64_t qk = ggml_blck_size(src0->type);- if (src0->ne[0] % qk) {- return false;- }-- if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) {- return false;- }- if (src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) {- return false;- }- break;- }-- case GGML_TYPE_F16:- if (src0->nb[1] < src0->nb[0]) {- return false;- }- if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) {- return false;- }- if (src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) {- return false;- }- break;-- case GGML_TYPE_F32:- if (src1->type != GGML_TYPE_F32) {- return false;- }- if (src0->nb[1] < src0->nb[0]) {- return false;- }- if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) {- return false;- }- if (src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) {- return false;- }- break;-- case GGML_TYPE_BF16:- if (src0->nb[1] < src0->nb[0]) {- return false;- }- if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) {- return false;- }- if (src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) {- return false;- }- break;-- default:- return false;- }-- struct htp_mm_kernel_params kparams;- ggml_hexagon_precompute_matmul_params(sess, src0, src1, dst, &kparams);- if (kparams.kernel_type == HTP_MM_KERNEL_UNSUPPORTED || (size_t) kparams.vtcm_size > sess->vtcm_size) {- HEX_VERBOSE("ggml-hex: %s supported MUL_MAT VTCM size needed (%d) > budget (%zu)\n", sess->c_name(), kparams.vtcm_size, sess->vtcm_size);- return false;- }-- return true;-}--static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * src1 = op->src[1];- const struct ggml_tensor * src2 = op->src[2];- const struct ggml_tensor * dst = op;-- if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32) {- return false;- }-- if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) {- return false;- }-- switch (src0->type) {- case GGML_TYPE_Q1_0:- case GGML_TYPE_Q2_0:- case GGML_TYPE_Q4_0:- case GGML_TYPE_Q4_1:- case GGML_TYPE_Q8_0:- case GGML_TYPE_IQ4_NL:- case GGML_TYPE_MXFP4:- case GGML_TYPE_Q4_K:- case GGML_TYPE_Q5_K:- case GGML_TYPE_Q6_K:- case GGML_TYPE_Q3_K:- case GGML_TYPE_Q2_K:- if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) {- return false;- }-- if (src0->ne[0] % ggml_blck_size(src0->type) != 0) {- return false;- }- break;-- case GGML_TYPE_F16:- case GGML_TYPE_F32:- case GGML_TYPE_BF16:- if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0)) {- return false;- }- if (src0->ne[0] % 32 != 0) {- return false;- }- break;-- default:- return false;- }-- struct htp_mm_kernel_params kparams;- ggml_hexagon_precompute_matmul_params(sess, src0, src1, dst, &kparams);- if (kparams.kernel_type == HTP_MM_KERNEL_UNSUPPORTED || (size_t) kparams.vtcm_size > sess->vtcm_size) {- HEX_VERBOSE("ggml-hex: %s supported MUL_MAT_ID VTCM size needed (%d) > budget (%zu)\n", sess->c_name(), kparams.vtcm_size, sess->vtcm_size);- return false;- }-- return true;-}--static bool ggml_hexagon_supported_binary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * src1 = op->src[1];- const struct ggml_tensor * dst = op;-- if (src0->type == GGML_TYPE_F32) {- if (src1->type != GGML_TYPE_F32) {- return false;- }- if (dst->type != GGML_TYPE_F32) {- return false;- }- }- else if (src0->type == GGML_TYPE_F16) {- if (src1->type != GGML_TYPE_F16) {- return false;- }- if (dst->type != GGML_TYPE_F16) {- return false;- }- }- else {- return false;- }-- if (ggml_is_permuted(src0) || ggml_is_permuted(dst)) {- return false;- }- if (!ggml_are_same_shape(src0, dst)) {- return false;- }- if (!ggml_can_repeat(src1, src0) || ggml_is_permuted(src1)) {- return false;- }-- struct htp_binary_kernel_params kparams;- return ggml_hexagon_precompute_binary_params(sess, op_remap_to_htp(op), src0, src1, dst, &kparams);-}--static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * src1 = op->src[1];- const struct ggml_tensor * src2 = op->src[2];- const struct ggml_tensor * dst = op;-- if (!src2) {- return false;- }- if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32) {- return false;- }- if (!ggml_are_same_shape(src0, dst)) {- return false;- }- if (src1->ne[0] != src0->ne[0] || src1->ne[2] != 1 || src1->ne[3] != 1) {- return false;- }- if (src2->ne[0] != src0->ne[1] || src2->ne[1] != src0->ne[2]) {- return false;- }- if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float) || src2->nb[0] != sizeof(int32_t)) {- return false;- }-- // REVISIT: add support for non-contiguous tensors- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {- return false;- }- if (src2->nb[1] < src2->ne[0] * sizeof(int32_t)) {- return false;- }-- struct htp_binary_kernel_params kparams;- return ggml_hexagon_precompute_binary_params(sess, HTP_OP_ADD_ID, src0, src1, dst, &kparams, src2);-}--static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * dst = op;-- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) {- return false;- }- if (dst->type != src0->type) {- return false;- }- if (!ggml_is_contiguous_rows(src0)) {- return false;- }-- // F16 device kernels only cover this explicit whitelist (must stay in sync with- // the is_f16 whitelist in execute_op_unary(), unary-ops.c).- if (src0->type == GGML_TYPE_F16) {- switch (op->op) {- case GGML_OP_NORM:- case GGML_OP_RMS_NORM:- case GGML_OP_L2_NORM:- case GGML_OP_SCALE:- case GGML_OP_CLAMP:- case GGML_OP_SQR:- case GGML_OP_SQRT:- case GGML_OP_LOG:- break;- case GGML_OP_UNARY:- if (ggml_get_unary_op(op) != GGML_UNARY_OP_ABS &&- ggml_get_unary_op(op) != GGML_UNARY_OP_STEP) {- return false;- }- break;- default:- return false;- }- }-- if (!ggml_are_same_shape(src0, dst)) {- return false;- }-- // dst must be contiguous; src0 may be non-contiguous- if (!ggml_is_contiguous(dst)) {- return false;- }-- return true;-- GGML_UNUSED(sess);-}--static bool ggml_hexagon_supported_sum(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * dst = op;-- if (src0->type != GGML_TYPE_F32) {- return false;- }- if (dst->type != GGML_TYPE_F32) {- return false;- }-- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {- return false;- }-- return true;-- GGML_UNUSED(sess);-}--static bool ggml_hexagon_supported_sum_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * dst = op;-- if (src0->type != GGML_TYPE_F32) {- return false;- }- if (dst->type != GGML_TYPE_F32) {- return false;- }-- // TODO: add support for non-contigiuos tensors- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {- return false;- }-- return true;-- GGML_UNUSED(sess);-}--static bool ggml_hexagon_supported_argmax(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * dst = op;-- if (src0->type != GGML_TYPE_F32) {- return false;- }- if (dst->type != GGML_TYPE_I32) {- return false;- }-- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {- return false;- }-- return true;-- GGML_UNUSED(sess);-}--static bool ggml_hexagon_supported_pool_2d(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const int32_t * params = op->op_params;-- if (params[0] != GGML_OP_POOL_AVG && params[0] != GGML_OP_POOL_MAX) {- return false;- }- if (src0->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) {- return false;- }- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(op)) {- return false;- }- if (!ggml_hexagon_pool_shape_fits(src0, op)) {- return false;- }-- const int32_t kernel_x = params[1];- const int32_t kernel_y = params[2];- const int32_t stride_x = params[3];- const int32_t stride_y = params[4];- const int32_t pad_x = params[5];- const int32_t pad_y = params[6];-- // Keep invalid parameters on CPU, avoiding unsafe unsigned values in HTP.- if (kernel_x <= 0 || kernel_y <= 0 || stride_x <= 0 || stride_y <= 0 ||- pad_x < 0 || pad_y < 0) {- return false;- }-- const uint32_t planes = (uint32_t) (src0->ne[2] * src0->ne[3]);- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, planes > 0 ? planes : 1);-- struct htp_pool_vtcm_layout layout;- return htp_pool_solve_layout(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1],- (uint32_t) op->ne[0], (uint32_t) op->ne[1],- n_threads, sess->vtcm_size);-}--static bool ggml_hexagon_supported_pool_1d(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const int32_t * params = op->op_params;-- if (params[0] != GGML_OP_POOL_AVG && params[0] != GGML_OP_POOL_MAX) {- return false;- }- if (src0->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) {- return false;- }- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(op)) {- return false;- }- if (!ggml_hexagon_pool_shape_fits(src0, op)) {- return false;- }-- const int32_t kernel = params[1];- const int32_t stride = params[2];- const int32_t pad = params[3];- if (kernel <= 0 || stride <= 0 || pad < 0) {- return false;- }-- const uint32_t planes = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, planes > 0 ? planes : 1);-- struct htp_pool_vtcm_layout layout;- return htp_pool_solve_layout(&layout, (uint32_t) src0->ne[0], 1, (uint32_t) op->ne[0], 1, n_threads, sess->vtcm_size);-}--static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * src1 = op->src[1];- const struct ggml_tensor * dst = op;-- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) {- return false;- }- if (dst->type != src0->type) {- return false;- }-- if (!ggml_is_contiguous_1(src0)) {- return false;- }- if (!ggml_is_contiguous(dst)) {- return false;- }-- if (src1) {- if (src1->type != src0->type) {- return false;- }- if (!ggml_are_same_shape(src0, src1)) {- return false;- }- if (!ggml_is_contiguous_1(src1)) {- return false;- }- }-- return true;-- GGML_UNUSED(sess);-}--static bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * src1 = op->src[1];- const struct ggml_tensor * src2 = op->src[2];- const struct ggml_tensor * dst = op;-- if (src2) {- return false; // FIXME: add support for sinks- }-- if (src0->type != GGML_TYPE_F32) {- return false;- }- if (dst->type != GGML_TYPE_F32) {- return false;- }-- if (src0->ne[2] > 512) {- return false;- }-- if (src1) {- if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) {- return false;- }- if (src0->ne[0] != src1->ne[0]) {- return false;- }- if (src1->ne[1] < src0->ne[1]) {- return false;- }- if (src0->ne[2] % src1->ne[2] != 0) {- return false;- }- if (src0->ne[3] % src1->ne[3] != 0) {- return false;- }- }-- if (src1) {- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {- return false;- }- } else {- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {- return false;- }- }-- // Reject non-HVX-aligned sizes when ne[0] > HVX_F32_LANES- // The HVX softmax implementation has issues with tail handling for larger non-aligned sizes- // Small sizes (ne[0] <= 32) work correctly with tail-only processing- const int64_t ne0 = src0->ne[0];- if (ne0 > 32 && (ne0 & (32 - 1)) != 0) {- return false;- }-- // HVX vector size constraints for softmax- #define SOFTMAX_MAX_ROW_SIZE 131072 // 128K elements max for numerical precision-- // Reject very large row sizes to avoid numerical precision issues- // Softmax accumulation over many elements can lead to precision loss- if (ne0 > SOFTMAX_MAX_ROW_SIZE) {- return false;- }-- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows);- struct htp_softmax_vtcm_layout layout;- htp_softmax_vtcm_layout_build(&layout, src0->ne[0], src1 ? src1->ne[0] : 1, src1 != nullptr, src1 && src1->type == GGML_TYPE_F16, n_threads);- if (layout.total_bytes > sess->vtcm_size) {- return false;- }-- return true;-- GGML_UNUSED(sess);-}--static bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0]; // values- const struct ggml_tensor * src1 = op->src[1]; // indices- const struct ggml_tensor * dst = op->src[2] ? op->src[2] : op;-- if (dst->type == GGML_TYPE_Q8_0 && src0->ne[0] < 32) {- return false;- }-- if (src0->type != GGML_TYPE_F32) {- return false;- }-- if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) {- return false;- }-- if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_Q8_0) {- return false;- }-- return true;-- GGML_UNUSED(sess);-}--static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0]; // values- const struct ggml_tensor * src1 = op->src[1]; // indices- const struct ggml_tensor * dst = op;-- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16 && src0->type != GGML_TYPE_I32 &&- src0->type != GGML_TYPE_Q1_0 && src0->type != GGML_TYPE_Q2_0 && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 &&- src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K ) {- return false;- }-- if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous_rows(src1) || !ggml_is_contiguous_rows(dst)) {- return false;- }-- if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) {- return false;- }-- if (src0->type == dst->type) {- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->type != GGML_TYPE_F16) {- return false;- }- } else if (src0->type == GGML_TYPE_I32) {- return false;- } else if (dst->type != GGML_TYPE_F32) {- return false;- }-- // Empty recurrent-state gathers are skipped at execution; do not split the graph for them.- if (ggml_is_empty(op)) {- return true;- }-- struct htp_get_rows_kernel_params kparams;- ggml_hexagon_precompute_get_rows_params(sess, src0, src1, dst, &kparams);- if (kparams.n_threads == 0 || (size_t) kparams.vtcm_size > sess->vtcm_size) {- return false;- }-- return true;-- GGML_UNUSED(sess);-}--static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0]; // values- const struct ggml_tensor * dst = op; // indices-- if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_I32) {- return false;- }-- const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];- const uint32_t n_threads_max = sess->n_threads > 0 ? sess->n_threads : 4;- const size_t vtcm_budget = sess->vtcm_size > 0 ? sess->vtcm_size : (8 * 1024 * 1024);-- struct htp_sort_vtcm_layout layout;- if (!htp_sort_solve_layout(&layout, src0->ne[0], total_rows, dst->ne[0], n_threads_max, vtcm_budget, false)) {- return false;- }-- return true;-}--static bool ggml_hexagon_supported_top_k(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0]; // values- const struct ggml_tensor * dst = op; // indices-- if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_I32) {- return false;- }-- const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];- const uint32_t n_threads_max = sess->n_threads > 0 ? sess->n_threads : 4;- const size_t vtcm_budget = sess->vtcm_size > 0 ? sess->vtcm_size : (8 * 1024 * 1024);-- struct htp_sort_vtcm_layout layout;- if (!htp_sort_solve_layout(&layout, src0->ne[0], total_rows, dst->ne[0], n_threads_max, vtcm_budget, true)) {- return false;- }-- return true;+ op_batch->reset();}-static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * src1 = op->src[1];- const struct ggml_tensor * src2 = op->src[2];- const struct ggml_tensor * dst = op;-- if (!ggml_are_same_shape(src0, dst)) {- return false;- }-- if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32) {- return false;- }-- if (src0->ne[0] <= 0) {- return false;- }-- const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];- if (src0_nrows == 0) {- return false;- }-- const int32_t * op_params = &op->op_params[0];- const int n_dims = op_params[1];- const int mode = op_params[2];- const int n_offs = op_params[15];-- // llama probes weight placement with a dummy rope where every param is 0 (llama-model-loader.cpp).- // Rejecting it puts rope_freqs on the CPU, which then splits the graph at every full-attention layer.- if (n_dims < 0 || n_dims % 2 != 0) {- return false;- }+void ggml_hexagon_session::enqueue_op(const htp_opnode & node) {+ auto clone_tensor_buffer = [this](const ggml_tensor * t) {+ auto sbuf = this->mmap_tensor(t);+ if (!sbuf) return;+ if (sbuf->sess != this) {+ this->clone_buffer(sbuf);+ }+ for (auto & sub : this->mdev.sessions) {+ sub->clone_buffer(sbuf);+ }+ };- // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems)- if (n_offs < 0 || (n_offs % 32 != 0) || (n_offs + n_dims > src0->ne[0])) {- return false;+ for (auto t : node.get_inputs()) {+ clone_tensor_buffer(t);}-- float freq_base;- memcpy(&freq_base, op_params + 5, sizeof(float));- if (freq_base < 0.0f) {- return false;+ for (auto t : node.get_outputs()) {+ clone_tensor_buffer(t);}- if (mode != GGML_ROPE_TYPE_NORMAL &&- mode != GGML_ROPE_TYPE_NEOX &&- mode != GGML_ROPE_TYPE_MROPE &&- mode != GGML_ROPE_TYPE_VISION &&- mode != GGML_ROPE_TYPE_IMROPE) {- return false;+ const bool fits_tensors = op_batch->fit_tensors(node);+ if (opt_opfusion && fits_tensors && op_batch->try_fuse(node)) {+ return;}- const bool is_mrope = (mode & GGML_ROPE_TYPE_MROPE) != 0;-- // n_dims == ne0/2, so the rotation spans the full row- if (mode == GGML_ROPE_TYPE_VISION) {- if (n_dims != (int) (src0->ne[0] / 2) || n_offs != 0) {- return false;- }+ if (op_batch->full() || !fits_tensors) {+ flush_async();}- if (is_mrope) {- const int32_t * sections = op_params + 11;- if (sections[0] <= 0 && sections[1] <= 0 && sections[2] <= 0) {- return false;+ if (op_batch->empty()) {+ start_batch();+ if (!op_batch->fit_tensors(node)) {+ GGML_ABORT("ggml-hex: %s op does not fit into empty batch (vmem/tensor/buffer limit exceeded)\n", c_name());}}- const int64_t min_pos_len = (is_mrope || mode == GGML_ROPE_TYPE_VISION) ? src0->ne[2] * 4 : src0->ne[2];- if (src1->ne[0] < min_pos_len || !ggml_is_contiguous(src1)) {- return false;- }+ op_batch->add_op(node);+}- if (src2) {- if (src2->type != GGML_TYPE_F32 || !ggml_is_contiguous(src2)) {- return false;- }- if (src2->ne[0] < (n_dims / 2)) {- return false;- }- }+void ggml_hexagon_session::enqueue_mdev_group() {+ htp_opnode group_node(HTP_OP_MDEV_GROUP);- // src0/dst elements within a row must be contiguous (nb[0] == sizeof(float)).- // nb[1] may exceed ne[0]*sizeof(float) when the tensor is a strided view of a larger one- if (src0->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) {- return false;- }- if (src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) {- return false;- }+ ggml_tensor dummy_t = make_fence_tensor(this->mdev_fence_slot, this->mdev.count);+ dummy_t.op_params[0] = (int32_t) this->mdev.idx;- const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows);- const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0;+ ggml_tensor * node = group_node.add_dummy(dummy_t);+ node->src[0] = node;+ group_node.init(node);+ group_node.outputs.clear();- struct htp_rope_vtcm_layout layout;- htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads, n_freq_factors);- if (layout.total_bytes > sess->vtcm_size) {- return false;+ for (auto & sub : this->mdev.sessions) {+ sub->clone_buffer(this->fence_buf);}- return true;+ op_batch->add_op(group_node);}-static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * src1 = op->src[1];- const struct ggml_tensor * dst = op;-- // Only support FP32 for now- if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {- return false;- }-- // Check IO tensor shapes and dims- if (src0->ne[3] != 1 || src1->ne[2] != 1 || src1->ne[3] != 1 || dst->ne[3] != 1) {- return false; // src0 should be effectively 3D- }-- const int d_conv = src1->ne[0];- const int d_inner = src0->ne[1];- const int n_t = dst->ne[1];- const int n_s = dst->ne[2];+void ggml_hexagon_session::enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor, uint32_t fence_seq) {+ const bool with_fence = sync_tensor != nullptr;+ htp_opnode cpy_node(with_fence ? HTP_OP_CPY_FENCE : HTP_OP_CPY);- if (d_conv == 0 || d_conv > 32 || d_inner == 0) {- return false;- }- if (src0->ne[0] != d_conv - 1 + n_t || src0->ne[1] != d_inner || src0->ne[2] != n_s) {- return false;- }- if (src1->ne[0] != d_conv || src1->ne[1] != d_inner) {- return false;- }- if (dst->ne[0] != d_inner || dst->ne[1] != n_t || dst->ne[2] != n_s) {- return false;- }- if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) {- return false;- }- if (src0->nb[1] != src0->ne[0] * sizeof(float) || src1->nb[1] != src1->ne[0] * sizeof(float)) {- return false;+ ggml_tensor* node = cpy_node.add_dummy(*dst);+ node->op = GGML_OP_CPY;+ node->src[0] = const_cast<ggml_tensor *>(src);+ node->src[1] = with_fence ? cpy_node.add_dummy(*sync_tensor) : nullptr;+ if (with_fence) {+ node->op_params[0] = (int32_t) fence_seq;}- struct htp_ssm_conv_kernel_params kparams;- ggml_hexagon_precompute_ssm_conv_params(sess, src0, src1, dst, &kparams);- if ((size_t) kparams.vtcm_size > sess->vtcm_size) {- return false;+ cpy_node.init(node);+ const bool ok = htp_kparams_solve(this, node, cpy_node.kernel_params);+ if (ok && !with_fence && cpy_node.kparams<htp_copy_kernel_params>()->total_elems == 0) {+ return;}-- return true;+ this->enqueue_op(cpy_node);}-static bool im2col_3d_calc_output_size(uint32_t input, uint32_t kernel, int32_t stride, int32_t padding,- int32_t dilation, uint32_t * output) {- if (stride <= 0 || padding < 0 || dilation <= 0 || kernel == 0) {- return false;- }-- const int64_t numerator = (int64_t) input + 2LL * padding - (int64_t) dilation * (kernel - 1) - 1;- if (numerator < 0) {- return false;- }+void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq, bool wait) {+ htp_opnode sync_node(HTP_OP_FENCE);- const uint64_t size = (uint64_t) (numerator / stride) + 1;- if (size == 0 || size > UINT32_MAX) {- return false;- }+ ggml_tensor* node = sync_node.add_dummy(*sync_tensor);+ node->op = GGML_OP_NONE;+ node->src[0] = node;+ node->op_params[0] = (int32_t) fence_seq;+ node->op_params[1] = wait ? 0 : 1;- *output = (uint32_t) size;- return true;+ sync_node.init(node);+ this->enqueue_op(sync_node);}-static bool ggml_hexagon_precompute_im2col_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_im2col_kernel_params * kparams+void ggml_hexagon_session::enqueue_allreduce(+ const ggml_tensor * dst,+ const std::vector<const ggml_tensor *> & src_tensors,+ const std::vector<const ggml_tensor *> & sync_tensors,+ uint32_t rank,+ uint32_t n_ranks,+ uint32_t fence_seq_entry,+ uint32_t fence_seq_exit) {- memset(kparams, 0, sizeof(*kparams));- kparams->kernel_type = HTP_IM2COL_KERNEL_UNSUPPORTED;+ htp_opnode ar_node(HTP_OP_ALLREDUCE);- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * src1 = op->src[1];- const struct ggml_tensor * dst = op;+ ggml_tensor* node = ar_node.add_dummy(*dst);+ node->op = GGML_OP_NONE;+ node->op_params[0] = (int32_t) fence_seq_entry;+ node->op_params[1] = (int32_t) fence_seq_exit;- if (!src0 || !src1 || !dst) {- return false;- }+ ar_node.init(node);- if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) {- return false;+ ar_node.inputs.clear();+ for (size_t i = 0; i < src_tensors.size(); i++) {+ ar_node.inputs.push_back(src_tensors[i]);}-- if (src1->nb[0] != sizeof(float) || !ggml_is_contiguous(dst)) {- return false;+ for (size_t i = 0; i < sync_tensors.size(); i++) {+ ar_node.inputs.push_back(ar_node.add_dummy(*sync_tensors[i]));}- const int32_t * op_params = (const int32_t *) op->op_params;- const int32_t s0 = op_params[0];- const int32_t s1 = op_params[1];- const int32_t p0 = op_params[2];- const int32_t p1 = op_params[3];- const int32_t d0 = op_params[4];- const int32_t d1 = op_params[5];- const bool is_2D = (op_params[6] == 1);-- if (s0 <= 0 || d0 <= 0 || p0 < 0) {- return false;- }- if (is_2D && (s1 <= 0 || d1 <= 0 || p1 < 0)) {- return false;+ ar_node.outputs.clear();+ for (size_t i = 0; i < src_tensors.size(); i++) {+ ar_node.outputs.push_back(src_tensors[i]);}- const uint32_t N = is_2D ? (uint32_t) src1->ne[3] : (uint32_t) src1->ne[2];- const uint32_t IC = is_2D ? (uint32_t) src1->ne[2] : (uint32_t) src1->ne[1];- const uint32_t IH = is_2D ? (uint32_t) src1->ne[1] : 1;- const uint32_t IW = (uint32_t) src1->ne[0];- const uint32_t KH = is_2D ? (uint32_t) src0->ne[1] : 1;- const uint32_t KW = (uint32_t) src0->ne[0];- const uint32_t OH = is_2D ? (uint32_t) dst->ne[2] : 1;- const uint32_t OW = (uint32_t) dst->ne[1];-- if (KW == 0 || KH == 0 || IW == 0 || IH == 0 || IC == 0 || OW == 0 || OH == 0 || N == 0) {- return false;- }+ htp_kparams_solve_allreduce(+ this, dst, rank, n_ranks, false, false, /*is_shard_ok=*/ false,+ ar_node.kparams<htp_allreduce_kernel_params>());- const uint64_t patch_stride_64 = (uint64_t) IC * KH * KW;- const uint64_t total_rows_64 = (uint64_t) N * OH;- if (patch_stride_64 > UINT32_MAX || total_rows_64 > UINT32_MAX) {- return false;- }- const uint32_t patch_stride = (uint32_t) patch_stride_64;- const uint32_t total_rows = (uint32_t) total_rows_64;+ this->enqueue_op(ar_node);+}- if (dst->ne[0] != patch_stride || dst->ne[1] != OW) {- return false;- }- if (is_2D && dst->ne[2] != OH) {- return false;- }+ggml_hexagon_shared_buffer * ggml_hexagon_session::mmap_tensor(const ggml_tensor * t) {+ if (!t) return nullptr;- const uint32_t dst_elem = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);-- bool is_patchembed = ggml_is_contiguous(src1) && (s0 == (int32_t) KW && p0 == 0 && d0 == 1);- if (is_2D) {- is_patchembed = is_patchembed && (s1 == (int32_t) KH && p1 == 0 && d1 == 1);- }-- const uint32_t max_threads = (sess && sess->n_threads > 0) ? (uint32_t) sess->n_threads : 8;- const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads;- const uint32_t vtcm_size = (sess && sess->vtcm_size > 0) ? sess->vtcm_size : 0;-- if (is_patchembed) {- const uint64_t src_row_bytes_raw = (uint64_t) IC * KH * IW * sizeof(float);- const uint64_t dst_row_bytes_raw = (uint64_t) OW * patch_stride * dst_elem;- if (src_row_bytes_raw <= UINT32_MAX && dst_row_bytes_raw <= UINT32_MAX) {- const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);- const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);-- struct htp_im2col_vtcm_layout layout;- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);-- if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {- kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;- kparams->is_2d = is_2D ? 1 : 0;- kparams->is_3d = 0;- kparams->n_threads = (uint8_t) n_threads;- kparams->vtcm_size = layout.total_bytes;- kparams->off_src = layout.off_src;- kparams->off_dst = layout.off_dst;- kparams->src_bytes_per_thread = layout.src_bytes_per_thread;- kparams->dst_bytes_per_thread = layout.dst_bytes_per_thread;- kparams->src_row_bytes = src_row_bytes;- kparams->dst_row_bytes = dst_row_bytes;- kparams->owb = OW;- kparams->wb = IW;- kparams->nbpr = 1;- kparams->N = N;- kparams->IC = IC;- kparams->ID = 1;- kparams->IH = IH;- kparams->IW = IW;- kparams->KD = 1;- kparams->KH = KH;- kparams->KW = KW;- kparams->OD = 1;- kparams->OH = OH;- kparams->OW = OW;- kparams->patch_stride = patch_stride;- kparams->total_rows = total_rows;- return true;- }- }+ auto sbuf = static_cast<ggml_hexagon_shared_buffer *>(t->buffer->context);+ if (!sbuf->mapped) {+ const bool is_weight = ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS;+ const bool extended = opt_dma64 && is_weight;+ sbuf->mmap(extended);}+ return sbuf;+}- for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) {- const uint64_t Wb = (uint64_t) (owb - 1) * s0 + (uint64_t) (KW - 1) * d0 + 1;- if (Wb > UINT32_MAX) {- if (owb == 1) break;- continue;- }-- const uint64_t src_row_bytes_raw = (uint64_t) IC * KH * Wb * sizeof(float);- const uint64_t dst_row_bytes_raw = (uint64_t) owb * patch_stride * dst_elem;- if (src_row_bytes_raw > UINT32_MAX || dst_row_bytes_raw > UINT32_MAX) {- if (owb == 1) break;- continue;- }-- const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);- const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);-- struct htp_im2col_vtcm_layout layout;- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);-- if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {- kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA;- kparams->is_2d = is_2D ? 1 : 0;- kparams->is_3d = 0;- kparams->n_threads = (uint8_t) n_threads;- kparams->vtcm_size = layout.total_bytes;- kparams->off_src = layout.off_src;- kparams->off_dst = layout.off_dst;- kparams->src_bytes_per_thread = layout.src_bytes_per_thread;- kparams->dst_bytes_per_thread = layout.dst_bytes_per_thread;- kparams->src_row_bytes = src_row_bytes;- kparams->dst_row_bytes = dst_row_bytes;- kparams->owb = owb;- kparams->wb = (uint32_t) Wb;- kparams->nbpr = (OW + owb - 1) / owb;- kparams->N = N;- kparams->IC = IC;- kparams->ID = 1;- kparams->IH = IH;- kparams->IW = IW;- kparams->KD = 1;- kparams->KH = KH;- kparams->KW = KW;- kparams->OD = 1;- kparams->OH = OH;- kparams->OW = OW;- kparams->patch_stride = patch_stride;- kparams->total_rows = total_rows;- return true;- }- if (owb == 1) break;- }+bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf)+{+ GGML_ASSERT(sbuf && sbuf->mem);+ if (sbuf->sess == this) return true;- return false;-}+ auto mem = sbuf->mem;+ int fd = mem->fd;-static bool ggml_hexagon_precompute_im2col_3d_params(- const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op,- struct htp_im2col_kernel_params * kparams-) {- memset(kparams, 0, sizeof(*kparams));- kparams->kernel_type = HTP_IM2COL_KERNEL_UNSUPPORTED;+ GGML_ASSERT(fd >= 0);- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * src1 = op->src[1];- const struct ggml_tensor * dst = op;+ if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return true;- if (!src0 || !src1 || !dst) {- return false;- }+ GGML_ASSERT(sbuf->mapped);- if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) {- return false;- }+ HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(),+ sbuf->c_name(), sbuf->base(), sbuf->size(), fd);- if (src1->nb[0] != sizeof(float) || !ggml_is_contiguous(dst)) {+ auto clone = std::make_unique<ggml_hexagon_shared_buffer>(this, *sbuf);+ try {+ clone->mmap(sbuf->extended);+ } catch (const std::exception & exc) {+ GGML_LOG_ERROR("ggml-hex: %s lazy mapping of buffer context failed: %s\n", this->c_name(), exc.what());return false;}- const int32_t * params = (const int32_t *) op->op_params;- const int32_t s0 = params[0], s1 = params[1], s2 = params[2];- const int32_t p0 = params[3], p1 = params[4], p2 = params[5];- const int32_t d0 = params[6], d1 = params[7], d2 = params[8];- const int32_t ic = params[9];+ this->cloned_buffers[fd] = std::move(clone);+ mem->mapped_clones.insert(this);+ return true;+}- if (s0 <= 0 || s1 <= 0 || s2 <= 0 ||- p0 < 0 || p1 < 0 || p2 < 0 ||- d0 <= 0 || d1 <= 0 || d2 <= 0 || ic <= 0) {- return false;- }+void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbuf) {+ GGML_ASSERT(sbuf && sbuf->mem);- if (src1->ne[3] % ic != 0) {- return false;- }+ auto mem = sbuf->mem;+ int fd = mem->fd;- const uint32_t IC = (uint32_t) ic;- const uint32_t N = (uint32_t) (src1->ne[3] / IC);- if (N == 0) {- return false;- }+ GGML_ASSERT(fd >= 0);- uint32_t OW, OH, OD;- if (!im2col_3d_calc_output_size(src1->ne[0], src0->ne[0], s0, p0, d0, &OW) ||- !im2col_3d_calc_output_size(src1->ne[1], src0->ne[1], s1, p1, d1, &OH) ||- !im2col_3d_calc_output_size(src1->ne[2], src0->ne[2], s2, p2, d2, &OD)) {- return false;+ auto it = this->cloned_buffers.find(fd);+ if (it != this->cloned_buffers.end()) {+ auto clone = std::move(it->second);+ this->cloned_buffers.erase(it);}+ mem->mapped_clones.erase(this);+}- const uint32_t KD = (uint32_t) src0->ne[2];- const uint32_t KH = (uint32_t) src0->ne[1];- const uint32_t KW = (uint32_t) src0->ne[0];- const uint32_t ID = (uint32_t) src1->ne[2];- const uint32_t IH = (uint32_t) src1->ne[1];- const uint32_t IW = (uint32_t) src1->ne[0];+void ggml_hexagon_session::unclone_buffer(const ggml_hexagon_shared_buffer * sbuf) {+ GGML_ASSERT(sbuf && sbuf->mem);- const uint64_t patch_stride_64 = (uint64_t) IC * KD * KH * KW;- const uint64_t total_rows_64 = (uint64_t) N * OD * OH;- const uint64_t dst_ne3_64 = (uint64_t) OD * N;+ auto mem = sbuf->mem;+ std::vector<ggml_hexagon_session *> sessions(mem->mapped_clones.begin(), mem->mapped_clones.end());- if (patch_stride_64 > UINT32_MAX || total_rows_64 > UINT32_MAX || dst_ne3_64 > UINT32_MAX) {- return false;+ for (auto * sess : sessions) {+ sess->release_buffer(sbuf);}- const uint32_t patch_stride = (uint32_t) patch_stride_64;- const uint32_t total_rows = (uint32_t) total_rows_64;- const uint32_t dst_ne3 = (uint32_t) dst_ne3_64;+}- if (dst->ne[0] != patch_stride || dst->ne[1] != OW || dst->ne[2] != OH || dst->ne[3] != dst_ne3) {- return false;- }+static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) {+ // Allocate a bunch pinned buffers till failure.+ // This is kind of expensive but handy for figuring out exactly how much we can mmap on a specific device.+ // Typically we're going to allocate all/most of these buffers anyway for the model weights.- const uint32_t dst_elem = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);-- const bool is_patchembed = ggml_is_contiguous(src1) &&- (s0 == (int32_t) KW && s1 == (int32_t) KH && s2 == (int32_t) KD &&- p0 == 0 && p1 == 0 && p2 == 0 &&- d0 == 1 && d1 == 1 && d2 == 1);-- const uint32_t max_threads = (sess && sess->n_threads > 0) ? (uint32_t) sess->n_threads : 8;- const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads;- const uint32_t vtcm_size = (sess && sess->vtcm_size > 0) ? sess->vtcm_size : 0;-- if (is_patchembed) {- const uint64_t src_row_bytes_raw = (uint64_t) IC * KD * KH * IW * sizeof(float);- const uint64_t dst_row_bytes_raw = (uint64_t) OW * patch_stride * dst_elem;- if (src_row_bytes_raw <= UINT32_MAX && dst_row_bytes_raw <= UINT32_MAX) {- const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);- const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);-- struct htp_im2col_vtcm_layout layout;- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);-- if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {- kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;- kparams->is_2d = 0;- kparams->is_3d = 1;- kparams->n_threads = (uint8_t) n_threads;- kparams->vtcm_size = layout.total_bytes;- kparams->off_src = layout.off_src;- kparams->off_dst = layout.off_dst;- kparams->src_bytes_per_thread = layout.src_bytes_per_thread;- kparams->dst_bytes_per_thread = layout.dst_bytes_per_thread;- kparams->src_row_bytes = src_row_bytes;- kparams->dst_row_bytes = dst_row_bytes;- kparams->owb = OW;- kparams->wb = IW;- kparams->nbpr = 1;- kparams->N = N;- kparams->IC = IC;- kparams->ID = ID;- kparams->IH = IH;- kparams->IW = IW;- kparams->KD = KD;- kparams->KH = KH;- kparams->KW = KW;- kparams->OD = OD;- kparams->OH = OH;- kparams->OW = OW;- kparams->patch_stride = patch_stride;- kparams->total_rows = total_rows;- return true;- }- }- }+ std::vector<ggml_hexagon_shared_buffer *> sbufs;- for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) {- const uint64_t Wb = (uint64_t) (owb - 1) * s0 + (uint64_t) (KW - 1) * d0 + 1;- if (Wb > UINT32_MAX) {- if (owb == 1) break;- continue;- }+ const size_t MiB = 1024 * 1024;+ const size_t GiB = MiB * 1024;- const uint64_t src_row_bytes_raw = (uint64_t) IC * KD * KH * Wb * sizeof(float);- const uint64_t dst_row_bytes_raw = (uint64_t) owb * patch_stride * dst_elem;- if (src_row_bytes_raw > UINT32_MAX || dst_row_bytes_raw > UINT32_MAX) {- if (owb == 1) break;- continue;- }+ size_t vmem = 0;+ size_t step = 256u * MiB;++ try {+ sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB;+ sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB;+ sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB;- const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);- const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);-- struct htp_im2col_vtcm_layout layout;- htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);-- if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {- kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA;- kparams->is_2d = 0;- kparams->is_3d = 1;- kparams->n_threads = (uint8_t) n_threads;- kparams->vtcm_size = layout.total_bytes;- kparams->off_src = layout.off_src;- kparams->off_dst = layout.off_dst;- kparams->src_bytes_per_thread = layout.src_bytes_per_thread;- kparams->dst_bytes_per_thread = layout.dst_bytes_per_thread;- kparams->src_row_bytes = src_row_bytes;- kparams->dst_row_bytes = dst_row_bytes;- kparams->owb = owb;- kparams->wb = (uint32_t) Wb;- kparams->nbpr = (OW + owb - 1) / owb;- kparams->N = N;- kparams->IC = IC;- kparams->ID = ID;- kparams->IH = IH;- kparams->IW = IW;- kparams->KD = KD;- kparams->KH = KH;- kparams->KW = KW;- kparams->OD = OD;- kparams->OH = OH;- kparams->OW = OW;- kparams->patch_stride = patch_stride;- kparams->total_rows = total_rows;- return true;+ while (1) {+ sbufs.push_back(new ggml_hexagon_shared_buffer(sess, step, true));+ vmem += step;}- if (owb == 1) break;- }+ } catch (...) { }- return false;-}+ for (auto b : sbufs) { delete b; }-static bool ggml_hexagon_supported_im2col(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- struct htp_im2col_kernel_params kparams;- return ggml_hexagon_precompute_im2col_params(sess, op, &kparams);+ return vmem - step; // backoff to account for overhead from internal mappings}-static bool ggml_hexagon_supported_im2col_3d(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- struct htp_im2col_kernel_params kparams;- return ggml_hexagon_precompute_im2col_3d_params(sess, op, &kparams);-}+void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) noexcept(false) {+ int phys_idx = config.physical_idx;+ int virt_idx = config.virtual_idx;-static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * dst = op;+ this->valid_session = false;+ this->valid_handle = false;+ this->valid_queue = false;+ this->valid_iface = false;- if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {- return false;- }+ this->name = config.name;+ this->phys_idx = phys_idx;+ this->virt_idx = virt_idx;+ this->domain_id = config.domain_id;+ this->session_id = 0;+ this->batch_req_seq = 0;+ this->batch_rsp_seq = 0;+ this->last_error = HTP_STATUS_OK;- const int32_t lp0 = ((const int32_t *) op->op_params)[0];- const int32_t rp0 = ((const int32_t *) op->op_params)[1];- const int32_t circular = ((const int32_t *) op->op_params)[8];+ GGML_LOG_DEBUG("ggml-hex: %s allocating new session : domain %u phys-idx %u virt-idx %u\n", this->name.c_str(), this->domain_id, phys_idx, virt_idx);- if (circular && (lp0 > src0->ne[0] || rp0 > src0->ne[0])) {- return false;+ if (config.domain_id < 0 || config.domain_name.empty()) {+ GGML_LOG_ERROR("ggml-hex: %s: invalid physical CDSP core %d\n", config.name.c_str(), config.physical_idx);+ throw std::runtime_error("ggml-hex: invalid physical CDSP core");}- return true;+ const std::string & dom_name = config.domain_name;- GGML_UNUSED(sess);-}+ // Create new session if virtual_idx > 0+ if (virt_idx > 0) {+ struct remote_rpc_reserve_new_session n {};+ n.domain_name_len = dom_name.size();+ n.domain_name = const_cast<char *>(dom_name.c_str());+ n.session_name = const_cast<char *>(this->name.c_str());+ n.session_name_len = this->name.size();+ n.session_id = virt_idx;-static bool ggml_hexagon_supported_cumsum(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * dst = op;+ int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n));+ if (err != AEE_SUCCESS) {+ GGML_LOG_ERROR("ggml-hex: %s failed to reserve new session (physical %d, virtual %d) : error 0x%x\n",+ this->c_name(), phys_idx, virt_idx, err);+ throw std::runtime_error("ggml-hex: remote_session_control(new-sess) failed (see log for details)");+ }- if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {- return false;+ // Save the IDs+ this->session_id = n.session_id;+ this->domain_id = n.effective_domain_id;+ this->valid_session = true;+ } else {+ struct remote_rpc_effective_domain_id eff {};+ eff.domain_name = const_cast<char *>(dom_name.c_str());+ eff.domain_name_len = dom_name.size();+ eff.session_id = 0;++ int err = remote_session_control(FASTRPC_GET_EFFECTIVE_DOMAIN_ID, (void *) &eff, sizeof(eff));+ if (err == AEE_SUCCESS) {+ this->domain_id = eff.effective_domain_id;+ } else {+ GGML_LOG_DEBUG("ggml-hex: %s FASTRPC_GET_EFFECTIVE_DOMAIN_ID returned 0x%x, using domain_id %d\n",+ this->name.c_str(), err, this->domain_id);+ }}- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {- return false;+ // Enable unsigned modules+ {+ struct remote_rpc_control_unsigned_module u;+ u.domain = this->domain_id;+ u.enable = 1;+ int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u));+ if (err != AEE_SUCCESS) {+ GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err);+ throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)");+ }}- return true;+ char session_uri[256];+ {+ char htp_uri[256];+ snprintf(htp_uri, sizeof(htp_uri), "file:///libggml-htp-v%u.so?htp_iface_skel_handle_invoke&_modver=1.0", opt_arch);- GGML_UNUSED(sess);-}+ struct remote_rpc_get_uri u = {};+ u.session_id = this->session_id;+ u.domain_name = const_cast<char *>(dom_name.c_str());+ u.domain_name_len = dom_name.size();+ u.module_uri = const_cast<char *>(htp_uri);+ u.module_uri_len = strlen(htp_uri);+ u.uri = session_uri;+ u.uri_len = sizeof(session_uri);-static bool ggml_hexagon_supported_diag(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * dst = op;+ int err = remote_session_control(FASTRPC_GET_URI, (void *) &u, sizeof(u));+ if (err != AEE_SUCCESS) {+ snprintf(session_uri, sizeof(session_uri), "%s&_dom=%s&_session=%u",+ htp_uri, dom_name.c_str(), this->session_id);- // diag only supports F32 currently- if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {- return false;+ GGML_LOG_WARN("ggml-hex: %s failed to get URI (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n",+ this->c_name(), phys_idx, virt_idx, err, session_uri);+ }}- // Input must have ne[1] == 1 (vector input)- if (src0->ne[1] != 1) {- return false;+ // Open session+ int err = htp_iface_open(session_uri, &this->handle);+ if (err != AEE_SUCCESS) {+ GGML_LOG_ERROR("ggml-hex: %s failed to open session : uri %s error 0x%x\n", this->c_name(), session_uri, err);+ throw std::runtime_error("ggml-hex: failed to open session (see log for details)");}- // Output must be square in first two dimensions- if (dst->ne[0] != dst->ne[1] || dst->ne[0] != src0->ne[0]) {- return false;+ this->valid_handle = true;++ // Query HW info and resolve session options+ {+ unsigned int hw_n_threads = 0;+ unsigned int hw_n_hvx = 0;+ unsigned int hw_n_hmx = 0;+ unsigned long long hw_vtcm_size = 0;+ int hw_err = htp_iface_hwinfo(this->handle, &hw_n_threads, &hw_n_hvx, &hw_n_hmx, &hw_vtcm_size);+ if (hw_err == 0) {+ const uint32_t max_n_threads = (std::min)((uint32_t) HTP_MAX_NTHREADS, (uint32_t) hw_n_threads);+ this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_threads) : max_n_threads;+ this->n_hvx = this->n_threads;+ this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0;+ this->vtcm_size = (uint64_t)hw_vtcm_size;+ GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n",+ this->c_name(), this->n_threads, this->n_hvx, this->n_hmx,+ (unsigned long long)(this->vtcm_size / (1024 * 1024)));+ } else {+ GGML_LOG_WARN("ggml-hex: %s failed to query hwinfo (0x%x), using defaults\n", this->c_name(), hw_err);+ const uint32_t default_n_threads = (std::min)(8u, (uint32_t) HTP_MAX_NTHREADS);+ this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads;+ this->n_hvx = this->n_threads;+ this->n_hmx = (opt_nhmx != 0) ? 1 : 0;+ this->vtcm_size = 8 * 1024 * 1024;+ }}- return true;+ // Enable FastRPC QoS mode+ {+ struct remote_rpc_control_latency l;+ l.enable = 1;- GGML_UNUSED(sess);-}+ int err = remote_handle64_control(this->handle, DSPRPC_CONTROL_LATENCY, (void *) &l, sizeof(l));+ if (err != 0) {+ GGML_LOG_WARN("ggml-hex: failed to enable fastrpc QOS mode: 0x%08x\n", (unsigned) err);+ }+ }-static bool ggml_hexagon_supported_solve_tri(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * src0 = op->src[0]; // A- const struct ggml_tensor * src1 = op->src[1]; // B- const struct ggml_tensor * dst = op; // X+ GGML_LOG_INFO("ggml-hex: %s new session : session-id %d domain-id %d uri %s handle 0x%lx\n", this->c_name(),+ this->session_id, this->domain_id, session_uri, (unsigned long) this->handle);- if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {- return false;- }+ const size_t req_q_size = (sizeof(htp_opbatch_req) * opt_opqueue * 2) + 1024;+ const size_t rsp_q_size = (sizeof(htp_opbatch_rsp) * opt_opqueue * 2) + 1024;- if (src0->ne[0] != src0->ne[1]) {- return false;+ // Now let's setup the DSP queue+ err = dspqueue_create(this->domain_id,+ 0, // Flags+ req_q_size, // Request queue size (in bytes)+ rsp_q_size, // Response queue size (in bytes)+ nullptr, // Read packet callback (we handle reads explicitly)+ nullptr, // Error callback (we handle errors during reads)+ (void *) this, // Callback context+ &queue);+ if (err != 0) {+ GGML_LOG_ERROR("ggml-hex: %s dspqueue_create failed: 0x%08x\n", this->name.c_str(), (unsigned) err);+ throw std::runtime_error("ggml-hex: failed to create dspqueue (see log for details)");}- if (src0->ne[1] != src1->ne[1]) {- return false;- }+ this->valid_queue = true;- if (src0->ne[2] != src1->ne[2] || src0->ne[3] != src1->ne[3]) {- return false;+ // Export queue for use on the DSP+ err = dspqueue_export(queue, &this->queue_id);+ if (err != 0) {+ GGML_LOG_ERROR("ggml-hex: dspqueue_export failed: 0x%08x\n", (unsigned) err);+ throw std::runtime_error("ggml-hex: dspqueue export failed (see log for details)");}- if (dst->ne[0] != src1->ne[0] || dst->ne[1] != src1->ne[1] || dst->ne[2] != src1->ne[2] || dst->ne[3] != src1->ne[3]) {- return false;+ if (opt_etm) {+ err = htp_iface_etm(this->handle, 1);+ if (err != 0) {+ GGML_LOG_ERROR("ggml-hex: failed to enable ETM tracing: 0x%08x\n", (unsigned) err);+ }}- return true;-- GGML_UNUSED(sess);-}--static bool ggml_hexagon_supported_tri(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {-- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * dst = op;-- if (src0->type != GGML_TYPE_F32) { return false; }- if (dst->type != GGML_TYPE_F32) { return false; }- if (!ggml_are_same_shape(src0, dst)) { return false; }- if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { return false; }-- return true;+ // Allocate buffers and state for op batching+ this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue);- GGML_UNUSED(sess);-}+ this->fence_buf = new ggml_hexagon_fence_buffer(this, &dev_ctx->fence_buffer_type, 64 * 1024);+ if (this->mdev.count > 1) {+ this->mdev_fence_slot = this->alloc_fence(this->mdev.count);+ }-static const char * ggml_backend_hexagon_name(ggml_backend_t backend) {- auto sess = static_cast<ggml_hexagon_session *>(backend->context);- return sess->c_name();-}+ if (!opt_vmem) {+ opt_vmem = ggml_hexagon_measure_max_vmem(this);+ GGML_LOG_INFO("ggml-hex: %s measured max vmem %zu\n", this->c_name(), opt_vmem);+ }+ const size_t shm_size = this->op_queue->shm_size();+ this->max_vmem = (opt_vmem > shm_size) ? (opt_vmem - shm_size) : opt_vmem;-static void ggml_backend_hexagon_free(ggml_backend_t backend) {- // we just need to delete the backend here- // the sessions are allocated & freed as part of the registry- delete backend;-}+ this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem);-static htp_op_code op_remap_to_htp(const ggml_tensor * t) {- switch (t->op) {- case GGML_OP_FLASH_ATTN_EXT: return HTP_OP_FLASH_ATTN_EXT;- case GGML_OP_MUL_MAT:- if (ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD) {- return HTP_OP_HADAMARD;- }- return HTP_OP_MUL_MAT;- case GGML_OP_MUL_MAT_ID: return HTP_OP_MUL_MAT_ID;- case GGML_OP_MUL: return HTP_OP_MUL;- case GGML_OP_ADD: return HTP_OP_ADD;- case GGML_OP_ADD_ID: return HTP_OP_ADD_ID;- case GGML_OP_SUB: return HTP_OP_SUB;- case GGML_OP_DIV: return HTP_OP_DIV;- case GGML_OP_DUP:- case GGML_OP_CPY: return HTP_OP_CPY;- case GGML_OP_CONT: return HTP_OP_CPY;- case GGML_OP_GET_ROWS: return HTP_OP_GET_ROWS;- case GGML_OP_SET_ROWS: return HTP_OP_SET_ROWS;- case GGML_OP_SUM: return HTP_OP_SUM;- case GGML_OP_SUM_ROWS: return HTP_OP_SUM_ROWS;- case GGML_OP_POOL_2D: return HTP_OP_POOL_2D;- case GGML_OP_POOL_1D: return HTP_OP_POOL_1D;- case GGML_OP_ARGSORT: return HTP_OP_ARGSORT;- case GGML_OP_TOP_K: return HTP_OP_TOP_K;- case GGML_OP_ARGMAX: return HTP_OP_ARGMAX;- case GGML_OP_NORM: return HTP_OP_NORM;- case GGML_OP_L2_NORM: return HTP_OP_L2_NORM;- case GGML_OP_RMS_NORM: return HTP_OP_RMS_NORM;- case GGML_OP_CONCAT: return HTP_OP_CONCAT;- case GGML_OP_SCALE: return HTP_OP_SCALE;- case GGML_OP_CLAMP: return HTP_OP_CLAMP;- case GGML_OP_LEAKY_RELU: return HTP_OP_LEAKY_RELU;- case GGML_OP_SQR: return HTP_OP_SQR;- case GGML_OP_SQRT: return HTP_OP_SQRT;- case GGML_OP_LOG: return HTP_OP_UNARY_LOG;- case GGML_OP_SOFT_MAX: return HTP_OP_SOFTMAX;- case GGML_OP_SSM_CONV: return HTP_OP_SSM_CONV;- case GGML_OP_GATED_DELTA_NET: return HTP_OP_GATED_DELTA_NET;- case GGML_OP_ROPE: return HTP_OP_ROPE;- case GGML_OP_REPEAT: return HTP_OP_REPEAT;- case GGML_OP_CUMSUM: return HTP_OP_CUMSUM;- case GGML_OP_FILL: return HTP_OP_FILL;- case GGML_OP_DIAG: return HTP_OP_DIAG;- case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI;- case GGML_OP_TRI: return HTP_OP_TRI;- case GGML_OP_PAD: return HTP_OP_PAD;- case GGML_OP_IM2COL: return HTP_OP_IM2COL;- case GGML_OP_IM2COL_3D: return HTP_OP_IM2COL_3D;- case GGML_OP_ROLL: return HTP_OP_ROLL;-- case GGML_OP_UNARY:- switch (ggml_get_unary_op(t)) {- case GGML_UNARY_OP_SILU: return HTP_OP_UNARY_SILU;- case GGML_UNARY_OP_GELU: return HTP_OP_UNARY_GELU;- case GGML_UNARY_OP_GELU_QUICK: return HTP_OP_UNARY_GELU_QUICK;- case GGML_UNARY_OP_GELU_ERF: return HTP_OP_UNARY_GELU_ERF;- case GGML_UNARY_OP_SIGMOID: return HTP_OP_UNARY_SIGMOID;- case GGML_UNARY_OP_NEG: return HTP_OP_UNARY_NEG;- case GGML_UNARY_OP_EXP: return HTP_OP_UNARY_EXP;- case GGML_UNARY_OP_SOFTPLUS: return HTP_OP_UNARY_SOFTPLUS;- case GGML_UNARY_OP_TANH: return HTP_OP_UNARY_TANH;- case GGML_UNARY_OP_ABS: return HTP_OP_UNARY_ABS;- case GGML_UNARY_OP_RELU: return HTP_OP_UNARY_RELU;- case GGML_UNARY_OP_STEP: return HTP_OP_UNARY_STEP;- default:- break;- }- break;+ // Start dspqueue/opbatch processing+ err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem, (uint32_t) opt_power_mode);+ if (err != 0) {+ GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err);+ throw std::runtime_error("ggml-hex: iface start failed (see log for details)");+ }+ this->valid_iface = true;+ GGML_LOG_INFO("ggml-hex: %s power mode: %s\n", this->c_name(), htp_power_mode_to_str(opt_power_mode));- case GGML_OP_GLU:- switch (ggml_get_glu_op(t)) {- case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU;- case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI;- case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP;- case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU;- case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK;- case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF;- default: break;- }- break;+ if (opt_profile) {+ htp_iface_pmu_conf pmu_conf{};+ std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events);- default:- GGML_ABORT("\nggml-hex: graph-compute %s is not supported\n", ggml_op_desc(t));+ err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf);+ if (err != 0) {+ GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err);+ }}- return HTP_OP_INVALID;}-static inline bool op_is_compute(ggml_tensor *node)-{- return !ggml_op_is_empty(node->op) && !ggml_is_empty(node) && (node->flags & GGML_TENSOR_FLAG_COMPUTE);-}+void ggml_hexagon_session::release() noexcept(true) {+ GGML_LOG_INFO("ggml-hex: releasing session: %s\n", this->name.c_str());-static bool mm_is_hmx_eligible(const ggml_tensor * t) {- if (opt_nhmx == 0) { return false; }+ this->mdev.sessions.clear();- const ggml_tensor * src0 = t->src[0];- const ggml_tensor * src1 = t->src[1];+ int err;- if (ggml_hexagon_matmul_can_collapse(src0, src1, t)) {- const ggml_tensor src1_c = ggml_hexagon_tensor_collapse_rows(src1);- return ggml_hexagon_matmul_is_hmx_eligible(src0, &src1_c, t, false, false);+ if (this->valid_iface) {+ // Stop dspqueue/opbatch processing+ err = htp_iface_stop(this->handle);+ if (err != 0) {+ GGML_ABORT("ggml-hex: htp_iface_stop failed: 0x%08x\n", (unsigned) err);+ }}- const bool is_matmul_id = (t->op == GGML_OP_MUL_MAT_ID);- const bool is_batched = (src0->ne[2] * src0->ne[3] > 1 || src1->ne[2] * src1->ne[3] > 1);-- return ggml_hexagon_matmul_is_hmx_eligible(src0, src1, t, is_matmul_id, is_batched);-}--static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) {- if (kparams->n_hmx) {- return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;+ delete this->op_batch;+ delete this->op_queue;+ for (auto & it : this->cpy_fence_slots) {+ free_fence((void *) it.second, 1);}+ this->cpy_fence_slots.clear();- if (!ggml_hexagon_tensor_is_tiled(src0)) {- return false;+ if (this->fence_buf) {+ unclone_buffer(this->fence_buf);+ delete this->fence_buf;+ this->fence_buf = nullptr;+ }+ while (!this->cloned_buffers.empty()) {+ release_buffer(this->cloned_buffers.begin()->second.get());}- return kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW;-}+ if (opt_etm) {+ err = htp_iface_etm(this->handle, 0);+ if (err != 0) {+ GGML_LOG_ERROR("ggml-hex: warn : failed to disable ETM tracing: 0x%08x\n", (unsigned) err);+ }+ }-static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) {- if (kparams->n_hmx) {- return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;+ if (opt_profile) {+ htp_iface_pmu_conf pmu_conf{};+ err = htp_iface_profiler(this->handle, 0, &pmu_conf);+ if (err != 0) {+ GGML_LOG_ERROR("ggml-hex: warn : failed to disable profiling: 0x%08x\n", (unsigned) err);+ }}- if (!ggml_hexagon_tensor_is_tiled(src0)) {- return false;+ if (this->valid_queue) {+ err = dspqueue_close(queue);+ if (err != 0) {+ GGML_ABORT("ggml-hex: dspqueue_close failed: 0x%08x\n", (unsigned) err);+ }}- return kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK;+ if (this->valid_handle) {+ htp_iface_close(this->handle);+ }}-static bool is_mergeable_mul_mat(const ggml_tensor * t) {- if (t->op != GGML_OP_MUL_MAT) return false;- if (ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD) return false;-- const ggml_tensor * src0 = t->src[0];- const ggml_tensor * src1 = t->src[1];- if (src0->ne[2] != 1 || src0->ne[3] != 1) return false;+ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev, uint32_t mdev_idx, uint32_t mdev_count) noexcept(false) {+ this->dev = dev;+ this->dev_ctx = static_cast<ggml_backend_hexagon_device_context *>(dev->context);+ this->mdev.idx = mdev_idx;+ this->mdev.count = mdev_count > 0 ? mdev_count : (uint32_t) (1 + config.mdev_group.size());+ op_batch = nullptr;+ op_queue = nullptr;+ fence_buf = nullptr;+ fence_seq = ((uintptr_t)this) & 0xFFFF;- if (mm_is_hmx_eligible(t)) {- return htp_is_hmx_weight_type(src0->type);+ try {+ allocate(config);+ if (this->mdev.idx == 0 && !config.mdev_group.empty()) {+ for (size_t i = 0; i < config.mdev_group.size(); i++) {+ this->mdev.sessions.push_back(std::make_unique<ggml_hexagon_session>(+ config.mdev_group[i], this->dev, (uint32_t) (i + 1), this->mdev.count));+ }+ }+ } catch (const std::exception & exc) {+ release();+ throw;}-- // HVX path requires F32 activations and tiled weights- if (src1->type != GGML_TYPE_F32) return false;-- return ggml_hexagon_tensor_is_tiled(src0);}-static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor * n2) {- if (!is_mergeable_mul_mat(n1) || !is_mergeable_mul_mat(n2)) {- return false;- }- if (n1->src[1] != n2->src[1]) {- return false;- }- if (n1->src[0]->ne[0] != n2->src[0]->ne[0]) {- return false;- }- if (n1->src[0]->type != n2->src[0]->type) {- return false;- }- if (ggml_hexagon_tensor_is_tiled(n1->src[0]) != ggml_hexagon_tensor_is_tiled(n2->src[0])) {- return false;- }- if (mm_is_hmx_eligible(n1) != mm_is_hmx_eligible(n2)) {- return false;- }- return true;+ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) {+ release();}-static bool is_mergeable_mul_mat_id(const ggml_tensor * t) {- if (t->op != GGML_OP_MUL_MAT_ID) return false;+// ** backend interface- const ggml_tensor * src0 = t->src[0];- if (mm_is_hmx_eligible(t)) {- return htp_is_hmx_weight_type(src0->type);- }+static const char * ggml_backend_hexagon_name(ggml_backend_t backend) {+ auto sess = static_cast<ggml_hexagon_session *>(backend->context);+ return sess->c_name();+}- return ggml_hexagon_tensor_is_tiled(src0);+static void ggml_backend_hexagon_free(ggml_backend_t backend) {+ // we just need to delete the backend here+ // the sessions are allocated & freed as part of the registry+ delete backend;}-static bool is_mergeable_mul_mat_id_pair(const ggml_tensor * n1, const ggml_tensor * n2) {- if (!is_mergeable_mul_mat_id(n1) || !is_mergeable_mul_mat_id(n2)) {- return false;- }- if (n1->src[1] != n2->src[1]) {- return false;- }- if (n1->src[2] != n2->src[2]) {- return false;- }- if (n1->src[0]->ne[0] != n2->src[0]->ne[0]) {- return false;- }- if (n1->src[0]->ne[2] != n2->src[0]->ne[2]) {- return false;- }- if (n1->src[0]->type != n2->src[0]->type) {- return false;- }- if (ggml_hexagon_tensor_is_tiled(n1->src[0]) != ggml_hexagon_tensor_is_tiled(n2->src[0])) {- return false;- }- if (mm_is_hmx_eligible(n1) != mm_is_hmx_eligible(n2)) {- return false;- }- return true;+static inline bool op_is_compute(ggml_tensor *node)+{+ return !ggml_op_is_empty(node->op) && !ggml_is_empty(node) && (node->flags & GGML_TENSOR_FLAG_COMPUTE);}static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) {@@ -6916,29 +2365,11 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggconst std::vector<htp_opnode> * nodes_ptr = nullptr;std::vector<htp_opnode> computed_nodes;- // Check for cache hitbool cache_hit = (graph->uid != 0 && sess->cached_uid == graph->uid);if (cache_hit) {nodes_ptr = &sess->cached_nodes;} else {- // Tag fusable tensors in graph- for (int i = 0; i < graph->n_nodes; i++) {- auto * extra = (ggml_hexagon_tensor_extra *) graph->nodes[i]->extra;- if (!extra) continue;-- extra->flags &= ~GGML_HEXAGON_TENSOR_FUSEABLE;-- if (graph->nodes[i]->op == GGML_OP_RMS_NORM && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {- extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE;- } else if ((graph->nodes[i]->op == GGML_OP_MUL_MAT &&- ggml_get_op_params_i32(graph->nodes[i], 1) != GGML_HINT_SRC0_IS_HADAMARD) ||- graph->nodes[i]->op == GGML_OP_MUL_MAT_ID) {- if ((i + 1 < graph->n_nodes && graph->nodes[i + 1]->op == GGML_OP_ADD && ggml_can_fuse(graph, i, { graph->nodes[i]->op, GGML_OP_ADD })) ||- ggml_node_has_n_uses(graph, i, 1)) {- extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE;- }- }- }+ htp_fusion_tag_graph(graph);computed_nodes.reserve(graph->n_nodes);@@ -6948,118 +2379,29 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggcontinue;}- htp_opnode node(HTP_OP_INVALID, n);- node.opcode = op_remap_to_htp(n);+ const htp_op_code opcode = htp_op_remap(n);+ if (opcode == HTP_OP_INVALID) {+ GGML_ABORT("\nggml-hex: graph-compute %s is not supported\n", ggml_op_desc(n));+ }++ htp_opnode node(opcode, n);if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID || node.opcode == HTP_OP_GET_ROWS) {ggml_tensor * src0 = node.node->src[0];- if (ggml_hexagon_needs_repack(src0)) {- if (opt_verbose) {- char dims_buf[64];- char strides_buf[64];- htp_opformat fmt;- fmt.format_tensor_dims(dims_buf, sizeof(dims_buf), src0);- fmt.format_tensor_strides(strides_buf, sizeof(strides_buf), src0);- GGML_LOG_DEBUG("ggml-hex: %s rt-repack %s|%s|%s|%s|size %zu\n",- sess->c_name(), src0->name, dims_buf, ggml_type_name(src0->type), strides_buf, ggml_nbytes(src0));- }+ if (htp_needs_repack(src0)) {+ HEX_VERBOSE("ggml-hex: %s rt-repack %s|size %zu\n", sess->c_name(), htp_format_tensor(src0, HTP_OPFMT_TENSOR).c_str(), ggml_nbytes(src0));std::vector<uint8_t> raw(ggml_nbytes(src0));memcpy(raw.data(), src0->data, raw.size());htp_repack_tensor_tiled(src0, raw.data(), raw.size());- auto * extra = (ggml_hexagon_tensor_extra *) src0->extra;- extra->flags |= GGML_HEXAGON_TENSOR_TILED;+ auto * extra = (htp_tensor_extra *) src0->extra;+ extra->flags |= HTP_TENSOR_TILED;}}- if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID) {- ggml_hexagon_precompute_matmul_params(sess,- node.node->src[0], node.node->src[1], node.node,- (struct htp_mm_kernel_params *)node.kernel_params- );- } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {- const ggml_tensor * src1 = node.node->src[1];- const ggml_tensor * src2 = (node.opcode == HTP_OP_ADD_ID) ? node.node->src[2] : nullptr;- GGML_ASSERT(ggml_hexagon_precompute_binary_params(sess,- node.opcode, node.node->src[0], src1, node.node,- (struct htp_binary_kernel_params *) node.kernel_params, src2));- } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) {- ggml_hexagon_precompute_flash_attn_params(sess,- node.node,- (struct htp_fa_kernel_params *)node.kernel_params- );- } else if (htp_op_is_unary(node.opcode)) {- auto inputs = node.get_inputs();- const struct ggml_tensor * src0 = inputs[0];- const struct ggml_tensor * src1 = inputs.size() > 1 ? inputs[1] : nullptr;- ggml_hexagon_precompute_unary_params(sess,- node.opcode, src0, src1, node.dst(),- (struct htp_unary_kernel_params *)node.kernel_params- );- } else if (node.opcode == HTP_OP_GET_ROWS) {- ggml_hexagon_precompute_get_rows_params(sess,- node.node->src[0], node.node->src[1], node.dst(),- (struct htp_get_rows_kernel_params *)node.kernel_params- );- } else if (node.opcode == HTP_OP_SET_ROWS) {- ggml_hexagon_precompute_set_rows_params(sess,- node.node->src[0], node.node->src[1], node.dst(),- (struct htp_set_rows_kernel_params *)node.kernel_params- );- } else if (node.opcode == HTP_OP_ROPE) {- ggml_hexagon_precompute_rope_params(sess,- node.node,- (struct htp_rope_kernel_params *)node.kernel_params- );- } else if (node.opcode == HTP_OP_SSM_CONV) {- ggml_hexagon_precompute_ssm_conv_params(sess,- node.node->src[0], node.node->src[1], node.dst(),- (struct htp_ssm_conv_kernel_params *)node.kernel_params- );- } else if (node.opcode == HTP_OP_SOFTMAX) {- ggml_hexagon_precompute_softmax_params(sess,- node.node,- (struct htp_softmax_kernel_params *)node.kernel_params- );- } else if (node.opcode == HTP_OP_GATED_DELTA_NET) {- ggml_hexagon_precompute_gated_delta_net_params(sess,- node.node,- (struct htp_gdn_kernel_params *)node.kernel_params- );- } else if (node.opcode == HTP_OP_ARGSORT || node.opcode == HTP_OP_TOP_K) {- ggml_hexagon_precompute_sort_params(sess,- node.node,- node.opcode == HTP_OP_TOP_K,- (struct htp_sort_kernel_params *) node.kernel_params- );- } else if (node.opcode == HTP_OP_POOL_2D || node.opcode == HTP_OP_POOL_1D) {- ggml_hexagon_precompute_pool_2d_params(- sess, node.node->src[0], node.dst(),- (struct htp_pool_2d_kernel_params *)node.kernel_params,- node.opcode == HTP_OP_POOL_1D);- } else if (node.opcode == HTP_OP_CONCAT) {- ggml_hexagon_precompute_concat_params(sess,- node.node,- (struct htp_concat_kernel_params *) node.kernel_params- );- } else if (node.opcode == HTP_OP_CPY || node.opcode == HTP_OP_CPY_FENCE) {- const bool ok = ggml_hexagon_precompute_cpy_params(sess,- node.node,- (struct htp_copy_kernel_params *) node.kernel_params- );- const auto * kparams = (const struct htp_copy_kernel_params *) node.kernel_params;- if (ok && node.opcode == HTP_OP_CPY && kparams->total_elems == 0) {- continue;- }- } else if (node.opcode == HTP_OP_IM2COL) {- ggml_hexagon_precompute_im2col_params(sess,- node.node,- (struct htp_im2col_kernel_params *) node.kernel_params- );- } else if (node.opcode == HTP_OP_IM2COL_3D) {- ggml_hexagon_precompute_im2col_3d_params(sess,- node.node,- (struct htp_im2col_kernel_params *) node.kernel_params- );+ GGML_ASSERT(htp_kparams_solve(sess, node.node, node.kernel_params));+ if (node.opcode == HTP_OP_CPY && node.kparams<htp_copy_kernel_params>()->total_elems == 0) {+ continue;}+computed_nodes.push_back(std::move(node));}@@ -7072,7 +2414,6 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg}}- // Queue and executefor (const auto & node : *nodes_ptr) {sess->enqueue_op(node);}@@ -7156,7 +2497,22 @@ static ggml_hexagon_mem_range ggml_hexagon_mem_range_from_tensor(const ggml_tensreturn mr;}-static void ggml_hexagon_mem_ranges_add_node(ggml_hexagon_mem_ranges & mrs, const htp_opnode & node) {+struct node_info {+ ggml_tensor * node;+ htp_opvec<ggml_tensor *, HTP_OP_MAX_FUSED> fused;++ const ggml_tensor * dst() const { return fused.empty() ? node : fused.back(); }++ bool is_empty() const {+ return ggml_op_is_empty(node->op);+ }++ void add_fused(ggml_tensor * t) {+ fused.push_back(t);+ }+};++static void ggml_hexagon_mem_ranges_add_node(ggml_hexagon_mem_ranges & mrs, const node_info & node) {if (node.is_empty()) return;for (int i = 0; i < GGML_MAX_SRC; i++) {@@ -7174,7 +2530,7 @@ static void ggml_hexagon_mem_ranges_add_node(ggml_hexagon_mem_ranges & mrs, consmrs.add(ggml_hexagon_mem_range_from_tensor(node.dst(), HEXAGON_MEM_RANGE_TYPE_DST));}-static bool ggml_hexagon_mem_ranges_check_node(const ggml_hexagon_mem_ranges & mrs, const htp_opnode & node) {+static bool ggml_hexagon_mem_ranges_check_node(const ggml_hexagon_mem_ranges & mrs, const node_info & node) {if (node.is_empty()) return true;for (int i = 0; i < GGML_MAX_SRC; i++) {@@ -7196,7 +2552,7 @@ static bool ggml_hexagon_mem_ranges_check_node(const ggml_hexagon_mem_ranges & mreturn mrs.check(ggml_hexagon_mem_range_from_tensor(node.dst(), HEXAGON_MEM_RANGE_TYPE_DST));}-static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<htp_opnode> & nodes) {+static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<node_info> & nodes) {const int n = nodes.size();std::vector<int> res;@@ -7216,16 +2572,16 @@ static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<htconst auto & node0 = nodes[i0];- if (!node0.stackable()) {+ if (!htp_fusion_can_stack(node0.node)) {res.push_back(i0);used[i0] = true;continue;}// that many nodes forward to search for stackable nodes that can reuse VTCM- constexpr int N_FORWARD = 16;+ constexpr int N_FORWARD = 64;- std::vector<int> stack;+ htp_opvec<int, N_FORWARD> stack;stack.push_back(i0);mrs.reset();@@ -7237,7 +2593,7 @@ static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<htconst auto & node1 = nodes[i1];- if (node1.stackable() && node1.same_input(node0) && ggml_hexagon_mem_ranges_check_node(mrs, node1)) {+ if (htp_fusion_can_stack(node0.node, node1.node) && ggml_hexagon_mem_ranges_check_node(mrs, node1)) {stack.push_back(i1);} else {ggml_hexagon_mem_ranges_add_node(mrs, node1);@@ -7258,52 +2614,16 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgrconst int n = gf->n_nodes;- constexpr int MAX_FUSE = 16;-- enum ggml_op ops[MAX_FUSE];-- std::vector<htp_opnode> nodes;+ std::vector<node_info> nodes;nodes.reserve(gf->n_nodes);// Pack nodes for reorderingfor (int i = 0; i < n; i++) {- htp_opnode node(HTP_OP_INVALID, gf->nodes[i]);-- // fuse only ops that start with these operations- // can be expanded when needed- if (node.op() == GGML_OP_ADD ||- node.op() == GGML_OP_NORM ||- node.op() == GGML_OP_RMS_NORM) {- ops[0] = node.op();-- int f = i + 1;- while (f < n && f < i + MAX_FUSE) {- // conservatively allow fusing only these ops- // can be expanded when needed- if (gf->nodes[f]->op != GGML_OP_ADD &&- gf->nodes[f]->op != GGML_OP_MUL &&- gf->nodes[f]->op != GGML_OP_NORM &&- gf->nodes[f]->op != GGML_OP_RMS_NORM) {- break;- }- ops[f - i] = gf->nodes[f]->op;- f++;- }-- f -= i;- for (; f > 1; f--) {- if (ggml_can_fuse(gf, i, ops, f)) {- break;- }- }+ node_info node = { gf->nodes[i], {} };- // add the fused tensors into the node info so we can unfuse them later- for (int k = 1; k < f; k++) {- ++i;-- // the .dst() becomes the last fused tensor- node.add_fused(gf->nodes[i]);- }+ const int f = htp_fusion_max(gf, i);+ for (int k = 1; k < f; k++) {+ node.add_fused(gf->nodes[++i]);}nodes.push_back(std::move(node));@@ -7349,29 +2669,12 @@ static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_if (!sess_src->clone_buffer(sess_dst->fence_buf)) { return false; }- if (++sess_dst->fence_seq == 0) sess_dst->fence_seq = 1;- uint32_t fence_seq = sess_dst->fence_seq;+ uint32_t fence_seq = sess_dst->next_fence_seq();HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq 0x%x\n",sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src), fence_seq);- // dummy fence extra (must be static)- static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE };-- ggml_tensor fence_tensor {};- fence_tensor.buffer = &sess_dst->fence_buf->backend_buffer;- fence_tensor.extra = &fence_extra;- fence_tensor.data = (void *) fence_slot;- fence_tensor.type = GGML_TYPE_I32;- fence_tensor.ne[0] = 1;- fence_tensor.ne[1] = 1;- fence_tensor.ne[2] = 1;- fence_tensor.ne[3] = 1;- fence_tensor.nb[0] = sizeof(int32_t);- fence_tensor.nb[1] = sizeof(int32_t);- fence_tensor.nb[2] = sizeof(int32_t);- fence_tensor.nb[3] = sizeof(int32_t);- fence_tensor.op = GGML_OP_NONE;+ ggml_tensor fence_tensor = sess_dst->make_fence_tensor(fence_slot);sess_src->enqueue_cpy(src, dst, &fence_tensor, fence_seq);sess_dst->enqueue_fence(&fence_tensor, fence_seq, /* wait = */ true);@@ -7409,9 +2712,9 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, ggggml_backend_buffer_init_tensor(dst->buffer, dst);}- auto * dst_extra = static_cast<ggml_hexagon_tensor_extra *>(dst->extra);- const auto * src_extra = static_cast<const ggml_hexagon_tensor_extra *>(src->extra);- dst_extra->flags = src_extra->flags & ~GGML_HEXAGON_TENSOR_FUSEABLE;+ auto * dst_extra = static_cast<htp_tensor_extra *>(dst->extra);+ const auto * src_extra = static_cast<const htp_tensor_extra *>(src->extra);+ dst_extra->flags = src_extra->flags & ~HTP_TENSOR_FUSEABLE;auto sess_src = static_cast<ggml_hexagon_session *>(backend_src->context);auto sess_dst = static_cast<ggml_hexagon_session *>(backend_dst->context);@@ -7433,24 +2736,10 @@ static ggml_backend_event_t ggml_backend_hexagon_device_event_new(ggml_backend_dauto sess = dev_ctx->session();ggml_hexagon_event * hex_event = new ggml_hexagon_event();- hex_event->fence_sess = sess;- hex_event->sess = sess;- hex_event->fence_slot = (volatile uint32_t *) sess->alloc_fence(1);-- static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE };- hex_event->fence_tensor.buffer = &sess->fence_buf->backend_buffer;- hex_event->fence_tensor.extra = &fence_extra;- hex_event->fence_tensor.data = (void *) hex_event->fence_slot;- hex_event->fence_tensor.type = GGML_TYPE_I32;- hex_event->fence_tensor.ne[0] = 1;- hex_event->fence_tensor.ne[1] = 1;- hex_event->fence_tensor.ne[2] = 1;- hex_event->fence_tensor.ne[3] = 1;- hex_event->fence_tensor.nb[0] = sizeof(int32_t);- hex_event->fence_tensor.nb[1] = sizeof(int32_t);- hex_event->fence_tensor.nb[2] = sizeof(int32_t);- hex_event->fence_tensor.nb[3] = sizeof(int32_t);- hex_event->fence_tensor.op = GGML_OP_NONE;+ hex_event->fence_sess = sess;+ hex_event->sess = sess;+ hex_event->fence_slot = (volatile uint32_t *) sess->alloc_fence(1);+ hex_event->fence_tensor = sess->make_fence_tensor(hex_event->fence_slot);HEX_VERBOSE("ggml-hex: %s event-new : event %p fence %p\n", ggml_backend_dev_name(dev), (void *)hex_event, (void *)hex_event->fence_slot);@@ -7505,9 +2794,8 @@ static void ggml_backend_hexagon_event_record(ggml_backend_t backend, ggml_backeauto sess = static_cast<ggml_hexagon_session *>(backend->context);auto hex_event = static_cast<ggml_hexagon_event *>(event->context);- if (++sess->fence_seq == 0) sess->fence_seq = 1;hex_event->sess = sess;- hex_event->seq = sess->fence_seq;+ hex_event->seq = sess->next_fence_seq();sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ false);@@ -7686,80 +2974,6 @@ static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_host_buffer_tyreturn &dev_ctx->host_buffer_type;}-static bool ggml_hexagon_supported_cpy(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- struct htp_copy_kernel_params kparams;- return ggml_hexagon_precompute_cpy_params(sess, op, &kparams);-}--static bool ggml_hexagon_supported_cont(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- struct htp_copy_kernel_params kparams;- return ggml_hexagon_precompute_cpy_params(sess, op, &kparams);-}--static bool ggml_hexagon_supported_repeat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- GGML_UNUSED(sess);- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * dst = op;-- // Support f32 and f16- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) return false;-- // src and dst must be the same type- if (src0->type != dst->type) return false;-- // dst dims must be multiples of src dims- if (dst->ne[0] % src0->ne[0] != 0) return false;- if (dst->ne[1] % src0->ne[1] != 0) return false;- if (dst->ne[2] % src0->ne[2] != 0) return false;- if (dst->ne[3] % src0->ne[3] != 0) return false;-- // require contiguous tensors (no transposition)- if (ggml_is_transposed(src0) || ggml_is_transposed(dst)) return false;-- return true;-}--static bool ggml_hexagon_supported_concat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- struct htp_concat_kernel_params kparams;- return ggml_hexagon_precompute_concat_params(sess, op, &kparams);-}--static bool ggml_hexagon_supported_fill(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- const struct ggml_tensor * dst = op;-- if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) {- return false;- }-- return true;- GGML_UNUSED(sess);-}--static bool ggml_hexagon_supported_roll(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {- GGML_UNUSED(sess);-- const struct ggml_tensor * src0 = op->src[0];- const struct ggml_tensor * dst = op;-- if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {- return false;- }-- if (!ggml_are_same_shape(src0, dst)) {- return false;- }-- if (src0->nb[0] != ggml_type_size(src0->type) || dst->nb[0] != ggml_type_size(dst->type)) {- return false;- }-- if (!ggml_is_contiguous(dst)) {- return false;- }-- return true;-}-// IMPORTANT: supports_op runs before buffer allocation and before tensor data is populated.// Never check buffer usage, buffer properties, or repacking state (is_tiled) here.// Support must depend only on static tensor properties (shape, type, strides).@@ -7782,194 +2996,12 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, conssupp = true;break;- case GGML_OP_MUL:- case GGML_OP_ADD:- case GGML_OP_SUB:- case GGML_OP_DIV:- supp = ggml_hexagon_supported_binary(sess, op);- break;-- case GGML_OP_MUL_MAT:- supp = ggml_hexagon_supported_mul_mat(sess, op);- break;-- case GGML_OP_MUL_MAT_ID:- supp = ggml_hexagon_supported_mul_mat_id(sess, op);- break;-- case GGML_OP_ADD_ID:- supp = ggml_hexagon_supported_add_id(sess, op);- break;-- case GGML_OP_NORM:- case GGML_OP_L2_NORM:- case GGML_OP_RMS_NORM:- case GGML_OP_SCALE:- case GGML_OP_CLAMP:- case GGML_OP_LEAKY_RELU:- supp = ggml_hexagon_supported_unary(sess, op);- break;-- case GGML_OP_SQR:- case GGML_OP_SQRT:- case GGML_OP_LOG:- supp = ggml_hexagon_supported_unary(sess, op);- break;-- case GGML_OP_SUM:- supp = ggml_hexagon_supported_sum(sess, op);- break;-- case GGML_OP_SUM_ROWS:- supp = ggml_hexagon_supported_sum_rows(sess, op);- break;-- case GGML_OP_ARGMAX:- supp = ggml_hexagon_supported_argmax(sess, op);- break;-- case GGML_OP_POOL_2D:- supp = ggml_hexagon_supported_pool_2d(sess, op);- break;-- case GGML_OP_POOL_1D:- supp = ggml_hexagon_supported_pool_1d(sess, op);- break;-- case GGML_OP_SOFT_MAX:- supp = ggml_hexagon_supported_softmax(sess, op);- break;-- case GGML_OP_UNARY:- switch (ggml_get_unary_op(op)) {- case GGML_UNARY_OP_NEG:- case GGML_UNARY_OP_EXP:- case GGML_UNARY_OP_SIGMOID:- case GGML_UNARY_OP_SOFTPLUS:- case GGML_UNARY_OP_TANH:- case GGML_UNARY_OP_ABS:- case GGML_UNARY_OP_SILU:- case GGML_UNARY_OP_GELU:- case GGML_UNARY_OP_GELU_QUICK:- case GGML_UNARY_OP_GELU_ERF:- case GGML_UNARY_OP_RELU:- case GGML_UNARY_OP_STEP:- supp = ggml_hexagon_supported_unary(sess, op);- break;- default:- supp = false;- break;- }- break;-- case GGML_OP_GLU:- switch (ggml_get_glu_op(op)) {- case GGML_GLU_OP_SWIGLU:- case GGML_GLU_OP_SWIGLU_OAI:- case GGML_GLU_OP_SWIGLU_CLAMP:- case GGML_GLU_OP_GEGLU:- supp = ggml_hexagon_supported_activations(sess, op);- break;- case GGML_GLU_OP_GEGLU_QUICK:- case GGML_GLU_OP_GEGLU_ERF:- supp = (op->src[0]->type == GGML_TYPE_F32) && ggml_hexagon_supported_activations(sess, op);- break;- default:- supp = false;- break;- }- break;-- case GGML_OP_ROPE:- supp = ggml_hexagon_supported_rope(sess, op);- break;-- case GGML_OP_FLASH_ATTN_EXT:- supp = ggml_hexagon_supported_flash_attn_ext(sess, op);- break;-- case GGML_OP_SET_ROWS:- supp = ggml_hexagon_supported_set_rows(sess, op);- break;-- case GGML_OP_GET_ROWS:- supp = ggml_hexagon_supported_get_rows(sess, op);- break;-- case GGML_OP_DUP:- case GGML_OP_CPY:- supp = ggml_hexagon_supported_cpy(sess, op);- break;-- case GGML_OP_CONT:- supp = ggml_hexagon_supported_cont(sess, op);- break;-- case GGML_OP_REPEAT:- supp = ggml_hexagon_supported_repeat(sess, op);- break;-- case GGML_OP_ARGSORT:- supp = ggml_hexagon_supported_argsort(sess, op);- break;-- case GGML_OP_TOP_K:- supp = ggml_hexagon_supported_top_k(sess, op);- break;-- case GGML_OP_SSM_CONV:- supp = ggml_hexagon_supported_ssm_conv(sess, op);- break;-- case GGML_OP_IM2COL:- supp = ggml_hexagon_supported_im2col(sess, op);- break;-- case GGML_OP_IM2COL_3D:- supp = ggml_hexagon_supported_im2col_3d(sess, op);- break;-- case GGML_OP_GATED_DELTA_NET:- supp = ggml_hexagon_supported_gated_delta_net(sess, op);- break;-- case GGML_OP_CUMSUM:- supp = ggml_hexagon_supported_cumsum(sess, op);- break;-- case GGML_OP_CONCAT:- supp = ggml_hexagon_supported_concat(sess, op);- break;-- case GGML_OP_FILL:- supp = ggml_hexagon_supported_fill(sess, op);- break;-- case GGML_OP_DIAG:- supp = ggml_hexagon_supported_diag(sess, op);- break;-- case GGML_OP_SOLVE_TRI:- supp = ggml_hexagon_supported_solve_tri(sess, op);- break;-- case GGML_OP_TRI:- supp = ggml_hexagon_supported_tri(sess, op);- break;-- case GGML_OP_PAD:- supp = ggml_hexagon_supported_pad(sess, op);- break;-- case GGML_OP_ROLL:- supp = ggml_hexagon_supported_roll(sess, op);- break;-default:+ supp = htp_kparams_check(sess, op);break;}- ggml_hexagon_dump_op_supp(sess->name, op, supp);+ ggml_hexagon_dump_op_supp(sess->c_name(), op, supp);return supp;}@@ -8086,24 +3118,10 @@ static void * ggml_backend_hexagon_comm_init(ggml_backend_t * backends, size_t nctx->backends.assign(backends, backends + n_backends);ctx->n_backends = n_backends;- static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE };for (size_t i = 0; i < n_backends; i++) {auto sess_i = static_cast<ggml_hexagon_session *>(backends[i]->context);- ctx->fence_slots[i] = (volatile uint32_t *) sess_i->alloc_fence(1);- ctx->fence_tensors[i] = {};- ctx->fence_tensors[i].buffer = &sess_i->fence_buf->backend_buffer;- ctx->fence_tensors[i].extra = &fence_extra;- ctx->fence_tensors[i].data = (void *) ctx->fence_slots[i];- ctx->fence_tensors[i].type = GGML_TYPE_I32;- ctx->fence_tensors[i].ne[0] = 4;- ctx->fence_tensors[i].ne[1] = 1;- ctx->fence_tensors[i].ne[2] = 1;- ctx->fence_tensors[i].ne[3] = 1;- ctx->fence_tensors[i].nb[0] = sizeof(int32_t);- ctx->fence_tensors[i].nb[1] = sizeof(int32_t);- ctx->fence_tensors[i].nb[2] = sizeof(int32_t);- ctx->fence_tensors[i].nb[3] = sizeof(int32_t);- ctx->fence_tensors[i].op = GGML_OP_NONE;+ ctx->fence_slots[i] = (volatile uint32_t *) sess_i->alloc_fence(1);+ ctx->fence_tensors[i] = sess_i->make_fence_tensor(ctx->fence_slots[i]);}return ctx;@@ -8157,8 +3175,7 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, structfor (size_t r = 0; r < n_backends; r++) {auto sess = static_cast<ggml_hexagon_session *>(comm_ctx->backends[r]->context);- struct htp_allreduce_kernel_params kparams;- if (!ggml_hexagon_precompute_allreduce_params(sess, tensors[r], (uint32_t) r, (uint32_t) n_backends, false, false, /*is_shard_ok=*/ false, &kparams)) {+ if (!htp_kparams_check<htp_kparams_solve_allreduce>(sess, tensors[r], (uint32_t) r, (uint32_t) n_backends, false, false, false)) {return false;}}@@ -8423,7 +3440,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {opt_verbose = str_verbose ? atoi(str_verbose) : 0;opt_opbatch = str_opbatch ? strtoul(str_opbatch, NULL, 0) : opt_opbatch;opt_opqueue = str_opqueue ? strtoul(str_opqueue, NULL, 0) : opt_opqueue;- opt_optrace = str_optrace ? strtoul(str_optrace, NULL, 0) : (opt_opbatch * 256);+ opt_optrace = str_optrace ? strtoul(str_optrace, NULL, 0) : (opt_opbatch * HTP_TRACE_MAX_EVENTS + 131072);opt_oppoll = str_oppoll ? strtoul(str_oppoll, NULL, 0) : opt_oppoll;opt_opfusion = str_opfusion ? atoi(str_opfusion) : opt_opfusion;opt_rt_repack = str_rt_repack ? atoi(str_rt_repack) : opt_rt_repack;diff --git a/ggml/src/ggml-hexagon/htp-common.h b/ggml/src/ggml-hexagon/htp-common.hnew file mode 100644index 0000000000..71a690a168--- /dev/null+++ b/ggml/src/ggml-hexagon/htp-common.h@@ -0,0 +1,106 @@+#ifndef HTP_COMMON_H+#define HTP_COMMON_H++#include "htp-opnode.h"+#include "htp-repack.h"++struct htp_tensor_extra {+ std::vector<uint8_t> shadow_buf;+ size_t shadow_size { 0 };+ uint32_t flags { 0 };+};++extern int opt_arch;+extern size_t opt_ndev;+extern size_t opt_nhvx;+extern int opt_nhmx;+extern size_t opt_vmem;+extern int opt_etm;+extern int opt_verbose;+extern int opt_profile;+extern bool opt_hostbuf;+extern bool opt_dma64;+extern int opt_power_mode;++extern size_t opt_mbuf_dyn;+extern size_t opt_mbuf_static;+extern size_t opt_mbuf_total;++extern int opt_mm_select;+extern int opt_fa_select;+extern int opt_fa_head_split;+extern int opt_gdn_select;+extern int opt_ar_select;+extern int opt_ar_scatter;++extern int opt_opbatch;+extern int opt_opqueue;+extern int opt_optrace;+extern int opt_oppoll;+extern int opt_opfusion;+extern int opt_rt_repack;++#ifndef HEX_VERBOSE+#define HEX_VERBOSE(...) \+ if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__)+#endif++static inline bool htp_tensor_is_fuseable(const struct ggml_tensor * t) {+ if (!t || !t->extra) return false;+ const auto * extra = (const struct htp_tensor_extra *) t->extra;+ return (extra->flags & HTP_TENSOR_FUSEABLE) != 0;+}++static inline bool htp_can_rt_repack(const struct ggml_tensor * t) {+ if (opt_rt_repack == 0 || !htp_is_tiled_weight_tensor(t)) {+ return false;+ }+ if (t->buffer && ggml_backend_buft_is_host(t->buffer->buft)) {+ return false;+ }+ return opt_rt_repack > 1 || ggml_is_quantized(t->type);+}++static inline bool htp_tensor_is_tiled(const struct ggml_tensor * t) {+ const auto * extra = (const struct htp_tensor_extra *) t->extra;+ if (extra && (extra->flags & HTP_TENSOR_TILED)) {+ return true;+ }+ return htp_can_rt_repack(t);+}++static inline bool htp_needs_repack(const struct ggml_tensor * t) {+ const auto * extra = (const struct htp_tensor_extra *) t->extra;+ if (extra && (extra->flags & HTP_TENSOR_TILED)) {+ return false;+ }+ return htp_can_rt_repack(t);+}++static inline bool htp_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) {+ const uintptr_t a0 = (uintptr_t) a->data;+ const uintptr_t b0 = (uintptr_t) b->data;+ const uintptr_t a1 = a0 + ggml_nbytes(a);+ const uintptr_t b1 = b0 + ggml_nbytes(b);++ return a0 < b1 && b0 < a1;+}++static inline bool htp_can_row_partition(const struct ggml_tensor * t) {+ if (t->ne[1] > 1 && (t->nb[1] & 127) != 0) return false;+ if (t->ne[2] > 1 && (t->nb[2] & 127) != 0) return false;+ if (t->ne[3] > 1 && (t->nb[3] & 127) != 0) return false;+ return true;+}++struct ggml_hexagon_session;++const char * htp_session_name(const struct ggml_hexagon_session * sess);+size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess);+uint32_t htp_session_n_threads(const struct ggml_hexagon_session * sess);+uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess);+uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess);++htp_op_code htp_op_remap(const struct ggml_tensor * t);++#endif // HTP_COMMON_Hdiff --git a/ggml/src/ggml-hexagon/htp-fusion.cpp b/ggml/src/ggml-hexagon/htp-fusion.cppnew file mode 100644index 0000000000..65aacf73f6--- /dev/null+++ b/ggml/src/ggml-hexagon/htp-fusion.cpp@@ -0,0 +1,650 @@+#include "htp-fusion.h"+#include "htp-kparams.h"+#include "ggml-impl.h"++#include <algorithm>+#include <cstring>++static inline bool htp_same_tensor_data(const ggml_tensor * a, const ggml_tensor * b) {+ return a && b && (a == b || a->data == b->data);+}++static inline bool htp_same_shape_and_contiguity(const ggml_tensor * a, const ggml_tensor * b) {+ return ggml_are_same_shape(a, b) && (ggml_is_contiguous(a) == ggml_is_contiguous(b));+}++static inline bool htp_inplace_overlap_unsafe(const ggml_tensor * dst, const ggml_tensor * in) {+ return htp_tensors_overlap(dst, in) && (dst->data != in->data || !ggml_are_same_stride(dst, in));+}++static inline bool htp_inputs_overlap_unsafe(const htp_opnode & prev, const ggml_tensor * dst, const ggml_tensor * extra = nullptr) {+ if (extra && htp_inplace_overlap_unsafe(dst, extra)) return true;+ for (const auto * in : prev.inputs) {+ if (htp_inplace_overlap_unsafe(dst, in)) return true;+ }+ return false;+}++static inline bool htp_match_consumer(+ const htp_opnode & prev,+ const htp_opnode & cur,+ bool commutative,+ const ggml_tensor ** other_src = nullptr+) {+ if (prev.outputs.size() != 1) {+ return false;+ }+ const ggml_tensor * prev_out = prev.dst();+ if (!htp_tensor_is_fuseable(prev_out)) {+ return false;+ }+ if (htp_same_tensor_data(cur.src0(), prev_out)) {+ if (other_src) *other_src = cur.src1();+ return true;+ }+ if (commutative && htp_same_tensor_data(cur.src1(), prev_out)) {+ if (other_src) *other_src = cur.src0();+ return true;+ }+ return false;+}++static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) {+ if (kparams->n_hmx) {+ return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;+ }+ return htp_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW);+}++static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) {+ if (kparams->n_hmx) {+ return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;+ }+ return htp_tensor_is_tiled(src0) &&+ (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK);+}++// --- Fusion rules ---++static bool fuse_allreduce_add(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {+ if (opt_ar_select != 2 || prev.opcode != HTP_OP_ALLREDUCE) return false;++ const auto * ar_kparams = prev.kparams<htp_allreduce_kernel_params>();+ const uint32_t rank = (uint32_t) ar_kparams->rank;+ const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks;++ const ggml_tensor * ar_local = prev.inputs[rank];+ if (!htp_tensor_is_fuseable(ar_local)) return false;++ const ggml_tensor * add_dst = cur.dst();+ const ggml_tensor * res_tensor = nullptr;+ if (htp_same_tensor_data(cur.src0(), ar_local)) {+ res_tensor = cur.src1();+ } else if (htp_same_tensor_data(cur.src1(), ar_local)) {+ res_tensor = cur.src0();+ } else {+ return false;+ }++ if (ar_local->type != res_tensor->type) return false;++ const bool is_same_shape = ggml_are_same_shape(ar_local, res_tensor);+ const bool is_row_bcast = !is_same_shape && (ar_local->ne[0] == res_tensor->ne[0] &&+ res_tensor->ne[1] == 1 && res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1);+ if (!is_same_shape && !is_row_bcast) return false;++ if (is_same_shape && !ggml_are_same_stride(ar_local, res_tensor)) {+ return false;+ }+ if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(add_dst)) return false;++ const bool is_shard_ok = n_ranks <= HTP_OP_MAX_OUTPUTS && add_dst->data == ar_local->data;++ struct htp_allreduce_kernel_params kparams;+ if (!htp_kparams_solve_allreduce(sess, add_dst, rank, n_ranks, true, is_row_bcast, is_shard_ok, &kparams)) {+ HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: solver failed\n", htp_session_name(sess));+ return false;+ }++ const bool scatter_ok = kparams.n_dsts > 1;+ kparams.mode = scatter_ok ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL;++ for (uint32_t r = 0; r < n_ranks; r++) {+ const ggml_tensor * ar_src = prev.inputs[r];+ if (!htp_tensors_overlap(add_dst, ar_src)) continue;+ if (scatter_ok && r == rank) continue;+ HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: dst overlaps allreduce src %u\n", htp_session_name(sess), r);+ return false;+ }++ prev.opcode = HTP_OP_ALLREDUCE_ADD;+ prev.inputs.push_back(res_tensor);+ if (kparams.mode != HTP_ALLREDUCE_SHARDED_FANOUT) {+ prev.outputs = { add_dst };+ }+ prev.fused.push_back(cur.node);+ prev.set_kparams(kparams);+ return true;+}++static bool fuse_rms_norm_mul(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {+ if (prev.opcode != HTP_OP_RMS_NORM) return false;++ const ggml_tensor * weight = nullptr;+ if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &weight)) return false;++ const ggml_tensor * src0 = prev.src0();+ const ggml_tensor * dst = cur.dst();+ if (src0->type != GGML_TYPE_F32 || src0->ne[0] != weight->ne[0] || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false;+ if (htp_inputs_overlap_unsafe(prev, dst, weight)) return false;++ const bool is_row_bcast = (weight->ne[1] == 1 && weight->ne[2] == 1 && weight->ne[3] == 1);+ const bool is_same_shape = ggml_are_same_shape(src0, weight);+ if (!is_row_bcast && !is_same_shape) return false;++ struct htp_unary_kernel_params kparams;+ if (!htp_kparams_solve_rms_norm_mul(sess, src0, weight, dst, &kparams)) {+ HEX_VERBOSE("ggml-hex: %s skip RMS_NORM_MUL fusion: VTCM needed (%d) > budget (%zu)\n",+ htp_session_name(sess), kparams.vtcm_size, htp_session_vtcm_size(sess));+ return false;+ }++ prev.opcode = HTP_OP_RMS_NORM_MUL;+ prev.inputs = { src0, weight };+ prev.outputs = { dst };+ prev.fused.push_back(cur.node);+ prev.set_kparams(kparams);+ return true;+}++static bool fuse_rms_norm_scale(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {+ GGML_UNUSED(sess);+ if (prev.opcode != HTP_OP_RMS_NORM && prev.opcode != HTP_OP_RMS_NORM_MUL) return false;+ if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false;++ const ggml_tensor * src0 = prev.src0();+ const ggml_tensor * dst = cur.dst();+ if (src0->type != GGML_TYPE_F32 || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false;+ if (htp_inputs_overlap_unsafe(prev, dst)) return false;++ float scale = 0.0f;+ float bias = 0.0f;+ memcpy(&scale, (const float *) cur.params + 0, sizeof(float));+ memcpy(&bias, (const float *) cur.params + 1, sizeof(float));+ if (bias != 0.0f || scale == 0.0f) return false;++ float prev_scale = 0.0f;+ memcpy(&prev_scale, &prev.params[1], sizeof(float));+ if (prev_scale != 0.0f) {+ scale *= prev_scale;+ }+ memcpy(&prev.params[1], &scale, sizeof(float));++ prev.outputs[0] = dst;+ prev.fused.push_back(cur.node);+ return true;+}++static bool fuse_mul_mat_add(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {+ if (prev.opcode != HTP_OP_MUL_MAT) return false;++ const ggml_tensor * src2 = nullptr;+ if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &src2)) return false;+ if (src2->type != GGML_TYPE_F32) return false;++ const ggml_tensor * src0 = prev.src0();+ const ggml_tensor * src1 = prev.src1();+ const ggml_tensor * dst = cur.dst();+ if (!ggml_are_same_shape(prev.dst(), dst)) return false;++ const auto * orig_kparams = prev.kparams<htp_mm_kernel_params>();+ struct htp_mm_kernel_params kparams;+ if (!htp_kparams_solve_matmul_add(sess, src0, src1, src2, dst, &kparams) ||+ orig_kparams->collapse != kparams.collapse) {+ return false;+ }++ const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3];+ if (kparams.n_hmx == 0 && src1_nrows != 1) return false;++ if (kparams.n_hmx > 0 && orig_kparams->n_hmx > 0) {+ if (kparams.m_chunk < orig_kparams->m_chunk ||+ kparams.n_chunk < orig_kparams->n_chunk ||+ kparams.n_act_threads < orig_kparams->n_act_threads) {+ HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, th %d->%d)\n",+ htp_session_name(sess), orig_kparams->m_chunk, kparams.m_chunk,+ orig_kparams->n_chunk, kparams.n_chunk,+ orig_kparams->n_act_threads, kparams.n_act_threads);+ return false;+ }+ }++ prev.opcode = HTP_OP_MUL_MAT_ADD;+ prev.inputs = { src0, src1, src2 };+ prev.outputs = { dst };+ prev.fused.push_back(cur.node);+ prev.set_kparams(kparams);+ return true;+}++static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {+ if (prev.opcode != HTP_OP_MUL_MAT && prev.opcode != HTP_OP_MUL_MAT_NX) return false;++ const uint32_t curr_n = (uint32_t) prev.outputs.size();+ if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 1 >= HTP_OP_MAX_INPUTS) return false;++ const ggml_tensor * w0 = prev.inputs[0];+ const ggml_tensor * x = prev.inputs[curr_n];+ const ggml_tensor * w_in = cur.src0();+ const ggml_tensor * x_in = cur.src1();+ const ggml_tensor * d_in = cur.dst();++ auto * orig_kparams = prev.kparams<htp_mm_kernel_params>();+ const auto * cur_kparams = cur.kparams<htp_mm_kernel_params>();++ if (x_in != x || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] ||+ w0->ne[2] != 1 || w0->ne[3] != 1 || w_in->ne[2] != 1 || w_in->ne[3] != 1 ||+ htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) ||+ orig_kparams->n_hmx != cur_kparams->n_hmx) {+ return false;+ }++ const bool collapse = orig_kparams->collapse && htp_matmul_can_collapse(w_in, x, d_in);+ if ((x->ne[2] > 1 || x->ne[3] > 1) && !collapse) return false;++ if (prev.opcode == HTP_OP_MUL_MAT_NX) {+ orig_kparams->n_weights = curr_n + 1;+ } else {+ struct htp_mm_kernel_params kparams;+ if (collapse) {+ const ggml_tensor x_collapsed = htp_tensor_collapse_rows(x);+ if (!htp_kparams_solve_mmnx(sess, w0, &x_collapsed, curr_n + 1, &kparams)) return false;+ kparams.collapse = 1;+ } else if (!htp_kparams_solve_mmnx(sess, w0, x, curr_n + 1, &kparams)) {+ return false;+ }+ if (!is_supported_mul_mat_nx_kernel(w0, &kparams)) return false;++ prev.set_kparams(kparams);+ prev.opcode = HTP_OP_MUL_MAT_NX;+ }++ prev.inputs.insert(prev.inputs.begin() + curr_n, w_in);+ prev.outputs.push_back(d_in);+ prev.fused.push_back(cur.node);+ return true;+}++static bool fuse_mul_mat_id_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {+ if (prev.opcode != HTP_OP_MUL_MAT_ID && prev.opcode != HTP_OP_MUL_MAT_ID_NX) return false;++ const uint32_t curr_n = (uint32_t) prev.outputs.size();+ if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 2 >= HTP_OP_MAX_INPUTS) return false;++ const ggml_tensor * w0 = prev.inputs[0];+ const ggml_tensor * x = prev.inputs[curr_n];+ const ggml_tensor * ids = prev.inputs[curr_n + 1];+ const ggml_tensor * w_in = cur.src0();+ const ggml_tensor * x_in = cur.src1();+ const ggml_tensor * ids_in = cur.node->src[2];+ const ggml_tensor * d_in = cur.dst();++ auto * orig_kparams = prev.kparams<htp_mm_kernel_params>();+ const auto * cur_kparams = cur.kparams<htp_mm_kernel_params>();++ if (x_in != x || ids_in != ids || w_in->type != w0->type ||+ w_in->ne[0] != w0->ne[0] || w_in->ne[2] != w0->ne[2] ||+ htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) ||+ orig_kparams->n_hmx != cur_kparams->n_hmx) {+ return false;+ }++ struct htp_mm_kernel_params kparams;+ if (d_in->nb[1] == prev.node->nb[1]) {+ kparams = *orig_kparams;+ kparams.n_weights = curr_n + 1;+ } else if (!htp_kparams_solve_mmidnx(sess, w0, x, d_in, curr_n + 1, &kparams)) {+ return false;+ }+ if (!is_supported_mul_mat_id_nx_kernel(w0, &kparams)) return false;++ prev.opcode = HTP_OP_MUL_MAT_ID_NX;+ prev.inputs.insert(prev.inputs.begin() + curr_n, w_in);+ prev.outputs.push_back(d_in);+ prev.fused.push_back(cur.node);+ prev.set_kparams(kparams);+ return true;+}++static bool fuse_gdn_cpy(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {+ GGML_UNUSED(sess);+ if (prev.opcode != HTP_OP_GATED_DELTA_NET || prev.outputs.size() != 1) return false;++ const ggml_tensor * gdn_out = prev.dst();+ const ggml_tensor * cpy_node = cur.node;+ const ggml_tensor * cpy_src = cur.src0();+ const ggml_tensor * cpy_dst = cur.dst();++ if (gdn_out->type != GGML_TYPE_F32 || cpy_src->type != GGML_TYPE_F32 || cpy_dst->type != GGML_TYPE_F32) return false;+ if ((gdn_out->flags & GGML_TENSOR_FLAG_OUTPUT) || (cpy_node->flags & GGML_TENSOR_FLAG_OUTPUT)) return false;++ const ggml_tensor * v = prev.node->src[2];+ if (!v) return false;++ const int64_t S_v = v->ne[0];+ const int64_t H = v->ne[1];+ const int64_t n_tokens = v->ne[2];+ const int64_t n_seqs = v->ne[3];+ const int64_t K = ggml_get_op_params_i32(prev.node, 0);+ const size_t tail_off = (size_t) S_v * H * n_tokens * n_seqs * sizeof(float);+ const int64_t D = S_v * S_v * H;+ const int64_t n_written = std::min<int64_t>(n_tokens, K);++ if (cpy_src->op != GGML_OP_VIEW || !htp_same_tensor_data(cpy_src->view_src, gdn_out) ||+ cpy_src->view_offs != tail_off || !ggml_is_contiguous(cpy_src)) {+ return false;+ }++ if (cpy_dst->ne[0] != D || cpy_dst->ne[1] != n_seqs || cpy_dst->ne[2] != n_written || !ggml_is_contiguous(cpy_dst)) {+ return false;+ }++ prev.outputs.push_back(cpy_dst);+ prev.fused.push_back(cur.node);+ return true;+}++static bool fuse_ssm_conv_silu(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {+ GGML_UNUSED(sess);+ if (prev.opcode != HTP_OP_SSM_CONV) return false;++ auto * kparams = prev.kparams<htp_ssm_conv_kernel_params>();+ if (kparams->silu) return false;+ if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false;++ const ggml_tensor * conv_dst = prev.dst();+ const ggml_tensor * dst = cur.dst();+ if (!ggml_are_same_layout(conv_dst, dst)) return false;+ if (htp_tensors_overlap(dst, prev.src0()) || htp_tensors_overlap(dst, prev.src1())) return false;++ kparams->silu = 1;+ prev.outputs[0] = dst;+ prev.fused.push_back(cur.node);+ return true;+}++static bool fuse_repeat_get_rows(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {+ if (prev.opcode != HTP_OP_REPEAT) return false;+ if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false;++ const ggml_tensor * repeat_out = prev.dst();+ const ggml_tensor * repeat_src0 = prev.src0();+ const ggml_tensor * gr_src1 = cur.src1();+ const ggml_tensor * gr_dst = cur.dst();++ if (repeat_src0->ne[0] != repeat_out->ne[0] || repeat_src0->ne[1] != repeat_out->ne[1]) return false;+ if (repeat_src0->type != repeat_out->type) return false;++ struct htp_get_rows_kernel_params kparams;+ if (!htp_kparams_solve_repeat_get_rows(sess, repeat_src0, gr_src1, gr_dst, &kparams) || kparams.vtcm_size == 0) return false;++ prev.opcode = HTP_OP_GET_ROWS;+ prev.inputs = { repeat_src0, gr_src1 };+ prev.outputs = { gr_dst };+ prev.fused.push_back(cur.node);+ prev.set_kparams(kparams);+ return true;+}++static bool fuse_unary_mul_glu(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {+ htp_op_code glu_op = HTP_OP_INVALID;+ switch (prev.opcode) {+ case HTP_OP_UNARY_SILU: glu_op = HTP_OP_GLU_SWIGLU; break;+ case HTP_OP_UNARY_GELU: glu_op = HTP_OP_GLU_GEGLU; break;+ case HTP_OP_UNARY_GELU_QUICK: glu_op = HTP_OP_GLU_GEGLU_QUICK; break;+ case HTP_OP_UNARY_GELU_ERF: glu_op = HTP_OP_GLU_GEGLU_ERF; break;+ case HTP_OP_UNARY_RELU: glu_op = HTP_OP_GLU_REGLU; break;+ case HTP_OP_UNARY_SIGMOID: glu_op = HTP_OP_GLU_SIGMOID; break;+ default: return false;+ }++ const ggml_tensor * up = nullptr;+ if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &up)) return false;++ const ggml_tensor * gate = prev.src0();+ const ggml_tensor * dst = cur.dst();+ if (htp_inputs_overlap_unsafe(prev, dst, up)) return false;++ struct htp_glu_kernel_params kparams;+ if (!htp_kparams_solve_unary_mul_glu(sess, glu_op, gate, up, dst, &kparams)) return false;++ prev.opcode = glu_op;+ prev.inputs = { gate, up };+ prev.outputs = { dst };+ prev.fused.push_back(cur.node);+ memset(prev.params, 0, sizeof(prev.params));+ prev.set_kparams(kparams);+ return true;+}++static bool fuse_mul_mul_scalar(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {+ if (prev.opcode != HTP_OP_MUL || prev.inputs.size() != 2) return false;++ const auto * k0 = prev.kparams<htp_binary_kernel_params>();+ const auto * k1 = cur.kparams<htp_binary_kernel_params>();+ if (k0->kernel_type != HTP_BINARY_KERNEL_SCALAR || k1->kernel_type != HTP_BINARY_KERNEL_SCALAR) return false;+ if (k0->src1_strided || k1->src1_strided) return false;++ const ggml_tensor * src2 = nullptr;+ if (!htp_match_consumer(prev, cur, /*commutative=*/ false, &src2)) return false;++ const ggml_tensor * src1 = prev.src1();+ const ggml_tensor * dst = cur.dst();+ if (!ggml_are_same_shape(src1, src2) || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false;+ if (htp_inputs_overlap_unsafe(prev, dst, src2)) return false;++ const size_t vtcm_size = htp_session_vtcm_size(sess);+ struct htp_binary_kernel_params kparams = *k0;+ kparams.src2_size = kparams.src1_size;+ kparams.rows_per_buffer = 0;+ if (kparams.src1_size > 65536 || kparams.src1_size + kparams.src2_size > vtcm_size) return false;++ struct htp_binary_vtcm_layout L;+ htp_binary_vtcm_layout_build(&L, &kparams, vtcm_size);+ if (L.rows_per_buffer == 0 || L.total_bytes > vtcm_size) return false;++ kparams.rows_per_buffer = L.rows_per_buffer;+ kparams.vtcm_size = L.total_bytes;++ prev.inputs.push_back(src2);+ prev.outputs[0] = dst;+ prev.fused.push_back(cur.node);+ prev.set_kparams(kparams);+ return true;+}++static bool fuse_binary_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {+ if (prev.opcode != cur.opcode) return false;++ const uint32_t curr_n = (uint32_t) prev.inputs.size();+ if (curr_n < 2 || curr_n >= HTP_OP_MAX_INPUTS) return false;++ const auto * k0 = prev.kparams<htp_binary_kernel_params>();+ const auto * k1 = cur.kparams<htp_binary_kernel_params>();+ const bool k0_ok = (k0->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) ||+ (k0->kernel_type == HTP_BINARY_KERNEL_CHUNKED && !k0->is_scalar);+ const bool k1_ok = (k1->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) ||+ (k1->kernel_type == HTP_BINARY_KERNEL_CHUNKED && !k1->is_scalar);+ if (!k0_ok || !k1_ok) return false;++ const bool is_comm = (cur.opcode == HTP_OP_ADD || cur.opcode == HTP_OP_MUL);+ const ggml_tensor * new_src = nullptr;+ if (!htp_match_consumer(prev, cur, is_comm, &new_src)) return false;++ const ggml_tensor * src0 = prev.inputs[0];+ const ggml_tensor * dst = cur.dst();+ if (!ggml_are_same_shape(src0, prev.inputs[1]) ||+ !ggml_are_same_shape(src0, new_src) ||+ !htp_same_shape_and_contiguity(prev.dst(), dst)) {+ return false;+ }+ if (htp_inputs_overlap_unsafe(prev, dst, new_src)) return false;++ const size_t vtcm_size = htp_session_vtcm_size(sess);+ struct htp_binary_kernel_params kparams = *k0;+ struct htp_binary_vtcm_layout L;+ memset(&L, 0, sizeof(L));++ const uint32_t n_srcs = curr_n + 1;+ if (k0->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE || k1->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {+ if (k0->kernel_type != HTP_BINARY_KERNEL_SAME_SHAPE) {+ kparams = *k1;+ }+ kparams.n_srcs = (uint8_t) n_srcs;+ kparams.rows_per_buffer = 0;++ auto check_split = [&](const ggml_tensor * t) {+ if (src0->ne[2] > 1 && t->nb[2] != t->ne[1] * t->nb[1]) kparams.split_at_ne01 = 1;+ if (src0->ne[3] > 1 && t->nb[3] != t->ne[2] * t->nb[2]) kparams.split_at_ne02 = 1;+ };+ for (const auto * in : prev.inputs) check_split(in);+ check_split(new_src);+ check_split(dst);++ htp_binary_vtcm_layout_build(&L, &kparams, vtcm_size);+ }++ if (L.rows_per_buffer == 0 || L.total_bytes == 0 || L.total_bytes > vtcm_size) {+ bool all_contig = ggml_is_contiguous(new_src) && ggml_is_contiguous(dst);+ for (const auto * in : prev.inputs) {+ all_contig = all_contig && ggml_is_contiguous(in);+ }+ if (!all_contig) return false;++ const uint32_t total_elems = (uint32_t) ggml_nelements(src0);+ const uint32_t elem_size = (uint32_t) ggml_type_size(src0->type);+ const uint32_t max_threads = htp_session_n_threads(sess);+ if (!htp_binary_solve_chunked(&L, &kparams, total_elems, elem_size, max_threads, n_srcs, false, vtcm_size)) {+ return false;+ }+ }++ kparams.rows_per_buffer = L.rows_per_buffer;+ kparams.vtcm_size = (uint32_t) L.total_bytes;++ prev.inputs.push_back(new_src);+ prev.outputs[0] = dst;+ prev.fused.push_back(cur.node);+ prev.set_kparams(kparams);+ return true;+}++bool htp_fusion_can_stack(const struct ggml_tensor * a) {+ if (!a || !a->src[0] || !a->src[1]) return false;+ switch (a->op) {+ case GGML_OP_MUL_MAT:+ if (!htp_fusion_enabled(HTP_FUSE_MUL_MAT_NX)) return false;+ if (htp_is_hadamard(a)) return false;+ if (a->src[0]->ne[2] != 1 || a->src[0]->ne[3] != 1) return false;+ return ggml_is_quantized(a->src[0]->type);+ case GGML_OP_MUL_MAT_ID:+ if (!htp_fusion_enabled(HTP_FUSE_MUL_MAT_ID_NX)) return false;+ return a->src[2] != nullptr && ggml_is_quantized(a->src[0]->type);+ default:+ return false;+ }+}++bool htp_fusion_can_stack(const struct ggml_tensor * a, const struct ggml_tensor * b) {+ if (!htp_fusion_can_stack(b) || a->op != b->op || a->src[1] != b->src[1]) {+ return false;+ }+ if (a->op == GGML_OP_MUL_MAT_ID && a->src[2] != b->src[2]) {+ return false;+ }+ const ggml_tensor * w0 = a->src[0];+ const ggml_tensor * w1 = b->src[0];+ return w0->type == w1->type && w0->ne[0] == w1->ne[0] && w0->ne[2] == w1->ne[2];+}++int htp_fusion_max(const struct ggml_cgraph * graph, int idx) {+ const int n = graph->n_nodes;+ const ggml_op op0 = graph->nodes[idx]->op;+ if (op0 != GGML_OP_ADD && op0 != GGML_OP_NORM && op0 != GGML_OP_RMS_NORM) {+ return 1;+ }++ enum ggml_op ops[HTP_OP_MAX_FUSED];+ ops[0] = op0;++ int f = idx + 1;+ while (f < n && f < idx + HTP_OP_MAX_FUSED) {+ const ggml_op op = graph->nodes[f]->op;+ if (op != GGML_OP_ADD && op != GGML_OP_MUL && op != GGML_OP_SCALE &&+ op != GGML_OP_NORM && op != GGML_OP_RMS_NORM) {+ break;+ }+ ops[f - idx] = op;+ f++;+ }++ f -= idx;+ for (; f > 1; f--) {+ if (ggml_can_fuse(graph, idx, ops, f)) {+ break;+ }+ }+ return f;+}++void htp_fusion_tag_graph(struct ggml_cgraph * graph) {+ for (int i = 0; i < graph->n_nodes; i++) {+ auto * extra = (htp_tensor_extra *) graph->nodes[i]->extra;+ if (!extra) continue;++ extra->flags &= ~HTP_TENSOR_FUSEABLE;++ const ggml_op op = graph->nodes[i]->op;+ if ((op == GGML_OP_RMS_NORM || op == GGML_OP_SCALE || op == GGML_OP_REPEAT ||+ op == GGML_OP_UNARY || op == GGML_OP_ADD || op == GGML_OP_SUB ||+ op == GGML_OP_MUL || op == GGML_OP_DIV || op == GGML_OP_MUL_MAT_ID ||+ op == GGML_OP_SSM_CONV ||+ (op == GGML_OP_MUL_MAT && !htp_is_hadamard(graph->nodes[i]))) &&+ ggml_node_has_n_uses(graph, i, 1)) {+ extra->flags |= HTP_TENSOR_FUSEABLE;+ }+ }+}++bool htp_fusion_try_fuse(const struct ggml_hexagon_session * sess, struct htp_opnode & prev, const struct htp_opnode & cur) {+ if (!opt_opfusion) return false;++ switch (cur.opcode) {+ case HTP_OP_ADD:+ return (htp_fusion_enabled(HTP_FUSE_ALLREDUCE_ADD) && fuse_allreduce_add(sess, prev, cur)) ||+ (htp_fusion_enabled(HTP_FUSE_MUL_MAT_ADD) && fuse_mul_mat_add(sess, prev, cur)) ||+ (htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur));+ case HTP_OP_MUL:+ return (htp_fusion_enabled(HTP_FUSE_RMS_NORM_MUL) && fuse_rms_norm_mul(sess, prev, cur)) ||+ (htp_fusion_enabled(HTP_FUSE_UNARY_MUL_GLU) && fuse_unary_mul_glu(sess, prev, cur)) ||+ (htp_fusion_enabled(HTP_FUSE_MUL_MUL_SCALAR) && fuse_mul_mul_scalar(sess, prev, cur)) ||+ (htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur));+ case HTP_OP_SUB:+ case HTP_OP_DIV:+ return htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur);+ case HTP_OP_SCALE:+ return htp_fusion_enabled(HTP_FUSE_RMS_NORM_SCALE) && fuse_rms_norm_scale(sess, prev, cur);+ case HTP_OP_UNARY_SILU:+ return htp_fusion_enabled(HTP_FUSE_SSM_CONV_SILU) && fuse_ssm_conv_silu(sess, prev, cur);+ case HTP_OP_GET_ROWS:+ return htp_fusion_enabled(HTP_FUSE_REPEAT_GET_ROWS) && fuse_repeat_get_rows(sess, prev, cur);+ case HTP_OP_CPY:+ return htp_fusion_enabled(HTP_FUSE_GDN_CPY) && fuse_gdn_cpy(sess, prev, cur);+ case HTP_OP_MUL_MAT:+ return htp_fusion_enabled(HTP_FUSE_MUL_MAT_NX) && fuse_mul_mat_nx(sess, prev, cur);+ case HTP_OP_MUL_MAT_ID:+ return htp_fusion_enabled(HTP_FUSE_MUL_MAT_ID_NX) && fuse_mul_mat_id_nx(sess, prev, cur);+ default:+ return false;+ }+}diff --git a/ggml/src/ggml-hexagon/htp-fusion.h b/ggml/src/ggml-hexagon/htp-fusion.hnew file mode 100644index 0000000000..e794c1daa3--- /dev/null+++ b/ggml/src/ggml-hexagon/htp-fusion.h@@ -0,0 +1,33 @@+#ifndef HTP_FUSION_H+#define HTP_FUSION_H++#include "htp-common.h"++enum htp_fusion_flags {+ HTP_FUSE_ALLREDUCE_ADD = (1 << 1), // 2+ HTP_FUSE_RMS_NORM_MUL = (1 << 2), // 4+ HTP_FUSE_MUL_MAT_ADD = (1 << 3), // 8+ HTP_FUSE_MUL_MAT_NX = (1 << 4), // 16+ HTP_FUSE_MUL_MAT_ID_NX = (1 << 5), // 32+ HTP_FUSE_GDN_CPY = (1 << 6), // 64+ HTP_FUSE_REPEAT_GET_ROWS = (1 << 7), // 128+ HTP_FUSE_UNARY_MUL_GLU = (1 << 8), // 256+ HTP_FUSE_MUL_MUL_SCALAR = (1 << 9), // 512+ HTP_FUSE_BINARY_NX = (1 << 10), // 1024+ HTP_FUSE_RMS_NORM_SCALE = (1 << 11), // 2048+ HTP_FUSE_SSM_CONV_SILU = (1 << 12), // 4096+};++static inline bool htp_fusion_enabled(int flag) {+ if (opt_opfusion <= 0) return false;+ if (opt_opfusion == 1) return true; // 1 enables all+ return (opt_opfusion & flag) != 0;+}++bool htp_fusion_can_stack(const struct ggml_tensor * a);+bool htp_fusion_can_stack(const struct ggml_tensor * a, const struct ggml_tensor * b);+int htp_fusion_max(const struct ggml_cgraph * graph, int idx);+void htp_fusion_tag_graph(struct ggml_cgraph * graph);+bool htp_fusion_try_fuse(const struct ggml_hexagon_session * sess, struct htp_opnode & prev, const struct htp_opnode & cur);++#endif // HTP_FUSION_Hdiff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cppnew file mode 100644index 0000000000..e886cd0239--- /dev/null+++ b/ggml/src/ggml-hexagon/htp-kparams.cpp@@ -0,0 +1,3278 @@+#define _USE_MATH_DEFINES++#include "htp-kparams.h"+#include "ggml-impl.h"++#include <algorithm>+#include <cmath>+#include <cstring>++#ifndef M_LOG2E+# define M_LOG2E 1.44269504088896340736+#endif++htp_op_code htp_op_remap(const struct ggml_tensor * t) {+ switch (t->op) {+ case GGML_OP_FLASH_ATTN_EXT: return HTP_OP_FLASH_ATTN_EXT;+ case GGML_OP_MUL_MAT: return htp_is_hadamard(t) ? HTP_OP_HADAMARD : HTP_OP_MUL_MAT;+ case GGML_OP_MUL_MAT_ID: return HTP_OP_MUL_MAT_ID;+ case GGML_OP_MUL: return HTP_OP_MUL;+ case GGML_OP_ADD: return HTP_OP_ADD;+ case GGML_OP_ADD_ID: return HTP_OP_ADD_ID;+ case GGML_OP_SUB: return HTP_OP_SUB;+ case GGML_OP_DIV: return HTP_OP_DIV;+ case GGML_OP_DUP:+ case GGML_OP_CPY:+ case GGML_OP_CONT: return HTP_OP_CPY;+ case GGML_OP_GET_ROWS: return HTP_OP_GET_ROWS;+ case GGML_OP_SET_ROWS: return HTP_OP_SET_ROWS;+ case GGML_OP_SUM: return HTP_OP_SUM;+ case GGML_OP_SUM_ROWS: return HTP_OP_SUM_ROWS;+ case GGML_OP_POOL_2D: return HTP_OP_POOL_2D;+ case GGML_OP_POOL_1D: return HTP_OP_POOL_1D;+ case GGML_OP_ARGSORT: return HTP_OP_ARGSORT;+ case GGML_OP_TOP_K: return HTP_OP_TOP_K;+ case GGML_OP_ARGMAX: return HTP_OP_ARGMAX;+ case GGML_OP_NORM: return HTP_OP_NORM;+ case GGML_OP_L2_NORM: return HTP_OP_L2_NORM;+ case GGML_OP_RMS_NORM: return HTP_OP_RMS_NORM;+ case GGML_OP_CONCAT: return HTP_OP_CONCAT;+ case GGML_OP_SCALE: return HTP_OP_SCALE;+ case GGML_OP_CLAMP: return HTP_OP_CLAMP;+ case GGML_OP_LEAKY_RELU: return HTP_OP_LEAKY_RELU;+ case GGML_OP_SQR: return HTP_OP_SQR;+ case GGML_OP_SQRT: return HTP_OP_SQRT;+ case GGML_OP_LOG: return HTP_OP_UNARY_LOG;+ case GGML_OP_SOFT_MAX: return HTP_OP_SOFTMAX;+ case GGML_OP_SSM_CONV: return HTP_OP_SSM_CONV;+ case GGML_OP_GATED_DELTA_NET: return HTP_OP_GATED_DELTA_NET;+ case GGML_OP_ROPE: return HTP_OP_ROPE;+ case GGML_OP_REPEAT: return HTP_OP_REPEAT;+ case GGML_OP_CUMSUM: return HTP_OP_CUMSUM;+ case GGML_OP_FILL: return HTP_OP_FILL;+ case GGML_OP_DIAG: return HTP_OP_DIAG;+ case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI;+ case GGML_OP_TRI: return HTP_OP_TRI;+ case GGML_OP_PAD: return HTP_OP_PAD;+ case GGML_OP_IM2COL: return HTP_OP_IM2COL;+ case GGML_OP_IM2COL_3D: return HTP_OP_IM2COL_3D;+ case GGML_OP_ROLL: return HTP_OP_ROLL;++ case GGML_OP_UNARY:+ switch (ggml_get_unary_op(t)) {+ case GGML_UNARY_OP_SILU: return HTP_OP_UNARY_SILU;+ case GGML_UNARY_OP_GELU: return HTP_OP_UNARY_GELU;+ case GGML_UNARY_OP_GELU_QUICK: return HTP_OP_UNARY_GELU_QUICK;+ case GGML_UNARY_OP_GELU_ERF: return HTP_OP_UNARY_GELU_ERF;+ case GGML_UNARY_OP_SIGMOID: return HTP_OP_UNARY_SIGMOID;+ case GGML_UNARY_OP_NEG: return HTP_OP_UNARY_NEG;+ case GGML_UNARY_OP_EXP: return HTP_OP_UNARY_EXP;+ case GGML_UNARY_OP_SOFTPLUS: return HTP_OP_UNARY_SOFTPLUS;+ case GGML_UNARY_OP_TANH: return HTP_OP_UNARY_TANH;+ case GGML_UNARY_OP_ABS: return HTP_OP_UNARY_ABS;+ case GGML_UNARY_OP_RELU: return HTP_OP_UNARY_RELU;+ case GGML_UNARY_OP_STEP: return HTP_OP_UNARY_STEP;+ default: break;+ }+ break;++ case GGML_OP_GLU:+ switch (ggml_get_glu_op(t)) {+ case GGML_GLU_OP_REGLU: return HTP_OP_GLU_REGLU;+ case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU;+ case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI;+ case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP;+ case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU;+ case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK;+ case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF;+ default: break;+ }+ break;++ default:+ break;+ }+ return HTP_OP_INVALID;+}++bool htp_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride) {+ size_t s = 0, next = 0;+ for (int i = 1; i < GGML_MAX_DIMS; i++) {+ if (ne[i] == 1) continue;+ if (s == 0) { s = nb[i]; next = s * ne[i]; continue; }+ if (nb[i] != next) return false;+ next *= ne[i];+ }+ *stride = s ? s : nb[1];+ return true;+}++bool htp_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst) {+ size_t s1, sd;+ return (dst->op == GGML_OP_MUL_MAT || dst->op == GGML_OP_ADD) &&+ src0->ne[2] == 1 && src0->ne[3] == 1 && src1->ne[2] * src1->ne[3] > 1 &&+ src1->nb[0] == ggml_type_size(src1->type) && htp_rows_stride(src1->ne, src1->nb, &s1) &&+ dst->nb[0] == ggml_type_size(dst->type) && htp_rows_stride(dst->ne, dst->nb, &sd);+}++ggml_tensor htp_tensor_collapse_rows(const struct ggml_tensor * t) {+ size_t stride = 0;+ htp_rows_stride(t->ne, t->nb, &stride);+ ggml_tensor c = *t;+ c.ne[1] = t->ne[1] * t->ne[2] * t->ne[3];+ c.ne[2] = 1;+ c.ne[3] = 1;+ c.nb[1] = stride;+ c.nb[2] = c.nb[1] * c.ne[1];+ c.nb[3] = c.nb[2];+ return c;+}++bool htp_kparams_solve_allreduce(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * dst,+ uint32_t rank,+ uint32_t n_ranks,+ bool has_add,+ bool is_row_bcast,+ bool is_shard_ok,+ struct htp_allreduce_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));+ kparams->rank = (int32_t) rank;+ kparams->n_ranks = (int32_t) n_ranks;+ kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0;++ const uint32_t max_threads = htp_session_n_threads(sess);+ const size_t vtcm_size = htp_session_vtcm_size(sess);++ const uint32_t nelem = (uint32_t) ggml_nelements(dst);+ const uint32_t elem_size = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);+ const bool is_contiguous = ggml_is_contiguous(dst);++ const uint32_t ne0 = (uint32_t) dst->ne[0];+ const uint32_t ne1 = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);+ kparams->ne0 = (int32_t) ne0;+ kparams->ne1 = (int32_t) ne1;++ const bool use_1d = is_contiguous && !(has_add && is_row_bcast && ne1 > 1);++ if (has_add) {+ if (opt_ar_scatter && is_shard_ok && use_1d && !is_row_bcast) {+ const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128);+ const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem);+ const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem);+ kparams->n_dsts = (int32_t) n_ranks;+ kparams->rank_elem_start = (int32_t) rank_elem_start;+ kparams->rank_nelem = (int32_t) (rank_elem_end - rank_elem_start);+ } else if (use_1d) {+ kparams->n_dsts = 1;+ kparams->rank_elem_start = 0;+ kparams->rank_nelem = (int32_t) nelem;+ } else {+ kparams->n_dsts = 1;+ kparams->rank_elem_start = 0;+ kparams->rank_nelem = (int32_t) ne1;+ }+ } else {+ kparams->n_dsts = (int32_t) n_ranks;+ if (use_1d) {+ const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128);+ const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem);+ const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem);+ const uint32_t rank_nelem = rank_elem_end - rank_elem_start;+ kparams->rank_elem_start = (int32_t) rank_elem_start;+ kparams->rank_nelem = (int32_t) rank_nelem;+ } else {+ const uint32_t rank_chunk_rows = (ne1 + n_ranks - 1) / n_ranks;+ const uint32_t rank_r0 = (std::min)(rank * rank_chunk_rows, ne1);+ const uint32_t rank_r1 = (std::min)(rank_r0 + rank_chunk_rows, ne1);+ const uint32_t rank_nrows = rank_r1 - rank_r0;+ kparams->rank_elem_start = (int32_t) rank_r0;+ kparams->rank_nelem = (int32_t) rank_nrows;+ }+ }++ kparams->mode = (kparams->n_dsts > 1) ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL;++ if (use_1d) {+ const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem;+ const uint32_t n_threads = (std::min)(max_threads, (std::max)(1u, rank_nelem / 128));+ kparams->n_threads = n_threads;+ const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast);++ uint32_t block_elems = 65536;+ if (block_elems > rank_nelem / n_threads && rank_nelem / n_threads > 128) {+ block_elems = hex_round_up(rank_nelem / (n_threads * 2), 128);+ }+ block_elems = (std::max)(128u, block_elems);++ kparams->block_elems = block_elems;+ kparams->vtcm_size_per_thread = 2 * block_elems * elem_size;+ kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;++ while ((size_t) kparams->vtcm_size > vtcm_size && block_elems > 128) {+ const size_t max_bytes_per_buf = vtcm_size / (n_vtcm_buffers * 2);+ block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128);+ if (block_elems < 128) break;+ kparams->block_elems = block_elems;+ kparams->vtcm_size_per_thread = 2 * block_elems * elem_size;+ kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;+ }++ if (vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) {+ HEX_VERBOSE("ggml-hex: %s allreduce 1D solver failed to fit VTCM (%d > %zu)\n",+ htp_session_name(sess), kparams->vtcm_size, vtcm_size);+ return false;+ }++ kparams->elems_per_thread = hex_round_up((rank_nelem + n_threads - 1) / n_threads, block_elems);+ kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_1D;+ return true;+ } else {+ const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem;+ const uint32_t n_threads = (std::min)(max_threads, (std::max)(1u, rank_nrows));+ kparams->n_threads = n_threads;+ const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast);++ const uint32_t row_bytes = ne0 * elem_size;+ const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128);+ kparams->row_size_aligned = row_size_aligned;++ const uint32_t nrows_per_thread = (rank_nrows + n_threads - 1) / n_threads;+ uint32_t block_rows = (std::min)(128u, nrows_per_thread);+ block_rows = (std::max)(1u, block_rows);+ kparams->block_elems = block_rows;++ kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned);+ kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;++ while ((size_t) kparams->vtcm_size > vtcm_size && block_rows > 1) {+ const size_t max_rows_per_buf = vtcm_size / (n_vtcm_buffers * 2 * row_size_aligned);+ block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf);+ kparams->block_elems = block_rows;+ kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned);+ kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread;+ if (max_rows_per_buf == 0) break;+ }++ if (vtcm_size < (size_t) kparams->vtcm_size || block_rows < 1) {+ HEX_VERBOSE("ggml-hex: %s allreduce 2D solver failed to fit VTCM (%d > %zu)\n",+ htp_session_name(sess), kparams->vtcm_size, vtcm_size);+ return false;+ }++ kparams->elems_per_thread = nrows_per_thread;+ kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_2D;+ return true;+ }+}++static bool htp_flash_attn_is_hmx_eligible(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * q,+ const struct ggml_tensor * k,+ const struct ggml_tensor * v,+ const struct ggml_tensor * sinks+) {+ if (htp_session_n_hmx(sess) == 0 || opt_fa_select < 2) {+ return false;+ }++ if ((k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0) ||+ (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_Q8_0)) {+ return false;+ }++ const uint32_t DK = q->ne[0];+ const uint32_t DV = v->ne[0];++ if (DK % 8 != 0 || DV % 8 != 0) {+ return false;+ }++ GGML_UNUSED(sinks);++ if (opt_fa_select > 2) {+ return true;+ }++ const uint32_t M = q->ne[1];+ if (M > 1) {+ return true;+ }++ const uint32_t n_threads = htp_session_n_threads(sess);+ const uint32_t n_head = q->ne[2];+ const uint32_t n_kv_heads = k->ne[2];+ const uint32_t G = n_kv_heads > 0 ? n_head / n_kv_heads : 1;+ const uint32_t S = k->ne[1];+ const uint32_t total_threads = n_threads * htp_session_mdev_count(sess);++ if (DK > 256 || (DK == 256 && (DV > 256 || n_kv_heads < total_threads))) {+ return true;+ }++ const bool is_tile_aligned = (G > 0 && (32 % G == 0));+ if (!is_tile_aligned) {+ if (n_threads >= 6) {+ return false;+ }+ return S >= 1024;+ }++ uint32_t s_cross = 512;+ if (DK == 256) {+ s_cross = 4096;+ } else if (DK <= 64) {+ s_cross = (n_threads >= 8) ? 2048 : ((n_threads >= 6) ? 768 : 512);+ } else {+ s_cross = (n_threads >= 8) ? 1024 : 512;+ }++ return S >= s_cross;+}++static bool htp_kparams_solve_flash_attn(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_fa_kernel_params * kparams+) {+ if (opt_fa_select < 1) {+ return false;+ }++ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * q = op->src[0];+ const struct ggml_tensor * k = op->src[1];+ const struct ggml_tensor * v = op->src[2];+ const struct ggml_tensor * mask = op->src[3];+ const struct ggml_tensor * sinks = op->src[4];+ const struct ggml_tensor * dst = op;++ if ((q->type != GGML_TYPE_F16 && q->type != GGML_TYPE_F32) ||+ (k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0) ||+ (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_Q8_0)) {+ return false;+ }+ if (mask && mask->type != GGML_TYPE_F16) {+ return false;+ }+ if (sinks && sinks->type != GGML_TYPE_F32) {+ return false;+ }+ if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) {+ return false;+ }++ const uint32_t n_threads = htp_session_n_threads(sess);+ const size_t vtcm_size = htp_session_vtcm_size(sess);++ const uint32_t neq0 = q->ne[0];+ const uint32_t neq1 = q->ne[1];+ const uint32_t neq2 = q->ne[2];+ const uint32_t nek1 = k->ne[1];+ const uint32_t nev0 = v->ne[0];++ const uint32_t DK = neq0;+ const uint32_t DV = nev0;++ const uint32_t n_kv_heads = k->ne[2];+ const uint32_t G = neq2 / n_kv_heads;++ float scale = 1.0f;+ float max_bias = 0.0f;+ float logit_softcap = 0.0f;+ memcpy(&scale, &op->op_params[0], sizeof(float));+ memcpy(&max_bias, &op->op_params[1], sizeof(float));+ memcpy(&logit_softcap, &op->op_params[2], sizeof(float));++ if (logit_softcap != 0.0f) {+ scale /= logit_softcap;+ }++ kparams->scale = scale;+ kparams->max_bias = max_bias;+ kparams->logit_softcap = logit_softcap;+ kparams->head_split = (opt_fa_head_split != 0) ? 1 : 0;+ kparams->G = G;++ const uint32_t n_head = q->ne[2];+ kparams->n_head_log2 = 1u << (uint32_t) std::floor(std::log2(n_head));+ kparams->m0 = std::pow(2.0f, -(max_bias) / kparams->n_head_log2);+ kparams->m1 = std::pow(2.0f, -(max_bias / 2.0f) / kparams->n_head_log2);++ if (htp_flash_attn_is_hmx_eligible(sess, q, k, v, sinks)) {+ const uint32_t DK_pad = hex_round_up(DK, 64);+ const uint32_t DV_pad = hex_round_up(DV, 64);+ size_t Br = 0, Bc = 0;+ int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, vtcm_size, n_threads, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head);+ if (ret == 0) {+ kparams->kernel_type = HTP_FA_KERNEL_HMX;+ if (logit_softcap == 0.0f) {+ kparams->scale = scale * (float) M_LOG2E;+ kparams->logit_softcap = 0.0f;+ } else {+ kparams->scale = scale;+ kparams->logit_softcap = logit_softcap * (float) M_LOG2E;+ }+ kparams->Br = Br;+ kparams->Bc = Bc;+ kparams->n_kv_blocks = (nek1 + Bc - 1) / Bc;+ kparams->n_threads = (kparams->n_kv_blocks >= 3 && n_threads >= 2) ? n_threads : 1;+ kparams->u.hmx.g_br = hex_align_up(G * Br, 32);+ kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && n_threads >= 2) ? 1 : 0;+ kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK_pad, DV_pad, Br, Bc, kparams->n_threads, kparams->u.hmx.pipeline != 0, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head);+ kparams->u.hmx.mask_broadcast = (mask != nullptr && mask->ne[2] == 1) ? 1 : 0;+ kparams->u.hmx.div_G = init_fastdiv_values(G);+ if (mask) {+ kparams->src3_div2 = init_fastdiv_values(mask->ne[2]);+ kparams->src3_div3 = init_fastdiv_values(mask->ne[3]);+ }+ return true;+ }+ }++ kparams->kernel_type = HTP_FA_KERNEL_HVX;+ kparams->Br = 1;+ kparams->Bc = 64;+ kparams->n_kv_blocks = (k->ne[1] + 64 - 1) / 64;+ kparams->n_threads = n_threads;++ const size_t size_q_row_padded = hex_round_up(q->ne[0] * ((q->type == GGML_TYPE_F32) ? 4 : 2), 128);+ const size_t size_k_row_padded = hex_round_up(k->ne[0] * 2, 128);+ const size_t size_v_row_padded = hex_round_up(v->ne[0] * 2, 128);++ kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, (q->type == GGML_TYPE_F32), mask != nullptr, sinks != nullptr, n_head, n_threads);+ if ((size_t) kparams->vtcm_size > vtcm_size) {+ HEX_VERBOSE("ggml-hex: skip flash_attn_ext because VTCM needed (%d) > budget (%zu)\n",+ kparams->vtcm_size, vtcm_size);+ return false;+ }++ kparams->u.hvx.size_q_row_padded = size_q_row_padded;+ kparams->u.hvx.size_k_row_padded = size_k_row_padded;+ kparams->u.hvx.size_v_row_padded = size_v_row_padded;+ kparams->u.hvx.src0_div21 = init_fastdiv_values(q->ne[2] * q->ne[1]);+ kparams->u.hvx.src0_div1 = init_fastdiv_values(q->ne[1]);+ kparams->broadcast_rk2 = init_fastdiv_values(q->ne[2] / k->ne[2]);+ kparams->broadcast_rk3 = init_fastdiv_values(q->ne[3] / k->ne[3]);+ kparams->broadcast_rv2 = init_fastdiv_values(q->ne[2] / v->ne[2]);+ kparams->broadcast_rv3 = init_fastdiv_values(q->ne[3] / v->ne[3]);+ if (mask) {+ kparams->src3_div2 = init_fastdiv_values(mask->ne[2]);+ kparams->src3_div3 = init_fastdiv_values(mask->ne[3]);+ }++ return true;+}++static bool htp_kparams_solve_gated_delta_net(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_gdn_kernel_params * kparams+) {+ if (opt_gdn_select < 1) {+ return false;+ }++ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * q = op->src[0];+ const struct ggml_tensor * k = op->src[1];+ const struct ggml_tensor * v = op->src[2];+ const struct ggml_tensor * g = op->src[3];+ const struct ggml_tensor * beta = op->src[4];+ const struct ggml_tensor * state = op->src[5];+ const struct ggml_tensor * dst = op;++ if (q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F32 || v->type != GGML_TYPE_F32 ||+ g->type != GGML_TYPE_F32 || beta->type != GGML_TYPE_F32 || state->type != GGML_TYPE_F32 ||+ dst->type != GGML_TYPE_F32) {+ return false;+ }++ if (!ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) || !ggml_is_contiguous_rows(v) ||+ !ggml_is_contiguous(g) || !ggml_is_contiguous(beta) || !ggml_is_contiguous(state) ||+ !ggml_is_contiguous(dst)) {+ return false;+ }++ const int64_t S_v = v->ne[0];+ const int64_t H = v->ne[1];+ const int64_t n_tokens = v->ne[2];+ const int64_t n_seqs = v->ne[3];+ const int64_t K = ggml_get_op_params_i32(op, 0);++ if (S_v <= 0 || S_v > 128 || H <= 0 || n_tokens <= 0 || n_seqs <= 0) {+ return false;+ }+ if (q->ne[0] != S_v || k->ne[0] != S_v || q->ne[1] <= 0 || k->ne[1] <= 0 ||+ q->ne[2] != n_tokens || k->ne[2] != n_tokens || q->ne[3] <= 0 || k->ne[3] <= 0 ||+ (n_seqs % q->ne[3]) != 0 || (n_seqs % k->ne[3]) != 0) {+ return false;+ }+ if ((g->ne[0] != 1 && g->ne[0] != S_v) || beta->ne[0] != 1) {+ return false;+ }+ if (state->ne[0] != S_v || state->ne[1] != S_v || state->ne[2] != H || state->ne[3] != n_seqs) {+ return false;+ }+ if (dst->ne[0] != S_v * H || dst->ne[1] != n_tokens * n_seqs + S_v * n_seqs * K ||+ dst->ne[2] != 1 || dst->ne[3] != 1) {+ return false;+ }++ const uint32_t rq3 = (uint32_t) (n_seqs / q->ne[3]);+ const uint32_t rk3 = (uint32_t) (n_seqs / k->ne[3]);+ const uint32_t total_rows = (uint32_t) (H * n_seqs);+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), total_rows);+ const size_t vtcm_size = htp_session_vtcm_size(sess);++ const bool can_use_hmx = (opt_gdn_select >= 2) &&+ (htp_session_n_hmx(sess) > 0) &&+ (S_v % 64 == 0) &&+ (n_tokens >= HTP_GDN_MIN_TOKENS) &&+ (g->ne[0] == 1) &&+ (K == 1);++ struct htp_gdn_hmx_vtcm_layout hmx_layout;+ struct htp_gdn_vtcm_layout hvx_layout;+ uint32_t n_heads_batch = 1;++ if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout, (uint32_t) S_v, HTP_GDN_CHUNK_SIZE, total_rows, vtcm_size, n_threads, true, &n_heads_batch)) {+ kparams->kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED;+ kparams->pipeline = hmx_layout.pipeline ? 1 : 0;+ kparams->chunk_size = HTP_GDN_CHUNK_SIZE;+ kparams->n_chunks = ((uint32_t) n_tokens + HTP_GDN_CHUNK_SIZE - 1) / HTP_GDN_CHUNK_SIZE;+ kparams->n_heads_batch = (uint16_t) n_heads_batch;+ kparams->vtcm_size = (uint32_t) hmx_layout.total_bytes;+ } else {+ htp_gdn_vtcm_layout_build(&hvx_layout, (uint32_t) S_v, n_threads);+ if (hvx_layout.total_bytes > vtcm_size) {+ return false;+ }+ kparams->kernel_type = HTP_GDN_KERNEL_HVX_RECURRENT;+ kparams->pipeline = 0;+ kparams->n_heads_batch = 1;+ kparams->vtcm_size = (uint32_t) hvx_layout.total_bytes;+ }++ kparams->n_threads = n_threads;+ kparams->S_v = (uint32_t) S_v;+ kparams->H = (uint32_t) H;+ kparams->n_tokens = (uint32_t) n_tokens;+ kparams->n_seqs = (uint32_t) n_seqs;+ kparams->K = (uint32_t) K;+ kparams->total_rows = total_rows;+ kparams->kda = (g->ne[0] == S_v) ? 1 : 0;+ kparams->state_seq_stride = (uint32_t) (state->nb[3] / sizeof(float));+ kparams->state_size_per_snap = (uint32_t) (S_v * S_v * H * n_seqs);+ kparams->scale = 1.0f / sqrtf((float) S_v);++ if (H > 0) kparams->div_H = init_fastdiv_values((uint32_t) H);+ if (q->ne[1] > 0) kparams->div_q1 = init_fastdiv_values((uint32_t) q->ne[1]);+ if (k->ne[1] > 0) kparams->div_k1 = init_fastdiv_values((uint32_t) k->ne[1]);+ if (rq3 > 0) kparams->div_rq3 = init_fastdiv_values(rq3);+ if (rk3 > 0) kparams->div_rk3 = init_fastdiv_values(rk3);+ return true;+}++static bool htp_matmul_is_hmx_eligible(+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * dst,+ bool is_matmul_id,+ bool is_batched+) {+ GGML_UNUSED(dst);++ if (src1->type != GGML_TYPE_F32 && (src1->type != GGML_TYPE_F16 || is_matmul_id)) {+ return false;+ }++ const int ne00 = src0->ne[0];+ const int ne01 = src0->ne[1];+ const int ne11 = src1->ne[1];+ const int ne12 = src1->ne[2];+ const int wtype = src0->type;++ if (ne01 % HTP_TENSOR_TILE_N_ROWS != 0 && is_matmul_id) {+ return false;+ }++ if (!htp_is_hmx_weight_type((ggml_type) wtype)) {+ return false;+ }++ if (ne00 % HTP_TENSOR_TILE_N_COLS != 0) {+ return false;+ }++ const bool is_tiled = htp_tensor_is_tiled(src0);+ if (ggml_is_quantized((ggml_type) wtype) && !is_tiled) {+ return false;+ }+ if (!is_matmul_id && is_batched && is_tiled) {+ return false;+ }++ if (src0->nb[0] > src0->nb[1] || src1->nb[0] > src1->nb[1]) {+ return false;+ }++ const int m = is_matmul_id ? ne12 : ne11;+ if (m <= HTP_MM_HMX_MIN_NROWS) {+ return false;+ }++ return true;+}++static bool htp_kparams_solve_hmx_mm(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * act,+ const struct ggml_tensor * dst,+ int wtype,+ int ne00_padded,+ int ne01_padded,+ int ne02,+ int ne11,+ int ne12,+ int ne11_padded,+ bool is_matmul_id,+ bool is_batched,+ size_t src2_size,+ size_t vtcm_budget,+ struct htp_mm_kernel_params * kparams+) {+ const bool is_tiled = htp_tensor_is_tiled(src0);+ const int aligned_tile_size = is_tiled ? htp_tensor_get_tile_padded_size(wtype) : 0;+ const int n_threads = (int) htp_session_n_threads(sess);+ const uint32_t mdev_count = htp_session_mdev_count(sess);+ const int ne10 = act->ne[0];++ int m_for_solver = ne11;+ int m_for_solver_padded = ne11_padded;+ if (!is_matmul_id && mdev_count > 1 && ((uint32_t) ne11 >= mdev_count)) {+ const bool dst_row_split = dst ? htp_can_row_partition(dst) : true;+ const bool act_row_split = htp_can_row_partition(act);+ if (dst_row_split && act_row_split) {+ m_for_solver = (ne11 + (int) mdev_count - 1) / (int) mdev_count;+ m_for_solver_padded = hex_round_up(std::max(m_for_solver, 32), 32);+ }+ }+ const bool pipeline = is_matmul_id ? false : htp_mm_hmx_pipeline(m_for_solver);+ const bool is_batched_val = is_matmul_id ? false : is_batched;+ const int group_size = (ne02 > 0 ? ne12 / ne02 : 1);++ size_t m_chunk = 0;+ size_t n_chunk = 0;+ size_t vtcm_size = 0;+ bool use_grouped = false;+ int act_threads_selected = 0;++ if (is_batched_val && wtype == GGML_TYPE_F16 && !is_tiled && group_size > 1) {+ if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, m_for_solver, group_size, n_threads, pipeline, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) {+ use_grouped = true;+ }+ }++ if (!use_grouped) {+ int m_id_rows = 0;+ if (dst && is_matmul_id) {+ const int n_experts = ne02 > 0 ? ne02 : 1;+ const size_t total_expert_rows = (size_t) dst->ne[1] * dst->ne[2];+ int m_per_expert = (int) ((total_expert_rows + n_experts - 1) / n_experts);+ if (mdev_count > 1 && htp_can_row_partition(dst)) {+ m_per_expert = (m_per_expert + (int) mdev_count - 1) / (int) mdev_count;+ }+ m_id_rows = hex_round_up(std::max(m_per_expert, 32), 32);+ }+ const uint32_t cost_m = is_matmul_id ? (uint32_t) m_id_rows : (uint32_t) m_for_solver;+ if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, m_for_solver_padded, cost_m, n_threads, pipeline, is_matmul_id, aligned_tile_size, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) {+ return false;+ }+ }++ kparams->n_hmx = 1;+ kparams->pipeline = pipeline ? 1 : 0;+ kparams->m_chunk = m_chunk;+ kparams->n_chunk = n_chunk;+ kparams->n_threads = n_threads;+ kparams->n_act_threads = act_threads_selected;+ kparams->tile_size = is_tiled ? htp_tensor_get_tile_size(wtype) : 0;+ kparams->aligned_tile_size = aligned_tile_size;+ kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);+ kparams->vtcm_size = vtcm_size;+ kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected);+ kparams->div_ne00_padded = init_fastdiv_values(ne00_padded);+ kparams->vtcm_bias_size = (int32_t) src2_size;+ kparams->kernel_type = (is_batched && !is_matmul_id) ? HTP_MM_KERNEL_HMX_BATCHED : HTP_MM_KERNEL_HMX_2D;+ return true;+}++static void htp_kparams_solve_hvx_mm(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * dst,+ int wtype,+ bool is_tiled,+ int ne02,+ int ne03,+ int ne10,+ int ne11,+ int ne12,+ int ne13,+ bool is_matmul_id,+ const size_t src2_row_size,+ size_t vtcm_budget,+ struct htp_mm_kernel_params * kparams+) {+ if (opt_mm_select < 1) {+ kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;+ return;+ }++ const uint32_t n_threads = htp_session_n_threads(sess);+ kparams->n_hmx = 0;+ kparams->n_threads = n_threads;++ GGML_UNUSED(ne02);+ GGML_UNUSED(ne03);++ const int src1_nrows = ne11 * ne12 * ne13;++ if (is_tiled) {+ kparams->tile_size = htp_tensor_get_tile_size(wtype);+ kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);++ const bool k_align = (ne10 % htp_tensor_get_tile_k(wtype) == 0);++ if (is_matmul_id) {+ kparams->kernel_type = (src1_nrows < (int) n_threads)+ ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW;+ kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);++ struct htp_mm_hvx_vtcm_layout L;+ uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;+ uint32_t best_n_prefetch = 2;+ for (uint32_t d = max_prefetch; d >= 2; d /= 2) {+ htp_mm_hvx_vtcm_layout_build(+ &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_threads,+ 0, src0->nb[1], kparams->act_row_size, 0, d, true, false+ );+ if (L.total_bytes <= vtcm_budget) {+ best_n_prefetch = d;+ break;+ }+ }+ if (L.total_bytes > vtcm_budget) {+ kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;+ return;+ }+ kparams->n_prefetch = best_n_prefetch;+ kparams->vtcm_size = L.total_bytes;+ kparams->vtcm_bias_size = 0;+ return;+ } else {+ if (k_align && opt_mm_select >= 1) {+ kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);+ kparams->kernel_type = (src1_nrows < (int) n_threads)+ ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW;++ struct htp_mm_hvx_vtcm_layout L;+ uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;+ uint32_t best_n_prefetch = 2;+ for (uint32_t d = max_prefetch; d >= 2; d /= 2) {+ htp_mm_hvx_vtcm_layout_build(+ &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_threads,+ dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, d, false, false+ );+ if (L.total_bytes <= vtcm_budget) {+ best_n_prefetch = d;+ break;+ }+ }++ uint32_t m_chunk = 0;+ if (htp_mm_hvx_solve_vtcm_params(+ kparams->kernel_type, wtype, ne10, src1_nrows, n_threads,+ dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, best_n_prefetch, vtcm_budget,+ &L, &m_chunk)) {+ kparams->n_prefetch = best_n_prefetch;+ kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0;+ kparams->vtcm_size = L.total_bytes;+ kparams->vtcm_bias_size = L.bias_bytes;+ return;+ }+ }++ kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;+ return;+ }+ } else if (is_matmul_id) {+ kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;+ return;+ } else if (wtype == GGML_TYPE_F16 || wtype == GGML_TYPE_F32 || wtype == GGML_TYPE_BF16) {+ int32_t ktype = HTP_MM_KERNEL_HVX_F16_F16_VTCM;+ int32_t act_elem = 2;+ if (wtype == GGML_TYPE_F32) {+ ktype = HTP_MM_KERNEL_HVX_F32_F32_VTCM;+ act_elem = 4;+ } else if (wtype == GGML_TYPE_BF16) {+ ktype = HTP_MM_KERNEL_HVX_BF16_F32_VTCM;+ act_elem = 4;+ }+ struct htp_mm_hvx_vtcm_layout L;+ uint32_t m_chunk = 0;+ if (htp_mm_hvx_solve_vtcm_params(+ ktype, wtype, ne10, src1_nrows, n_threads,+ dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget,+ &L, &m_chunk)) {+ kparams->kernel_type = ktype;+ kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0;+ kparams->act_row_size = hex_round_up(ne10 * act_elem, 128);+ kparams->vtcm_size = L.total_bytes;+ kparams->vtcm_bias_size = L.bias_bytes;+ kparams->n_prefetch = 16;+ return;+ }+ }++ kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;+}++static bool htp_kparams_solve_matmul_impl(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * dst,+ const size_t src2_row_size,+ const size_t src2_size,+ struct htp_mm_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const int ne00 = src0->ne[0];+ const int ne01 = src0->ne[1];+ const int ne02 = src0->ne[2];+ const int ne03 = src0->ne[3];++ const int ne10 = src1->ne[0];+ const int ne11 = src1->ne[1];+ const int ne12 = src1->ne[2];+ const int ne13 = src1->ne[3];++ const bool is_tiled = htp_tensor_is_tiled(src0);+ const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;+ const int ne00_padded = is_tiled ? hex_round_up(ne00, htp_tensor_get_tile_k(wtype)) : ne00;+ const int ne11_padded = hex_round_up(ne11, 32);+ const int ne01_tiled = hex_round_up(ne01, 32);++ const bool is_matmul_id = (dst->op == GGML_OP_MUL_MAT_ID);+ const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1);+ const size_t vtcm_budget = htp_session_vtcm_size(sess);++ const bool hmx_enabled = (htp_session_n_hmx(sess) > 0) && (opt_mm_select >= 2);+ if (hmx_enabled && htp_matmul_is_hmx_eligible(src0, src1, dst, is_matmul_id, is_batched)) {+ if (htp_kparams_solve_hmx_mm(sess, src0, src1, dst, wtype, ne00_padded, ne01_tiled, ne02, ne11, ne12, ne11_padded, is_matmul_id, is_batched, src2_size, vtcm_budget, kparams)) {+ goto finalize;+ }+ }++ htp_kparams_solve_hvx_mm(sess, src0, src1, dst, wtype, is_tiled, ne02, ne03, ne10, ne11, ne12, ne13, is_matmul_id, src2_row_size, vtcm_budget, kparams);++finalize:+ kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11);+ kparams->div_ne1 = init_fastdiv_values(ne11);+ kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1);+ kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1);+ kparams->div_ne12 = init_fastdiv_values(ne12);++ if (kparams->kernel_type == HTP_MM_KERNEL_UNSUPPORTED || (size_t) kparams->vtcm_size > vtcm_budget) {+ HEX_VERBOSE("ggml-hex: %s supported %s VTCM size needed (%d) > budget (%zu)\n",+ htp_session_name(sess), is_matmul_id ? "MUL_MAT_ID" : "MUL_MAT", kparams->vtcm_size, vtcm_budget);+ return false;+ }+ return true;+}++static bool htp_matmul_check_types(+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * dst+) {+ const bool is_matmul_id = (dst->op == GGML_OP_MUL_MAT_ID);++ if (dst->type != GGML_TYPE_F32) {+ return false;+ }++ if (is_matmul_id) {+ const struct ggml_tensor * src2 = dst->src[2];+ if (src1->type != GGML_TYPE_F32 || !src2 || src2->type != GGML_TYPE_I32) {+ return false;+ }+ if (ggml_get_op_params_i32(dst, 3) == GGML_PREC_F32) {+ return false;+ }+ switch (src0->type) {+ case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q8_0:+ case GGML_TYPE_IQ4_NL:+ case GGML_TYPE_MXFP4:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ case GGML_TYPE_Q6_K:+ case GGML_TYPE_Q3_K:+ case GGML_TYPE_Q2_K:+ if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) {+ return false;+ }+ if (src0->ne[0] % ggml_blck_size(src0->type) != 0) {+ return false;+ }+ return true;++ case GGML_TYPE_F16:+ case GGML_TYPE_F32:+ case GGML_TYPE_BF16:+ if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0)) {+ return false;+ }+ return src0->ne[0] % 32 == 0;++ default:+ return false;+ }+ }++ if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) {+ return false;+ }+ if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3] ||+ src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) {+ return false;+ }++ switch (src0->type) {+ case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q8_0:+ case GGML_TYPE_IQ4_NL:+ case GGML_TYPE_MXFP4:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ case GGML_TYPE_Q6_K:+ case GGML_TYPE_Q3_K:+ case GGML_TYPE_Q2_K:+ if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) {+ return false;+ }+ return src0->ne[0] % ggml_blck_size(src0->type) == 0;++ case GGML_TYPE_F32:+ if (src1->type != GGML_TYPE_F32) {+ return false;+ }+ return src0->nb[1] >= src0->nb[0];++ case GGML_TYPE_F16:+ case GGML_TYPE_BF16:+ return src0->nb[1] >= src0->nb[0];++ default:+ return false;+ }+}++static bool htp_kparams_solve_matmul(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_mm_kernel_params * kparams+) {+ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * src1 = op->src[1];+ const struct ggml_tensor * dst = op;++ if (!htp_matmul_check_types(src0, src1, dst)) {+ return false;+ }+ if (htp_matmul_can_collapse(src0, src1, dst)) {+ const ggml_tensor src1_collapsed = htp_tensor_collapse_rows(src1);+ const ggml_tensor dst_collapsed = htp_tensor_collapse_rows(dst);+ const bool ok = htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, 0, 0, kparams);+ kparams->collapse = 1;+ return ok;+ }+ return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, 0, 0, kparams);+}++static bool htp_matmul_add_can_collapse(+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * src2,+ const struct ggml_tensor * dst+) {+ if (!htp_matmul_can_collapse(src0, src1, dst)) {+ return false;+ }+ if (!src2) {+ return true;+ }+ const int64_t src2_nrows = src2->ne[1] * src2->ne[2] * src2->ne[3];+ if (src2_nrows == 1) {+ return true;+ }+ size_t s2;+ return src2->nb[0] == ggml_type_size(src2->type) &&+ ggml_are_same_shape(src2, dst) &&+ htp_rows_stride(src2->ne, src2->nb, &s2);+}++bool htp_kparams_solve_matmul_add(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * src2,+ const struct ggml_tensor * dst,+ struct htp_mm_kernel_params * kparams+) {+ if (htp_matmul_add_can_collapse(src0, src1, src2, dst)) {+ const ggml_tensor src1_collapsed = htp_tensor_collapse_rows(src1);+ const ggml_tensor dst_collapsed = htp_tensor_collapse_rows(dst);+ const ggml_tensor src2_collapsed = (src2 && (src2->ne[1] * src2->ne[2] * src2->ne[3] > 1))+ ? htp_tensor_collapse_rows(src2)+ : (src2 ? *src2 : ggml_tensor{});+ const struct ggml_tensor * p_src2 = src2 ? &src2_collapsed : nullptr;+ const size_t src2_size = p_src2 ? hex_round_up(ggml_nbytes(p_src2), 128) : 0;+ const bool ok = htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, p_src2 ? p_src2->nb[1] : 0, src2_size, kparams);+ kparams->collapse = 1;+ return ok;+ }+ const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0;+ return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, kparams);+}++bool htp_kparams_solve_mmnx(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * act,+ int32_t n_weights,+ struct htp_mm_kernel_params * kparams+) {+ const uint32_t n_threads = htp_session_n_threads(sess);+ const uint32_t n_hmx = htp_session_n_hmx(sess);+ const size_t vtcm_budget = htp_session_vtcm_size(sess);++ memset(kparams, 0, sizeof(*kparams));+ kparams->n_threads = n_threads;++ const int ne00 = src0->ne[0];+ const int ne01 = src0->ne[1];+ const int ne02 = src0->ne[2];+ const int ne03 = src0->ne[3];++ const int ne10 = act->ne[0];+ const int ne11 = act->ne[1];+ const int ne12 = act->ne[2];+ const int ne13 = act->ne[3];++ const bool is_tiled = htp_tensor_is_tiled(src0);+ const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;+ const int ne00_padded = is_tiled ? hex_round_up(ne00, 32) : ne00;+ const int ne11_padded = hex_round_up(ne11, 32);+ const int ne01_tiled = hex_round_up(ne01, 32);+ const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1);++ const bool hmx_enabled = (n_hmx > 0) && (opt_mm_select >= 2);+ if (hmx_enabled && htp_matmul_is_hmx_eligible(src0, act, nullptr, false, is_batched)) {+ if (htp_kparams_solve_hmx_mm(sess, src0, act, nullptr, wtype, ne00_padded, ne01_tiled, ne02, ne11, ne12, ne11_padded, false, is_batched, 0, vtcm_budget, kparams)) {+ kparams->n_weights = n_weights;+ goto finalize;+ }+ }++ if (!is_tiled || act->type != GGML_TYPE_F32 || ne10 % 32 != 0) {+ kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;+ return false;+ }++ {+ const int act_nrows = ne11 * ne12 * ne13;+ const size_t act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);+ const size_t src0_row_size = src0->nb[1];++ const uint32_t max_prefetch = (act_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;+ uint32_t best_n_prefetch = 2;+ for (uint32_t d = max_prefetch; d >= 2; d /= 2) {+ struct htp_mm_hvx_vtcm_layout L;+ htp_mm_hvx_vtcm_layout_build(+ &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_threads,+ 0, src0_row_size, act_row_size, 0, d, false, true+ );+ if (L.total_bytes <= vtcm_budget) {+ best_n_prefetch = d;+ break;+ }+ }++ struct htp_mm_hvx_vtcm_layout L;+ htp_mm_hvx_vtcm_layout_build(+ &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_threads,+ 0, src0_row_size, act_row_size, 0, best_n_prefetch, false, true+ );++ if (opt_mm_select >= 1 && L.total_bytes <= vtcm_budget) {+ kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW;+ kparams->tile_size = htp_tensor_get_tile_size(wtype);+ kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);+ kparams->act_row_size = act_row_size;+ kparams->vtcm_bias_size = 0;+ kparams->vtcm_size = L.total_bytes;+ kparams->n_prefetch = best_n_prefetch;+ kparams->n_weights = n_weights;+ } else {+ kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED;+ return false;+ }+ }++finalize:+ kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11);+ kparams->div_ne1 = init_fastdiv_values(ne11);+ kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1);+ kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1);+ kparams->div_ne12 = init_fastdiv_values(ne12);+ return (size_t) kparams->vtcm_size <= vtcm_budget;+}++bool htp_kparams_solve_mmidnx(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * act,+ const struct ggml_tensor * dst,+ int32_t n_weights,+ struct htp_mm_kernel_params * kparams+) {+ const bool ok = htp_kparams_solve_matmul_impl(sess, src0, act, dst, 0, 0, kparams);+ kparams->n_weights = n_weights;+ return ok;+}++static bool htp_kparams_solve_binary(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op_tensor,+ struct htp_binary_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const uint32_t op = htp_op_remap(op_tensor);+ const struct ggml_tensor * src0 = op_tensor->src[0];+ const struct ggml_tensor * src1 = op_tensor->src[1];+ const struct ggml_tensor * src2 = (op == HTP_OP_ADD_ID) ? op_tensor->src[2] : nullptr;+ const struct ggml_tensor * dst = op_tensor;++ const bool is_add_id = (op == HTP_OP_ADD_ID);+ if (is_add_id) {+ if (!src2) {+ return false;+ }+ if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32) {+ return false;+ }+ if (!ggml_are_same_shape(src0, dst)) {+ return false;+ }+ if (src1->ne[0] != src0->ne[0] || src1->ne[2] != 1 || src1->ne[3] != 1) {+ return false;+ }+ if (src2->ne[0] != src0->ne[1] || src2->ne[1] != src0->ne[2]) {+ return false;+ }+ if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float) || src2->nb[0] != sizeof(int32_t)) {+ return false;+ }+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {+ return false;+ }+ if (src2->nb[1] < src2->ne[0] * sizeof(int32_t)) {+ return false;+ }+ } else {+ if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) ||+ src1->type != src0->type || dst->type != src0->type) {+ return false;+ }+ if (ggml_is_permuted(src0) || ggml_is_permuted(dst) || !ggml_are_same_shape(src0, dst)) {+ return false;+ }+ if (!ggml_can_repeat(src1, src0) || ggml_is_permuted(src1)) {+ return false;+ }+ }++ const uint32_t max_threads_sess = htp_session_n_threads(sess);+ const size_t vtcm_size = htp_session_vtcm_size(sess);++ const size_t elem_size = ggml_type_size(src0->type);+ const size_t src0_row_size = src0->ne[0] * elem_size;+ const size_t src1_row_size = src1->ne[0] * elem_size;+ const size_t dst_row_size = dst->ne[0] * elem_size;++ const size_t src0_row_size_aligned = hex_round_up(src0_row_size, 128);+ const size_t src1_row_size_aligned = hex_round_up(src1_row_size, 128);+ const size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128);++ if (!htp_binary_strides_safe(src0, src1, dst, elem_size)) {+ return false;+ }++ const bool is_scalar = !is_add_id && src1->ne[0] == 1;+ const bool is_row_bcast = !is_add_id && !is_scalar &&+ src1->ne[0] == src0->ne[0] &&+ (src0->ne[1] > 1 || src0->ne[2] > 1 || src0->ne[3] > 1) &&+ src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1;+ const bool is_same_shape = !is_add_id && !is_scalar &&+ src1->ne[0] == src0->ne[0] &&+ src1->ne[1] == src0->ne[1] &&+ (src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) &&+ (src1->ne[3] == src0->ne[3] || src1->ne[3] == 1);+ const bool is_complex = !is_add_id && !is_scalar && !is_same_shape && !is_row_bcast && (src1->ne[0] == src0->ne[0]);+ const bool is_contig = ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst);+ const bool is_scalar_broadcast = !is_add_id && (ggml_nelements(src1) == 1);++ if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) {+ const uint32_t total_elems = (uint32_t) ggml_nelements(src0);+ const uint32_t max_chunk_elems = 32768 / elem_size;+ const uint32_t min_chunk_elems = 256;+ const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_threads_sess) - 1) / (2 * max_threads_sess), 32);+ const uint32_t chunk_size = (std::min)(max_chunk_elems, (std::max)(target_chunk_elems, min_chunk_elems));+ const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128);+ const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size;+ const uint32_t n_threads = (total_chunks >= 2) ? (std::min)(max_threads_sess, total_chunks) : 1;++ kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED;+ kparams->n_threads = n_threads;+ kparams->n_srcs = 2;+ kparams->rows_per_buffer = 1;+ kparams->src0_row_size_aligned = src0_row_size_aligned;+ kparams->src1_row_size_aligned = is_scalar_broadcast ? 0 : src1_row_size_aligned;+ kparams->dst_row_size_aligned = dst_row_size_aligned;+ kparams->src1_size = 0;+ kparams->chunk_size = chunk_size;+ kparams->chunk_bytes = chunk_bytes;+ kparams->is_scalar = is_scalar_broadcast ? 1 : 0;++ struct htp_binary_vtcm_layout L;+ htp_binary_vtcm_layout_build(&L, kparams, vtcm_size);+ if (L.total_bytes == 0 || L.total_bytes > vtcm_size) {+ return false;+ }++ kparams->vtcm_size = L.total_bytes;+ return true;+ }++ enum htp_binary_kernel_type kernel_type;+ size_t src1_size = 0;+ size_t src2_size = 0;++ if (is_add_id) {+ kernel_type = HTP_BINARY_KERNEL_ADD_ID;+ src1_size = hex_round_up(src1->ne[1] * src1_row_size_aligned, 128);+ src2_size = hex_round_up(ggml_nelements(src2) * sizeof(int32_t), 128);+ if (src1_size + src2_size > vtcm_size) {+ return false;+ }+ } else if (is_row_bcast) {+ kernel_type = HTP_BINARY_KERNEL_ROW_BCAST;+ src1_size = src1_row_size_aligned;+ } else if (is_scalar) {+ src1_size = hex_round_up(ggml_nelements(src1) * elem_size, 128);+ if (src1_size > vtcm_size) {+ return false;+ }+ kernel_type = HTP_BINARY_KERNEL_SCALAR;+ kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;+ } else if (is_same_shape) {+ kernel_type = HTP_BINARY_KERNEL_SAME_SHAPE;+ } else if (is_complex) {+ src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128);+ if (src1_size > vtcm_size) {+ return false;+ }+ kernel_type = HTP_BINARY_KERNEL_COMPLEX;+ kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;+ } else {+ src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128);+ if (src1_size > vtcm_size) {+ return false;+ }+ kernel_type = HTP_BINARY_KERNEL_REPEAT;+ kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0;+ }++ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];+ const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size));+ const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread);++ kparams->kernel_type = kernel_type;+ kparams->n_threads = (std::min)(max_threads_sess, max_threads);+ kparams->n_srcs = 2;+ kparams->src0_row_size_aligned = src0_row_size_aligned;+ kparams->src1_row_size_aligned = src1_row_size_aligned;+ kparams->dst_row_size_aligned = dst_row_size_aligned;+ kparams->src1_size = src1_size;+ kparams->src2_size = src2_size;++ const bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]);+ const bool src1_contig_dim1 = (src1->nb[2] == src1->ne[1] * src1->nb[1]);+ const bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]);++ const bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]);+ const bool src1_contig_dim2 = (src1->nb[3] == src1->ne[2] * src1->nb[2]);+ const bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]);++ bool split_at_ne01 = false;+ if (src0->ne[2] > 1) {+ if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {+ split_at_ne01 = (src1->ne[2] != src0->ne[2]) || !src0_contig_dim1 || !src1_contig_dim1 || !dst_contig_dim1;+ } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {+ split_at_ne01 = !src0_contig_dim1 || !dst_contig_dim1;+ } else {+ split_at_ne01 = true;+ }+ }+ kparams->split_at_ne01 = split_at_ne01 ? 1 : 0;++ bool split_at_ne02 = false;+ if (src0->ne[3] > 1) {+ if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {+ split_at_ne02 = (src1->ne[3] != src0->ne[3]) || !src0_contig_dim2 || !src1_contig_dim2 || !dst_contig_dim2;+ } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {+ split_at_ne02 = !src0_contig_dim2 || !dst_contig_dim2;+ } else {+ split_at_ne02 = true;+ }+ }+ kparams->split_at_ne02 = split_at_ne02 ? 1 : 0;++ kparams->div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);+ kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]);+ kparams->div_ne012 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[2]));+ kparams->div_ne11 = init_fastdiv_values((uint32_t) src1->ne[1]);+ kparams->div_ne12 = init_fastdiv_values((uint32_t) src1->ne[2]);+ kparams->div_ne13 = init_fastdiv_values((uint32_t) src1->ne[3]);++ struct htp_binary_vtcm_layout L;+ htp_binary_vtcm_layout_build(&L, kparams, vtcm_size);+ if (L.rows_per_buffer == 0 || L.total_bytes > vtcm_size) {+ if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) {+ const uint32_t total_elems = (uint32_t) ggml_nelements(src0);+ return htp_binary_solve_chunked(&L, kparams, total_elems, (uint32_t) elem_size,+ max_threads_sess, 2, is_scalar_broadcast, vtcm_size);+ }+ return false;+ }++ kparams->rows_per_buffer = L.rows_per_buffer;+ kparams->vtcm_size = L.total_bytes;++ return true;+}++static bool htp_kparams_solve_unary_impl(+ const struct ggml_hexagon_session * sess,+ uint32_t op,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * dst,+ struct htp_unary_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ if (!htp_op_is_unary((htp_op_code) op)) {+ return false;+ }++ if (op == HTP_OP_HADAMARD) {+ const int64_t n = src0->ne[0];+ if (n < 32 || (n & (n - 1)) != 0) {+ return false;+ }+ if (!dst->src[0] || dst->src[0]->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 ||+ (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16)) {+ return false;+ }+ } else {+ if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || dst->type != src0->type) {+ return false;+ }+ if (op == HTP_OP_TRI && (src0->type != GGML_TYPE_F32 || !ggml_is_contiguous(src0))) {+ return false;+ }+ if (src0->type == GGML_TYPE_F16) {+ switch (op) {+ case HTP_OP_NORM:+ case HTP_OP_RMS_NORM:+ case HTP_OP_L2_NORM:+ case HTP_OP_SCALE:+ case HTP_OP_CLAMP:+ case HTP_OP_SQR:+ case HTP_OP_SQRT:+ case HTP_OP_UNARY_LOG:+ case HTP_OP_UNARY_ABS:+ case HTP_OP_UNARY_STEP:+ break;+ default:+ return false;+ }+ }+ }++ if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous(dst) || !ggml_are_same_shape(src0, dst)) {+ return false;+ }++ const size_t elem_size = ggml_type_size(src0->type);++ struct ggml_tensor src0_c = *src0;+ struct ggml_tensor dst_c = *dst;+ if (htp_unary_is_elementwise(op)) {+ HTP_UNARY_COLLAPSE_DIMS(src0_c.ne, src0_c.nb, dst_c.ne, dst_c.nb, elem_size);+ src0 = &src0_c;+ dst = &dst_c;+ }++ const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type);++ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];+ const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_data_row_size - 1) / dst_data_row_size));+ const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread);+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);+ const size_t vtcm_size = htp_session_vtcm_size(sess);++ kparams->n_threads = n_threads;++ const bool broadcast_weight = (op == HTP_OP_RMS_NORM_MUL) && (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1);+ kparams->broadcast_weight = broadcast_weight;++ const bool src0_contig = ggml_is_contiguous_1(src0);+ const bool src1_contig = (op == HTP_OP_RMS_NORM_MUL && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true;+ const uint32_t max_block = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1];++ kparams->src0_contig = src0_contig ? 1 : 0;+ kparams->src1_contig = src1_contig ? 1 : 0;++ struct htp_unary_vtcm_layout L;+ uint32_t col_tile = 0;+ uint32_t vtcm_row_per_thread = 0;++ htp_unary_solve_layout(&L, op, src0->ne[0], dst->ne[0],+ op == HTP_OP_RMS_NORM_MUL ? src1->ne[0] : 0,+ max_block, broadcast_weight, n_threads, vtcm_size, elem_size,+ &col_tile, &vtcm_row_per_thread);++ kparams->col_tile = col_tile;+ kparams->vtcm_size = L.total_bytes;+ kparams->block = col_tile ? 0 : vtcm_row_per_thread;++ const uint32_t tiles_per_row = col_tile > 0 ? (src0->ne[0] + col_tile - 1) / col_tile : 1;+ kparams->tiles_per_row = tiles_per_row;+ kparams->div_ne01 = init_fastdiv_values(src0->ne[1]);+ kparams->div_ne02 = init_fastdiv_values(src0->ne[2]);+ kparams->div_ne012 = init_fastdiv_values(src0->ne[1] * src0->ne[2]);+ kparams->div_tpr = init_fastdiv_values(tiles_per_row);++ return (col_tile > 0 || vtcm_row_per_thread > 0) && L.total_bytes <= vtcm_size;+}++static bool htp_kparams_solve_unary(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_unary_kernel_params * kparams+) {+ const htp_op_code opcode = htp_op_remap(op);+ const struct ggml_tensor * src0 = (opcode == HTP_OP_HADAMARD) ? op->src[1] : op->src[0];+ const struct ggml_tensor * src1 = (opcode == HTP_OP_HADAMARD) ? nullptr : op->src[1];+ return htp_kparams_solve_unary_impl(sess, opcode, src0, src1, op, kparams);+}++bool htp_kparams_solve_rms_norm_mul(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * dst,+ struct htp_unary_kernel_params * kparams+) {+ return htp_kparams_solve_unary_impl(sess, HTP_OP_RMS_NORM_MUL, src0, src1, dst, kparams);+}++bool htp_kparams_solve_repeat_get_rows(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * dst,+ struct htp_get_rows_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16 && src0->type != GGML_TYPE_I32 &&+ src0->type != GGML_TYPE_Q1_0 && src0->type != GGML_TYPE_Q2_0 && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 &&+ src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K) {+ return false;+ }++ if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous_rows(src1) || !ggml_is_contiguous_rows(dst)) {+ return false;+ }++ if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) {+ return false;+ }++ if (src0->type == dst->type) {+ if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->type != GGML_TYPE_F16) {+ return false;+ }+ } else if (src0->type == GGML_TYPE_I32) {+ return false;+ } else if (dst->type != GGML_TYPE_F32) {+ return false;+ }++ const uint32_t ne00 = src0->ne[0];+ const uint32_t ne02 = src0->ne[2];+ const uint32_t ne03 = src0->ne[3];++ const uint32_t ne10 = src1->ne[0];+ const uint32_t ne11 = src1->ne[1];+ const uint32_t ne12 = src1->ne[2];+ const uint32_t nr = ne10 * ne11 * ne12;++ const bool tiled = htp_tensor_is_tiled(src0);+ const int wtype = (tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;++ if (tiled) {+ kparams->kernel_type = HTP_GET_ROWS_KERNEL_TILED;+ } else if (src0->type == dst->type) {+ kparams->kernel_type = HTP_GET_ROWS_KERNEL_SAMETYPE;+ } else {+ kparams->kernel_type = HTP_GET_ROWS_KERNEL_FLAT;+ }++ const uint32_t total_tasks = nr;+ const uint32_t min_tasks_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst->nb[1] - 1) / dst->nb[1]));+ const uint32_t max_threads = (std::max)(1u, total_tasks / min_tasks_per_thread);+ const size_t vtcm_size = htp_session_vtcm_size(sess);++ kparams->n_threads = (std::min)(htp_session_n_threads(sess), max_threads);++ struct htp_get_rows_vtcm_layout vtcm_layout = {};+ while (kparams->n_threads > 0) {+ htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, kparams->n_threads);+ if (vtcm_layout.total_bytes <= vtcm_size) {+ break;+ }+ --kparams->n_threads;+ }++ if (kparams->n_threads == 0 && total_tasks > 0) {+ htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, 1);+ }++ kparams->vtcm_size = (total_tasks == 0) ? 0 : vtcm_layout.total_bytes;+ kparams->total_tasks = total_tasks;++ kparams->div_ne10 = ne10 > 0 ? init_fastdiv_values(ne10) : fastdiv_values{0, 0};+ kparams->div_ne10_ne11 = (ne10 * ne11) > 0 ? init_fastdiv_values(ne10 * ne11) : fastdiv_values{0, 0};+ kparams->div_ne02 = ne02 > 0 ? init_fastdiv_values(ne02) : fastdiv_values{0, 0};+ kparams->div_ne03 = ne03 > 0 ? init_fastdiv_values(ne03) : fastdiv_values{0, 0};++ if (total_tasks == 0) {+ return true;+ }+ return kparams->n_threads > 0 && (size_t) kparams->vtcm_size <= vtcm_size;+}++static bool htp_kparams_solve_get_rows(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_get_rows_kernel_params * kparams+) {+ return htp_kparams_solve_repeat_get_rows(sess, op->src[0], op->src[1], op, kparams);+}++static bool htp_kparams_solve_set_rows(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_set_rows_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * src1 = op->src[1];+ const struct ggml_tensor * dst = op->src[2] ? op->src[2] : op;++ if (dst->type == GGML_TYPE_Q8_0 && src0->ne[0] < 32) {+ return false;+ }+ if (src0->type != GGML_TYPE_F32) {+ return false;+ }+ if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) {+ return false;+ }+ if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_Q8_0) {+ return false;+ }++ const uint32_t nr = src0->ne[1];++ kparams->n_threads = (std::min)(htp_session_n_threads(sess), nr);+ kparams->total_tasks = nr;++ struct htp_set_rows_vtcm_layout vtcm_layout;+ htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_threads);+ kparams->vtcm_size = vtcm_layout.total_bytes;++ return vtcm_layout.total_bytes <= htp_session_vtcm_size(sess);+}++static bool htp_kparams_solve_softmax(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_softmax_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * src1 = op->src[1];+ const struct ggml_tensor * src2 = op->src[2];+ const struct ggml_tensor * dst = op;++ if (src2) {+ return false;+ }+ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src0->ne[2] > 512) {+ return false;+ }+ if (src1) {+ if ((src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) ||+ src0->ne[0] != src1->ne[0] || src1->ne[1] < src0->ne[1] ||+ src0->ne[2] % src1->ne[2] != 0 || src0->ne[3] % src1->ne[3] != 0 ||+ !ggml_is_contiguous(src1)) {+ return false;+ }+ }+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {+ return false;+ }++ const int64_t ne0 = src0->ne[0];+ if ((ne0 > 32 && (ne0 & (32 - 1)) != 0) || ne0 > 131072) {+ return false;+ }++ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];+ const size_t dst_data_row_size = op->ne[0] * ggml_type_size(op->type);+ const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_data_row_size - 1) / dst_data_row_size));+ const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread);+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);++ float scale = 1.0f;+ float max_bias = 0.0f;+ memcpy(&scale, &op->op_params[0], sizeof(float));+ memcpy(&max_bias, &op->op_params[1], sizeof(float));++ kparams->scale = scale;+ kparams->max_bias = max_bias;++ const uint32_t n_head = src0->ne[2];+ const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));+ kparams->n_head = n_head;+ kparams->n_head_log2 = n_head_log2;++ if (max_bias > 0.0f && n_head_log2 > 0) {+ kparams->m0 = powf(2.0f, -(max_bias) / n_head_log2);+ kparams->m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);+ } else {+ kparams->m0 = 1.0f;+ kparams->m1 = 1.0f;+ }++ kparams->use_src1 = (src1 != nullptr) ? 1 : 0;+ kparams->use_f16 = (src1 != nullptr && src1->type == GGML_TYPE_F16) ? 1 : 0;++ const uint32_t ne00 = src0->ne[0];+ const uint32_t ne10 = src1 ? src1->ne[0] : 1;++ struct htp_softmax_vtcm_layout layout;+ htp_softmax_vtcm_layout_build(&layout, ne00, ne10, kparams->use_src1 != 0, kparams->use_f16 != 0, n_threads);+ if (layout.total_bytes > htp_session_vtcm_size(sess)) {+ return false;+ }++ kparams->n_threads = n_threads;+ kparams->vtcm_size = (uint32_t) layout.total_bytes;++ if (!kparams->use_src1) {+ kparams->kernel_type = HTP_SOFTMAX_KERNEL_NOMASK;+ } else if (kparams->use_f16) {+ kparams->kernel_type = HTP_SOFTMAX_KERNEL_MASK_F16;+ } else {+ kparams->kernel_type = HTP_SOFTMAX_KERNEL_MASK_F32;+ }++ if (src0->ne[1] > 0) kparams->div_ne01 = init_fastdiv_values(src0->ne[1]);+ if (src0->ne[2] > 0) kparams->div_ne02 = init_fastdiv_values(src0->ne[2]);+ const uint32_t ne12 = src1 ? src1->ne[2] : 1;+ const uint32_t ne13 = src1 ? src1->ne[3] : 1;+ if (ne12 > 0) kparams->div_ne12 = init_fastdiv_values(ne12);+ if (ne13 > 0) kparams->div_ne13 = init_fastdiv_values(ne13);++ return true;+}++static bool htp_kparams_solve_rope(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_rope_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * src1 = op->src[1];+ const struct ggml_tensor * src2 = op->src[2];+ const struct ggml_tensor * dst = op;++ if (!ggml_are_same_shape(src0, dst)) {+ return false;+ }+ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32) {+ return false;+ }+ if (src0->ne[0] <= 0) {+ return false;+ }++ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];+ if (src0_nrows == 0) {+ return false;+ }++ const int32_t * op_params = &op->op_params[0];+ const int n_dims = op_params[1];+ const int mode = op_params[2];+ const int n_offs = op_params[15];++ if (n_dims < 0 || n_dims % 2 != 0) {+ return false;+ }+ if (n_offs < 0 || (n_offs % 32 != 0) || (n_offs + n_dims > src0->ne[0])) {+ return false;+ }++ float freq_base;+ memcpy(&freq_base, op_params + 5, sizeof(float));+ if (freq_base < 0.0f) {+ return false;+ }++ if (mode != GGML_ROPE_TYPE_NORMAL &&+ mode != GGML_ROPE_TYPE_NEOX &&+ mode != GGML_ROPE_TYPE_MROPE &&+ mode != GGML_ROPE_TYPE_VISION &&+ mode != GGML_ROPE_TYPE_IMROPE) {+ return false;+ }++ const bool is_mrope = (mode & GGML_ROPE_TYPE_MROPE) != 0;++ if (mode == GGML_ROPE_TYPE_VISION && (n_dims != (int) (src0->ne[0] / 2) || n_offs != 0)) {+ return false;+ }+ if (is_mrope) {+ const int32_t * sections = op_params + 11;+ if (sections[0] <= 0 && sections[1] <= 0 && sections[2] <= 0) {+ return false;+ }+ }++ const int64_t min_pos_len = (is_mrope || mode == GGML_ROPE_TYPE_VISION) ? src0->ne[2] * 4 : src0->ne[2];+ if (src1->ne[0] < min_pos_len || !ggml_is_contiguous(src1)) {+ return false;+ }++ if (src2) {+ if (src2->type != GGML_TYPE_F32 || !ggml_is_contiguous(src2) || src2->ne[0] < (n_dims / 2)) {+ return false;+ }+ }++ if (src0->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float) ||+ src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) {+ return false;+ }++ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), src0_nrows);+ const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0;++ struct htp_rope_vtcm_layout layout;+ htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads, n_freq_factors);+ if (layout.total_bytes > htp_session_vtcm_size(sess)) {+ return false;+ }++ kparams->n_threads = n_threads;+ kparams->vtcm_size = (uint32_t) layout.total_bytes;+ kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]);+ kparams->div_ne1 = init_fastdiv_values(dst->ne[1]);++ return true;+}++static bool htp_kparams_solve_sum_rows(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_sum_rows_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * dst = op;++ if (src0->type != GGML_TYPE_F32) {+ return false;+ }+ if (op->op == GGML_OP_ARGMAX) {+ if (dst->type != GGML_TYPE_I32) {+ return false;+ }+ } else if (dst->type != GGML_TYPE_F32) {+ return false;+ }+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {+ return false;+ }++ const uint32_t ne00 = src0->ne[0];+ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];+ const uint32_t max_threads_sess = htp_session_n_threads(sess);++ if (op->op == GGML_OP_SUM) {+ const uint32_t total_elems = (uint32_t) ggml_nelements(src0);+ const uint32_t n_threads = (total_elems >= 1024) ? (std::min)(max_threads_sess, (uint32_t) HTP_MAX_NTHREADS) : 1;+ const uint32_t raw_chunk = (total_elems + n_threads - 1) / n_threads;++ kparams->n_threads = n_threads;+ kparams->elems_per_thread = hex_round_up(raw_chunk, 32);+ } else if (op->op == GGML_OP_ARGMAX && src0_nrows <= 1) {+ const uint32_t n_threads = (ne00 >= 1024) ? (std::min)(max_threads_sess, (uint32_t) HTP_MAX_NTHREADS) : 1;+ const uint32_t raw_chunk = (ne00 + n_threads - 1) / n_threads;++ kparams->n_threads = n_threads;+ kparams->elems_per_thread = hex_round_up(raw_chunk, 32);+ } else {+ const uint32_t min_rpt = (src0->nb[1] < 1024) ? 32 : 1;+ const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rpt);++ kparams->n_threads = (std::min)(max_threads_sess, max_threads);+ }++ return true;+}++bool htp_kparams_solve_unary_mul_glu(+ const struct ggml_hexagon_session * sess,+ uint32_t op,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * dst,+ struct htp_glu_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ if (!htp_op_is_glu((htp_op_code) op)) {+ return false;+ }+ if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || dst->type != src0->type) {+ return false;+ }+ if ((op == HTP_OP_GLU_GEGLU_QUICK || op == HTP_OP_GLU_GEGLU_ERF) && src0->type != GGML_TYPE_F32) {+ return false;+ }++ const size_t elem_size = ggml_type_size(src0->type);+ const auto is_valid_src = [elem_size](const struct ggml_tensor * t) {+ return ggml_is_contiguous_1(t) && t->nb[0] == elem_size &&+ (t->ne[1] <= 1 || t->nb[1] >= (size_t) t->ne[0] * elem_size);+ };++ if (!is_valid_src(src0) || !ggml_is_contiguous(dst)) {+ return false;+ }+ if (src1 && (src1->type != src0->type || !ggml_are_same_shape(src0, src1) || !is_valid_src(src1))) {+ return false;+ }++ const size_t dst_row_size = dst->ne[0] * elem_size;+ const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];+ const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size));+ const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread);+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);+ const size_t vtcm_size = htp_session_vtcm_size(sess);++ const size_t row_size_aligned = hex_round_up(dst_row_size, 128);+ const uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * 3 * row_size_aligned));+ if (vtcm_row_per_thread < 2) {+ return false;+ }++ struct htp_glu_vtcm_layout L;+ htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, vtcm_row_per_thread);++ kparams->n_threads = n_threads;+ kparams->vtcm_row_per_thread = L.vtcm_row_per_thread;+ kparams->vtcm_size = (uint32_t) L.total_bytes;++ return true;+}++static bool htp_kparams_solve_glu(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_glu_kernel_params * kparams+) {+ return htp_kparams_solve_unary_mul_glu(sess, htp_op_remap(op), op->src[0], op->src[1], op, kparams);+}++static bool htp_kparams_solve_repeat(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_repeat_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * dst = op;++ if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || src0->type != dst->type) {+ return false;+ }+ if (!ggml_can_repeat(src0, dst) || ggml_is_transposed(src0) || ggml_is_transposed(dst)) {+ return false;+ }++ const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3];+ const size_t dst_row_size = dst->ne[0] * ggml_type_size(dst->type);+ const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_row_size - 1) / dst_row_size));+ const uint32_t max_threads = (std::max)(1u, total_dst_rows / min_rows_per_thread);+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);++ kparams->n_threads = n_threads;+ kparams->nr0 = dst->ne[0] / src0->ne[0];+ kparams->nr1 = dst->ne[1] / src0->ne[1];+ kparams->nr2 = dst->ne[2] / src0->ne[2];+ kparams->nr3 = dst->ne[3] / src0->ne[3];++ kparams->div_ne1 = init_fastdiv_values(dst->ne[1]);+ kparams->div_ne12 = init_fastdiv_values(dst->ne[1] * dst->ne[2]);+ kparams->div_ne01 = init_fastdiv_values(src0->ne[1]);+ kparams->div_ne02 = init_fastdiv_values(src0->ne[2]);+ kparams->div_ne03 = init_fastdiv_values(src0->ne[3]);++ return true;+}++static bool htp_kparams_solve_ssm_conv(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_ssm_conv_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * src1 = op->src[1];+ const struct ggml_tensor * dst = op;++ if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {+ return false;+ }+ if (src0->ne[3] != 1 || src1->ne[2] != 1 || src1->ne[3] != 1 || dst->ne[3] != 1) {+ return false;+ }++ const int64_t d_conv_64 = src1->ne[0];+ const int64_t d_inner_64 = src0->ne[1];+ const int64_t n_t_64 = dst->ne[1];+ const int64_t n_s_64 = dst->ne[2];++ if (d_conv_64 <= 0 || d_conv_64 > 32 || d_inner_64 <= 0) {+ return false;+ }+ if (src0->ne[0] != d_conv_64 - 1 + n_t_64 || src0->ne[1] != d_inner_64 || src0->ne[2] != n_s_64) {+ return false;+ }+ if (src1->ne[0] != d_conv_64 || src1->ne[1] != d_inner_64) {+ return false;+ }+ if (dst->ne[0] != d_inner_64 || dst->ne[1] != n_t_64 || dst->ne[2] != n_s_64) {+ return false;+ }+ if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) {+ return false;+ }+ if (src0->nb[1] != src0->ne[0] * sizeof(float) || src1->nb[1] != src1->ne[0] * sizeof(float)) {+ return false;+ }++ const uint32_t d_conv = (uint32_t) d_conv_64;+ const uint32_t d_inner = (uint32_t) d_inner_64;+ const uint32_t n_t = (uint32_t) n_t_64;+ const uint32_t n_s = (uint32_t) n_s_64;+ const uint32_t ncs = (uint32_t) src0->ne[0];++ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), (d_inner + 31) / 32);+ const size_t vtcm_size = htp_session_vtcm_size(sess);++ kparams->n_threads = n_threads;+ kparams->d_conv = d_conv;+ kparams->d_inner = d_inner;+ kparams->n_t = n_t;+ kparams->n_s = n_s;++ const uint32_t raw_rpt = (d_inner + n_threads - 1) / n_threads;+ const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, 32);++ if (n_t == 1) {+ kparams->d_inner_tile = d_inner_per_thread;+ } else {+ const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128) + 128;+ const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128);+ const uint32_t vtcm_src1_per_thread = src1_raw_bytes + src1_T_bytes;++ const size_t vtcm_budget = (vtcm_size > 0 ? vtcm_size / n_threads : (1024 * 1024));+ const uint32_t src0_block_T = hex_round_up(ncs * 32 * sizeof(float), 128);+ const size_t fixed_bytes = vtcm_src1_per_thread + src0_block_T;+ const size_t avail_for_tiles = vtcm_budget > fixed_bytes ? vtcm_budget - fixed_bytes : (128 * 1024);++ uint32_t target_max_tile = hex_round_up((d_inner_per_thread + 3) / 4, 32);+ target_max_tile = (std::max)(target_max_tile, 32u);+ target_max_tile = (std::min)(target_max_tile, 128u);++ uint32_t d_inner_tile = (uint32_t) (avail_for_tiles / (2 * (ncs + n_t) * sizeof(float)));+ d_inner_tile = (d_inner_tile / 32) * 32;+ if (d_inner_tile == 0) {+ d_inner_tile = 32;+ }+ if (d_inner_tile > target_max_tile) {+ d_inner_tile = target_max_tile;+ }+ if (d_inner_tile > d_inner_per_thread) {+ d_inner_tile = d_inner_per_thread;+ }+ kparams->d_inner_tile = d_inner_tile;+ }++ struct htp_ssm_conv_vtcm_layout L;+ htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_thread, d_conv, n_t, kparams->d_inner_tile, n_threads);++ kparams->vtcm_size = (uint32_t) L.total_bytes;+ kparams->div_d_inner_tile = init_fastdiv_values(kparams->d_inner_tile);++ return L.total_bytes <= vtcm_size;+}++static bool htp_kparams_solve_sort(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_sort_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * dst = op;+ const bool is_top_k = (op->op == GGML_OP_TOP_K);++ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_I32) {+ return false;+ }++ const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];+ const uint32_t ne00 = src0->ne[0];+ const uint32_t k = dst->ne[0];+ const uint32_t n_threads_max = htp_session_n_threads(sess) > 0 ? htp_session_n_threads(sess) : 4;+ const size_t vtcm_budget = htp_session_vtcm_size(sess) > 0 ? htp_session_vtcm_size(sess) : (8 * 1024 * 1024);++ int32_t order = GGML_SORT_ORDER_DESC;+ if (!is_top_k) {+ order = ((const int32_t *) op->op_params)[0];+ }++ struct htp_sort_vtcm_layout layout;+ if (!htp_sort_solve_layout(&layout, ne00, total_rows, k, n_threads_max, vtcm_budget, is_top_k)) {+ return false;+ }++ kparams->n_threads = layout.n_threads;+ kparams->order = (uint32_t) order;+ kparams->is_top_k = is_top_k ? 1 : 0;+ kparams->chunk_elems = layout.chunk_elems;+ kparams->n_chunks = layout.n_chunks;+ kparams->vtcm_size = (uint32_t) layout.total_bytes;+ kparams->merge_elems = layout.merge_elems;+ kparams->n_slots = layout.n_slots;+ kparams->div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);+ kparams->div_ne01_ne02 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[2]));++ return true;+}++static void htp_pool_interior_range(+ uint32_t src_x, uint32_t dst_x, uint32_t kernel_x, uint32_t stride_x, int32_t pad_x,+ uint32_t * ox_lo, uint32_t * ox_hi+) {+ const uint32_t lo_raw = ((uint32_t) pad_x + stride_x - 1) / stride_x;+ *ox_lo = (lo_raw < dst_x) ? lo_raw : dst_x;++ const int32_t numer_hi = (int32_t) src_x - (int32_t) kernel_x + pad_x;+ if (numer_hi < 0) {+ *ox_hi = 0;+ } else {+ const uint32_t hi_raw = (uint32_t) numer_hi / stride_x + 1;+ *ox_hi = (hi_raw < dst_x) ? hi_raw : dst_x;+ }+ if (*ox_hi < *ox_lo) {+ *ox_hi = *ox_lo;+ }+}++static bool htp_kparams_solve_pool_2d(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_pool_2d_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * dst = op;+ const bool is_pool_1d = (op->op == GGML_OP_POOL_1D);++ const int32_t * params = dst->op_params;+ if (params[0] != GGML_OP_POOL_AVG && params[0] != GGML_OP_POOL_MAX) {+ return false;+ }+ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {+ return false;+ }+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {+ return false;+ }+ if (ggml_nbytes(src0) > UINT32_MAX - 256 || ggml_nbytes(dst) > UINT32_MAX - 256) {+ return false;+ }++ const int32_t kernel_x = params[1];+ const int32_t kernel_y = is_pool_1d ? 1 : params[2];+ const int32_t stride_x = params[is_pool_1d ? 2 : 3];+ const int32_t stride_y = is_pool_1d ? 1 : params[4];+ const int32_t pad_x = params[is_pool_1d ? 3 : 5];+ const int32_t pad_y = is_pool_1d ? 0 : params[6];++ if (kernel_x <= 0 || kernel_y <= 0 || stride_x <= 0 || stride_y <= 0 ||+ pad_x < 0 || pad_y < 0) {+ return false;+ }++ const uint32_t src_x = (uint32_t) src0->ne[0];+ const uint32_t src_y = is_pool_1d ? 1 : (uint32_t) src0->ne[1];+ const uint32_t dst_x = (uint32_t) dst->ne[0];+ const uint32_t dst_y = is_pool_1d ? 1 : (uint32_t) dst->ne[1];+ const uint32_t dst_plane_bytes = dst_x * dst_y * sizeof(float);+ const uint32_t planes = (uint32_t) (is_pool_1d+ ? (uint64_t) src0->ne[1] * (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3]+ : (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3]);+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), planes > 0 ? planes : 1);+ const uint32_t planes_per_chunk = (dst_plane_bytes > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_plane_bytes, HEX_L2_LINE_SIZE)) : 1;++ struct htp_pool_vtcm_layout layout;+ if (!htp_pool_solve_layout(&layout, src_x, src_y, dst_x, dst_y, n_threads, htp_session_vtcm_size(sess))) {+ return false;+ }++ kparams->src_x = src_x;+ kparams->src_y = src_y;+ kparams->dst_x = dst_x;+ kparams->dst_y = dst_y;+ kparams->kernel_x = (uint32_t) kernel_x;+ kparams->kernel_y = (uint32_t) kernel_y;+ kparams->stride_x = (uint32_t) stride_x;+ kparams->stride_y = (uint32_t) stride_y;+ kparams->pad_x = pad_x;+ kparams->pad_y = pad_y;+ kparams->n_threads = n_threads;+ kparams->planes = planes;+ kparams->planes_per_chunk = planes_per_chunk;+ kparams->pool_op = (uint32_t) params[0];++ const bool exact_tiling = (kparams->pad_x == 0 && kparams->pad_y == 0 &&+ kparams->stride_x == kparams->kernel_x && kparams->stride_y == kparams->kernel_y);+ const bool fast_ok = exact_tiling && (kparams->kernel_x == 1 || kparams->kernel_x == 2);+ const bool narrow_ok = fast_ok && ((uint64_t) kparams->dst_x * kparams->kernel_x <= 32);+ const bool global_ok = exact_tiling &&+ kparams->kernel_x == kparams->src_x && kparams->kernel_y == kparams->src_y &&+ kparams->dst_x == 1 && kparams->dst_y == 1;+ const bool block_ok = exact_tiling &&+ kparams->kernel_y == 1 && kparams->stride_y == 1 &&+ kparams->kernel_x >= 32;++ if (global_ok) {+ kparams->kernel_type = HTP_POOL_KERNEL_GLOBAL;+ } else if (block_ok) {+ kparams->kernel_type = HTP_POOL_KERNEL_BLOCK;+ } else if (narrow_ok) {+ kparams->kernel_type = HTP_POOL_KERNEL_HVX_NARROW;+ } else if (fast_ok) {+ kparams->kernel_type = HTP_POOL_KERNEL_HVX;+ } else {+ kparams->kernel_type = HTP_POOL_KERNEL_GENERAL;+ }++ kparams->avg_divide_count = (is_pool_1d && kparams->pool_op == GGML_OP_POOL_AVG) ? 1 : 0;+ htp_pool_interior_range(kparams->src_x, kparams->dst_x, kparams->kernel_x,+ kparams->stride_x, kparams->pad_x,+ &kparams->ox_lo, &kparams->ox_hi);+ kparams->inv_kernel_area = 1.0f / (float) (kparams->kernel_x * kparams->kernel_y);++ return true;+}++static bool htp_kparams_solve_concat(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_concat_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));+ kparams->kernel_type = HTP_CONCAT_KERNEL_UNSUPPORTED;++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * src1 = op->src[1];+ const struct ggml_tensor * dst = op;++ if (!src0 || !src1 || !dst) {+ return false;+ }++ int dim = ((const int32_t *) op->op_params)[0];+ if (dim < 0 || dim >= GGML_MAX_DIMS) {+ return false;+ }+ kparams->dim = dim;++ if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_I32) {+ return false;+ }+ if (src0->type != dst->type || src1->type != dst->type) {+ return false;+ }++ const uint32_t type_size = ggml_type_size(dst->type);+ const uint32_t n_threads = htp_session_n_threads(sess);+ const uint32_t mdev_count = htp_session_mdev_count(sess);+ const size_t vtcm_size = htp_session_vtcm_size(sess);++ for (int d = 0; d < GGML_MAX_DIMS; d++) {+ const int64_t ne_d = (d == dim) ? src0->ne[d] + src1->ne[d] : src0->ne[d];+ if (dst->ne[d] != ne_d || (d != dim && src1->ne[d] != dst->ne[d])) {+ return false;+ }+ }++ const uint32_t max_gather_ne0 = 128 / type_size;+ const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);+ const bool b_dense = (src1->nb[0] == type_size);+ const bool gather_strides_ok =+ src0->nb[1] >= (size_t) src0->ne[0] * type_size &&+ (b_dense ? src1->nb[1] >= (size_t) src1->ne[0] * type_size+ : src1->nb[0] >= (size_t) src1->ne[1] * type_size);++ if (dim == 0 &&+ (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16) &&+ dst->ne[0] <= max_gather_ne0 &&+ dst->nb[0] == type_size && dst->nb[1] == (size_t) dst->ne[0] * type_size &&+ src0->nb[0] == type_size &&+ (src1->nb[0] == type_size || src1->nb[1] == type_size) &&+ gather_strides_ok &&+ htp_gather_rows_tensor_outer_contiguous(dst) &&+ htp_gather_rows_tensor_outer_contiguous(src0) &&+ htp_gather_rows_tensor_outer_contiguous(src1)) {++ if (htp_gather_rows_solve_layout(&kparams->u.gather,+ (uint32_t) src0->ne[0], (uint32_t) src0->nb[1],+ (uint32_t) src1->ne[0], (uint32_t) src1->nb[1], b_dense,+ type_size, total_rows,+ n_threads, mdev_count, vtcm_size)) {+ kparams->kernel_type = HTP_CONCAT_KERNEL_GATHER_ROWS;+ kparams->n_threads = (uint8_t) n_threads;+ kparams->vtcm_size = kparams->u.gather.slice * n_threads;+ return true;+ }+ }++ const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size);+ if (dma_strides_ok) {+ kparams->kernel_type = HTP_CONCAT_KERNEL_REGULAR;+ kparams->n_threads = 1;+ return true;+ }++ const bool is_src1_transposed = (src1->nb[0] > src1->nb[1]);+ const bool is_src0_transposed = (src0->nb[0] > src0->nb[1]);+ const bool transposed_rows_ok = (src0->nb[0] == type_size && src1->nb[1] == type_size && dst->nb[0] == type_size);++ if (dim == 0 && is_src1_transposed && !is_src0_transposed && transposed_rows_ok &&+ (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16)) {++ const uint32_t nth = n_threads > 0 ? n_threads : 8;+ struct htp_concat_transposed_vtcm_layout layout;+ htp_concat_transposed_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src1->ne[0], type_size, nth);++ if (vtcm_size > 0 && layout.total_bytes > vtcm_size) {+ return false;+ }++ kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED;+ kparams->n_threads = nth;+ kparams->vtcm_size = layout.total_bytes;+ kparams->u.transposed.div_ne2 = init_fastdiv_values((uint32_t) dst->ne[2]);+ return true;+ }++ return false;+}++static bool htp_kparams_solve_cpy(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_copy_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));+ kparams->kernel_type = HTP_COPY_KERNEL_UNSUPPORTED;++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * dst = op;++ if (!src0 || !dst) {+ return false;+ }++ if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_I32) {+ return false;+ }+ if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_I32) {+ return false;+ }++ const int64_t nelem_src = ggml_nelements(src0);+ const int64_t nelem_dst = ggml_nelements(dst);+ if (nelem_src != nelem_dst || nelem_src < 0) {+ return false;+ }++ const uint32_t src_type_size = ggml_type_size(src0->type);+ const uint32_t dst_type_size = ggml_type_size(dst->type);++ kparams->src0_type_size = (uint8_t) src_type_size;+ kparams->dst_type_size = (uint8_t) dst_type_size;+ kparams->total_elems = (uint32_t) nelem_src;++ if (nelem_src == 0) {+ kparams->kernel_type = HTP_COPY_KERNEL_1D_CONTIG;+ return true;+ }++ if (nelem_src == 1) {+ if (src0->type == dst->type) {+ kparams->kernel_type = HTP_COPY_KERNEL_SCALAR;+ return true;+ }+ if ((src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) ||+ (src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32) ||+ (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) ||+ (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32)) {+ kparams->kernel_type = HTP_COPY_KERNEL_SCALAR;+ return true;+ }+ return false;+ }++ const bool sametype = (src0->type == dst->type);+ const bool same_extents = ggml_are_same_shape(src0, dst);++ const bool transposed = (src0->nb[0] > src0->nb[1]) || (dst->nb[0] > dst->nb[1]) ||+ (src0->nb[0] != src_type_size) || (dst->nb[0] != dst_type_size) ||+ (src0->nb[1] < (size_t) src0->ne[0] * src_type_size) || (dst->nb[1] < (size_t) dst->ne[0] * dst_type_size);+ const bool sameshape = same_extents && !transposed;++ const bool src_is_contiguous = ggml_is_contiguous(src0);+ const bool dst_is_contiguous = ggml_is_contiguous(dst);++ const uint32_t n_threads = htp_session_n_threads(sess);+ const uint32_t mdev_count = htp_session_mdev_count(sess);+ const size_t vtcm_size = htp_session_vtcm_size(sess);++ if (sametype) {+ if (src_is_contiguous && dst_is_contiguous) {+ kparams->kernel_type = HTP_COPY_KERNEL_1D_CONTIG;+ return true;+ }++ const uint32_t max_gather_ne0 = 128 / src_type_size;+ const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);+ if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) &&+ dst_is_contiguous && !src_is_contiguous &&+ src0->nb[0] == src_type_size && src0->ne[0] <= max_gather_ne0 &&+ src0->nb[1] >= (size_t) src0->ne[0] * src_type_size &&+ total_rows > 1 &&+ htp_gather_rows_tensor_outer_contiguous(src0)) {++ if (htp_gather_rows_solve_layout(&kparams->u.gather,+ (uint32_t) src0->ne[0], (uint32_t) src0->nb[1],+ 0, 0, false,+ src_type_size, total_rows,+ n_threads, mdev_count, vtcm_size)) {+ kparams->kernel_type = HTP_COPY_KERNEL_GATHER_ROWS;+ kparams->n_threads = (uint8_t) n_threads;+ kparams->vtcm_size = kparams->u.gather.slice * n_threads;+ return true;+ }+ }++ if (sameshape) {+ kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE;+ kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);+ return true;+ }++ kparams->kernel_type = HTP_COPY_KERNEL_RESHAPE;+ kparams->n_threads = n_threads > 0 ? (uint8_t) n_threads : 4;+ kparams->u.reshape.div_ne0 = init_fastdiv_values((uint32_t) dst->ne[0]);+ kparams->u.reshape.div_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[1] * dst->ne[0]));+ kparams->u.reshape.div_ne2_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[2] * dst->ne[1] * dst->ne[0]));+ kparams->u.reshape.div_ne00 = init_fastdiv_values((uint32_t) src0->ne[0]);+ kparams->u.reshape.div_ne01_ne00 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[0]));+ kparams->u.reshape.div_ne02_ne01_ne00 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1] * src0->ne[0]));+ return true;+ }++ if (!sameshape) {+ return false;+ }++ const bool valid_conversion = (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) ||+ (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) ||+ (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) ||+ (src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32);+ if (!valid_conversion) {+ return false;+ }++ const uint32_t nth = n_threads > 0 ? n_threads : 4;+ struct htp_copy_convert_vtcm_layout layout;+ htp_copy_convert_solve_layout(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1], src_type_size, dst_type_size, nth);++ if (vtcm_size > 0 && layout.total_bytes > vtcm_size) {+ return false;+ }++ kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_CONVERT;+ kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);+ kparams->n_threads = (uint8_t) nth;+ kparams->vtcm_size = (uint32_t) layout.total_bytes;+ kparams->u.convert.blk_rows = layout.blk_rows;+ kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);+ kparams->u.convert.div_ne02_ne01 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1]));++ return true;+}++static bool im2col_3d_calc_output_size(+ uint32_t input, uint32_t kernel, int32_t stride, int32_t padding, int32_t dilation, uint32_t * output+) {+ if (stride <= 0 || padding < 0 || dilation <= 0 || kernel == 0) {+ return false;+ }++ const int64_t numerator = (int64_t) input + 2LL * padding - (int64_t) dilation * (kernel - 1) - 1;+ if (numerator < 0) {+ return false;+ }++ const uint64_t size = (uint64_t) (numerator / stride) + 1;+ if (size == 0 || size > UINT32_MAX) {+ return false;+ }++ *output = (uint32_t) size;+ return true;+}++static bool htp_kparams_solve_im2col(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_im2col_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));+ kparams->kernel_type = HTP_IM2COL_KERNEL_UNSUPPORTED;++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * src1 = op->src[1];+ const struct ggml_tensor * dst = op;++ if (!src0 || !src1 || !dst) {+ return false;+ }+ if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) {+ return false;+ }+ if (src1->nb[0] != sizeof(float) || !ggml_is_contiguous(dst)) {+ return false;+ }++ const int32_t * op_params = (const int32_t *) op->op_params;+ const int32_t s0 = op_params[0];+ const int32_t s1 = op_params[1];+ const int32_t p0 = op_params[2];+ const int32_t p1 = op_params[3];+ const int32_t d0 = op_params[4];+ const int32_t d1 = op_params[5];+ const bool is_2D = (op_params[6] == 1);++ if (s0 <= 0 || d0 <= 0 || p0 < 0) {+ return false;+ }+ if (is_2D && (s1 <= 0 || d1 <= 0 || p1 < 0)) {+ return false;+ }++ const uint32_t N = is_2D ? (uint32_t) src1->ne[3] : (uint32_t) src1->ne[2];+ const uint32_t IC = is_2D ? (uint32_t) src1->ne[2] : (uint32_t) src1->ne[1];+ const uint32_t IH = is_2D ? (uint32_t) src1->ne[1] : 1;+ const uint32_t IW = (uint32_t) src1->ne[0];+ const uint32_t KH = is_2D ? (uint32_t) src0->ne[1] : 1;+ const uint32_t KW = (uint32_t) src0->ne[0];+ const uint32_t OH = is_2D ? (uint32_t) dst->ne[2] : 1;+ const uint32_t OW = (uint32_t) dst->ne[1];++ if (KW == 0 || KH == 0 || IW == 0 || IH == 0 || IC == 0 || OW == 0 || OH == 0 || N == 0) {+ return false;+ }++ const uint64_t patch_stride_64 = (uint64_t) IC * KH * KW;+ const uint64_t total_rows_64 = (uint64_t) N * OH;+ if (patch_stride_64 > UINT32_MAX || total_rows_64 > UINT32_MAX) {+ return false;+ }+ const uint32_t patch_stride = (uint32_t) patch_stride_64;+ const uint32_t total_rows = (uint32_t) total_rows_64;++ if (dst->ne[0] != patch_stride || dst->ne[1] != OW || (is_2D && dst->ne[2] != OH)) {+ return false;+ }++ const uint32_t dst_elem = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);++ bool is_patchembed = ggml_is_contiguous(src1) && (s0 == (int32_t) KW && p0 == 0 && d0 == 1);+ if (is_2D) {+ is_patchembed = is_patchembed && (s1 == (int32_t) KH && p1 == 0 && d1 == 1);+ }++ const uint32_t max_threads = (sess && htp_session_n_threads(sess) > 0) ? htp_session_n_threads(sess) : 8;+ const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads;+ const uint32_t vtcm_size = (sess && htp_session_vtcm_size(sess) > 0) ? (uint32_t) htp_session_vtcm_size(sess) : 0;+ const uint32_t row_size = (uint32_t) dst->nb[2];+ const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;++ if (is_patchembed) {+ const uint64_t src_row_bytes_raw = (uint64_t) IC * KH * IW * sizeof(float);+ const uint64_t dst_row_bytes_raw = (uint64_t) OW * patch_stride * dst_elem;+ if (src_row_bytes_raw <= UINT32_MAX && dst_row_bytes_raw <= UINT32_MAX) {+ const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);+ const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);++ struct htp_im2col_vtcm_layout layout;+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);++ if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {+ kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;+ kparams->n_threads = (uint8_t) n_threads;+ kparams->vtcm_size = (uint32_t) layout.total_bytes;+ kparams->src_row_bytes = src_row_bytes;+ kparams->dst_row_bytes = dst_row_bytes;+ kparams->owb = OW;+ kparams->wb = IW;+ kparams->nbpr = 1;+ kparams->N = N;+ kparams->IC = IC;+ kparams->ID = 1;+ kparams->IH = IH;+ kparams->IW = IW;+ kparams->KD = 1;+ kparams->KH = KH;+ kparams->KW = KW;+ kparams->OD = 1;+ kparams->OH = OH;+ kparams->OW = OW;+ kparams->patch_stride = patch_stride;+ kparams->total_rows = total_rows;+ kparams->rows_per_chunk = rows_per_chunk;+ kparams->div_OH = init_fastdiv_values(OH);+ kparams->div_OD = init_fastdiv_values(1);+ return true;+ }+ }+ }++ for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) {+ const uint64_t Wb = (uint64_t) (owb - 1) * s0 + (uint64_t) (KW - 1) * d0 + 1;+ if (Wb > UINT32_MAX) {+ if (owb == 1) break;+ continue;+ }++ const uint64_t src_row_bytes_raw = (uint64_t) IC * KH * Wb * sizeof(float);+ const uint64_t dst_row_bytes_raw = (uint64_t) owb * patch_stride * dst_elem;+ if (src_row_bytes_raw > UINT32_MAX || dst_row_bytes_raw > UINT32_MAX) {+ if (owb == 1) break;+ continue;+ }++ const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);+ const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);++ struct htp_im2col_vtcm_layout layout;+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);++ if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {+ kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA;+ kparams->n_threads = (uint8_t) n_threads;+ kparams->vtcm_size = (uint32_t) layout.total_bytes;+ kparams->src_row_bytes = src_row_bytes;+ kparams->dst_row_bytes = dst_row_bytes;+ kparams->owb = owb;+ kparams->wb = (uint32_t) Wb;+ kparams->nbpr = (OW + owb - 1) / owb;+ kparams->N = N;+ kparams->IC = IC;+ kparams->ID = 1;+ kparams->IH = IH;+ kparams->IW = IW;+ kparams->KD = 1;+ kparams->KH = KH;+ kparams->KW = KW;+ kparams->OD = 1;+ kparams->OH = OH;+ kparams->OW = OW;+ kparams->patch_stride = patch_stride;+ kparams->total_rows = total_rows;+ kparams->rows_per_chunk = rows_per_chunk;+ kparams->div_OH = init_fastdiv_values(OH);+ kparams->div_OD = init_fastdiv_values(1);+ return true;+ }+ if (owb == 1) break;+ }++ return false;+}++static bool htp_kparams_solve_im2col_3d(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_im2col_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));+ kparams->kernel_type = HTP_IM2COL_KERNEL_UNSUPPORTED;++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * src1 = op->src[1];+ const struct ggml_tensor * dst = op;++ if (!src0 || !src1 || !dst) {+ return false;+ }+ if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) {+ return false;+ }+ if (src1->nb[0] != sizeof(float) || !ggml_is_contiguous(dst)) {+ return false;+ }++ const int32_t * params = (const int32_t *) op->op_params;+ const int32_t s0 = params[0], s1 = params[1], s2 = params[2];+ const int32_t p0 = params[3], p1 = params[4], p2 = params[5];+ const int32_t d0 = params[6], d1 = params[7], d2 = params[8];+ const int32_t ic = params[9];++ if (s0 <= 0 || s1 <= 0 || s2 <= 0 ||+ p0 < 0 || p1 < 0 || p2 < 0 ||+ d0 <= 0 || d1 <= 0 || d2 <= 0 || ic <= 0) {+ return false;+ }+ if (src1->ne[3] % ic != 0) {+ return false;+ }++ const uint32_t IC = (uint32_t) ic;+ const uint32_t N = (uint32_t) (src1->ne[3] / IC);+ if (N == 0) {+ return false;+ }++ uint32_t OW, OH, OD;+ if (!im2col_3d_calc_output_size(src1->ne[0], src0->ne[0], s0, p0, d0, &OW) ||+ !im2col_3d_calc_output_size(src1->ne[1], src0->ne[1], s1, p1, d1, &OH) ||+ !im2col_3d_calc_output_size(src1->ne[2], src0->ne[2], s2, p2, d2, &OD)) {+ return false;+ }++ const uint32_t KD = (uint32_t) src0->ne[2];+ const uint32_t KH = (uint32_t) src0->ne[1];+ const uint32_t KW = (uint32_t) src0->ne[0];+ const uint32_t ID = (uint32_t) src1->ne[2];+ const uint32_t IH = (uint32_t) src1->ne[1];+ const uint32_t IW = (uint32_t) src1->ne[0];++ const uint64_t patch_stride_64 = (uint64_t) IC * KD * KH * KW;+ const uint64_t total_rows_64 = (uint64_t) N * OD * OH;+ const uint64_t dst_ne3_64 = (uint64_t) OD * N;++ if (patch_stride_64 > UINT32_MAX || total_rows_64 > UINT32_MAX || dst_ne3_64 > UINT32_MAX) {+ return false;+ }+ const uint32_t patch_stride = (uint32_t) patch_stride_64;+ const uint32_t total_rows = (uint32_t) total_rows_64;+ const uint32_t dst_ne3 = (uint32_t) dst_ne3_64;++ if (dst->ne[0] != patch_stride || dst->ne[1] != OW || dst->ne[2] != OH || dst->ne[3] != dst_ne3) {+ return false;+ }++ const uint32_t dst_elem = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);++ const bool is_patchembed = ggml_is_contiguous(src1) &&+ (s0 == (int32_t) KW && s1 == (int32_t) KH && s2 == (int32_t) KD &&+ p0 == 0 && p1 == 0 && p2 == 0 &&+ d0 == 1 && d1 == 1 && d2 == 1);++ const uint32_t max_threads = (sess && htp_session_n_threads(sess) > 0) ? htp_session_n_threads(sess) : 8;+ const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads;+ const uint32_t vtcm_size = (sess && htp_session_vtcm_size(sess) > 0) ? (uint32_t) htp_session_vtcm_size(sess) : 0;+ const uint32_t row_size = (uint32_t) dst->nb[2];+ const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;++ if (is_patchembed) {+ const uint64_t src_row_bytes_raw = (uint64_t) IC * KD * KH * IW * sizeof(float);+ const uint64_t dst_row_bytes_raw = (uint64_t) OW * patch_stride * dst_elem;+ if (src_row_bytes_raw <= UINT32_MAX && dst_row_bytes_raw <= UINT32_MAX) {+ const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);+ const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);++ struct htp_im2col_vtcm_layout layout;+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);++ if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {+ kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;+ kparams->n_threads = (uint8_t) n_threads;+ kparams->vtcm_size = (uint32_t) layout.total_bytes;+ kparams->src_row_bytes = src_row_bytes;+ kparams->dst_row_bytes = dst_row_bytes;+ kparams->owb = OW;+ kparams->wb = IW;+ kparams->nbpr = 1;+ kparams->N = N;+ kparams->IC = IC;+ kparams->ID = ID;+ kparams->IH = IH;+ kparams->IW = IW;+ kparams->KD = KD;+ kparams->KH = KH;+ kparams->KW = KW;+ kparams->OD = OD;+ kparams->OH = OH;+ kparams->OW = OW;+ kparams->patch_stride = patch_stride;+ kparams->total_rows = total_rows;+ kparams->rows_per_chunk = rows_per_chunk;+ kparams->div_OH = init_fastdiv_values(OH);+ kparams->div_OD = init_fastdiv_values(OD);+ return true;+ }+ }+ }++ for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) {+ const uint64_t Wb = (uint64_t) (owb - 1) * s0 + (uint64_t) (KW - 1) * d0 + 1;+ if (Wb > UINT32_MAX) {+ if (owb == 1) break;+ continue;+ }++ const uint64_t src_row_bytes_raw = (uint64_t) IC * KD * KH * Wb * sizeof(float);+ const uint64_t dst_row_bytes_raw = (uint64_t) owb * patch_stride * dst_elem;+ if (src_row_bytes_raw > UINT32_MAX || dst_row_bytes_raw > UINT32_MAX) {+ if (owb == 1) break;+ continue;+ }++ const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);+ const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);++ struct htp_im2col_vtcm_layout layout;+ htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);++ if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {+ kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA;+ kparams->n_threads = (uint8_t) n_threads;+ kparams->vtcm_size = (uint32_t) layout.total_bytes;+ kparams->src_row_bytes = src_row_bytes;+ kparams->dst_row_bytes = dst_row_bytes;+ kparams->owb = owb;+ kparams->wb = (uint32_t) Wb;+ kparams->nbpr = (OW + owb - 1) / owb;+ kparams->N = N;+ kparams->IC = IC;+ kparams->ID = ID;+ kparams->IH = IH;+ kparams->IW = IW;+ kparams->KD = KD;+ kparams->KH = KH;+ kparams->KW = KW;+ kparams->OD = OD;+ kparams->OH = OH;+ kparams->OW = OW;+ kparams->patch_stride = patch_stride;+ kparams->total_rows = total_rows;+ kparams->rows_per_chunk = rows_per_chunk;+ kparams->div_OH = init_fastdiv_values(OH);+ kparams->div_OD = init_fastdiv_values(OD);+ return true;+ }+ if (owb == 1) break;+ }++ return false;+}++static bool htp_kparams_solve_pad(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_pad_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * dst = op;++ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {+ return false;+ }++ const int32_t lp0 = ((const int32_t *) op->op_params)[0];+ const int32_t rp0 = ((const int32_t *) op->op_params)[1];+ const bool circular = (((const int32_t *) op->op_params)[8] != 0);++ if (circular && (lp0 > src0->ne[0] || rp0 > src0->ne[0])) {+ return false;+ }++ const uint32_t type_size = (uint32_t) ggml_type_size(src0->type);+ const uint32_t total_dst_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);+ const uint32_t max_threads = (std::max)(1u, total_dst_rows);+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);++ struct htp_pad_vtcm_layout layout;+ htp_pad_vtcm_layout_build(&layout, (uint32_t) src0->ne[0] * type_size, (uint32_t) dst->ne[0] * type_size, n_threads);++ const bool use_dma = (src0->nb[0] == type_size) && (src0->ne[0] >= 512) &&+ (htp_session_vtcm_size(sess) >= layout.total_bytes);++ kparams->n_threads = n_threads;+ kparams->total_rows = total_dst_rows;+ kparams->use_dma = use_dma ? 1 : 0;+ kparams->elem_size = type_size;+ kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0;++ if (circular && use_dma) {+ kparams->kernel_type = HTP_PAD_KERNEL_HVX_CIRCULAR_DMA;+ } else if (circular) {+ kparams->kernel_type = HTP_PAD_KERNEL_HVX_CIRCULAR;+ } else if (use_dma) {+ kparams->kernel_type = HTP_PAD_KERNEL_HVX_DMA;+ } else {+ kparams->kernel_type = HTP_PAD_KERNEL_HVX;+ }++ kparams->div_ne1 = init_fastdiv_values((uint32_t) dst->ne[1]);+ kparams->div_ne1_ne2 = init_fastdiv_values((uint32_t) (dst->ne[1] * dst->ne[2]));+ kparams->div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);+ kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]);+ kparams->div_ne03 = init_fastdiv_values((uint32_t) src0->ne[3]);++ return true;+}++static bool htp_kparams_solve_cumsum(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_cumsum_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * dst = op;++ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {+ return false;+ }+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {+ return false;+ }++ const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);+ const uint32_t max_threads = (std::max)(1u, total_rows);+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);++ struct htp_cumsum_vtcm_layout layout;+ htp_cumsum_vtcm_layout_build(&layout, (uint32_t) src0->nb[1], (uint32_t) dst->nb[1], n_threads);++ const bool use_dma = (htp_session_vtcm_size(sess) >= layout.total_bytes);++ kparams->n_threads = n_threads;+ kparams->total_rows = total_rows;+ kparams->use_dma = use_dma ? 1 : 0;+ kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0;++ return true;+}++static bool htp_kparams_solve_fill(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_fill_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * dst = op;++ if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) {+ return false;+ }+ if (!ggml_is_contiguous_rows(dst) || ggml_is_permuted(dst)) {+ return false;+ }++ const uint32_t ne1 = (uint32_t) dst->ne[1];+ const uint32_t ne2 = (uint32_t) dst->ne[2];+ const uint32_t ne3 = (uint32_t) dst->ne[3];+ const uint32_t nb1 = (uint32_t) dst->nb[1];++ const uint32_t total_rows = ne1 * ne2 * ne3;+ const uint32_t max_threads = (std::max)(1u, total_rows);+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);+ const uint32_t elem_size = (uint32_t) ggml_type_size(dst->type);+ const uint32_t rows_per_chunk = (nb1 > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(nb1, HEX_L2_LINE_SIZE)) : 1;++ kparams->n_threads = n_threads;+ kparams->total_rows = total_rows;+ kparams->elem_size = elem_size;+ kparams->kernel_type = ggml_is_contiguous(dst) ? HTP_FILL_KERNEL_CONTIGUOUS : HTP_FILL_KERNEL_STRIDED;+ kparams->rows_per_chunk = rows_per_chunk;+ kparams->div_ne1 = init_fastdiv_values(ne1);+ kparams->div_ne1_ne2 = init_fastdiv_values(ne1 * ne2);++ return true;+}++static bool htp_kparams_solve_roll(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_roll_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * dst = op;++ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {+ return false;+ }+ if (!ggml_are_same_shape(src0, dst)) {+ return false;+ }+ if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous(dst)) {+ return false;+ }++ const uint32_t ne0 = (uint32_t) dst->ne[0];+ const uint32_t ne1 = (uint32_t) dst->ne[1];+ const uint32_t ne2 = (uint32_t) dst->ne[2];+ const uint32_t ne3 = (uint32_t) dst->ne[3];++ const int32_t s0 = ((const int32_t *) op->op_params)[0];+ uint32_t i0_src0 = 0;+ if (-s0 < 0) {+ i0_src0 = (uint32_t) (-s0 + (int32_t) ne0);+ } else if ((uint32_t) (-s0) >= ne0) {+ i0_src0 = (uint32_t) (-s0) - ne0;+ } else {+ i0_src0 = (uint32_t) (-s0);+ }++ const uint32_t total_rows = ne1 * ne2 * ne3;+ const uint32_t max_threads = (std::max)(1u, total_rows);+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);++ kparams->n_threads = n_threads;+ kparams->total_rows = total_rows;+ kparams->i0_src0 = i0_src0;+ kparams->n0 = ne0 - i0_src0;++ if (htp_session_mdev_count(sess) <= 1) {+ if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) {+ kparams->kernel_type = HTP_ROLL_KERNEL_DMA_CONTIGUOUS;+ } else {+ kparams->kernel_type = HTP_ROLL_KERNEL_DMA_STRIDED;+ }+ } else {+ kparams->kernel_type = HTP_ROLL_KERNEL_HVX;+ }++ kparams->div_ne1 = init_fastdiv_values(ne1);+ kparams->div_ne2_ne1 = init_fastdiv_values(ne2 * ne1);++ return true;+}++static bool htp_kparams_solve_diag(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_diag_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * dst = op;++ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {+ return false;+ }+ if (src0->ne[1] != 1 || dst->ne[0] != dst->ne[1] || dst->ne[0] != src0->ne[0]) {+ return false;+ }+ if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous_rows(dst) ||+ ggml_is_permuted(src0) || ggml_is_permuted(dst)) {+ return false;+ }++ const uint32_t total_batches = (uint32_t) (src0->ne[2] * src0->ne[3]);+ const uint32_t max_threads = (std::max)(1u, total_batches);+ const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads);+ const uint32_t dst_batch_size = (uint32_t) (dst->ne[1] * dst->nb[1]);++ bool can_split = (dst->ne[0] == 1 || dst->nb[0] == sizeof(float)) && !ggml_is_permuted(dst);+ uint32_t batches_per_chunk = 1;+ if (can_split) {+ if (dst->ne[2] > 1 && (dst->nb[2] & (HEX_L2_LINE_SIZE - 1)) == 0 &&+ (dst->ne[3] <= 1 || (dst->nb[3] & (HEX_L2_LINE_SIZE - 1)) == 0)) {+ batches_per_chunk = 1;+ } else if (dst->nb[2] == dst_batch_size &&+ (dst->ne[3] <= 1 || dst->nb[3] == dst->nb[2] * dst->ne[2])) {+ batches_per_chunk = (dst_batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_batch_size, HEX_L2_LINE_SIZE)) : 1;+ } else {+ can_split = false;+ }+ }++ const uint32_t src_batch_size = (uint32_t) (src0->ne[0] * sizeof(float));+ const uint32_t dst_row_size = (uint32_t) (dst->ne[0] * sizeof(float));++ struct htp_diag_vtcm_layout layout;+ htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_threads);++ const bool use_dma = (htp_session_vtcm_size(sess) >= layout.total_bytes);++ kparams->n_threads = n_threads;+ kparams->total_batches = total_batches;+ kparams->use_dma = use_dma ? 1 : 0;+ kparams->can_split = can_split ? 1 : 0;+ kparams->batches_per_chunk = batches_per_chunk;+ kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0;+ kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]);++ return true;+}++static bool htp_kparams_solve_solve_tri(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ struct htp_solve_tri_kernel_params * kparams+) {+ memset(kparams, 0, sizeof(*kparams));++ const struct ggml_tensor * src0 = op->src[0];+ const struct ggml_tensor * src1 = op->src[1];+ const struct ggml_tensor * dst = op;++ if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {+ return false;+ }+ if (src0->ne[0] != src0->ne[1] || src0->ne[1] != src1->ne[1] ||+ src0->ne[2] != src1->ne[2] || src0->ne[3] != src1->ne[3] ||+ !ggml_are_same_shape(src1, dst)) {+ return false;+ }++ const uint32_t max_threads_sess = htp_session_n_threads(sess);+ const uint32_t k = (uint32_t) src1->ne[0];+ const uint32_t col_block = 32;+ const uint32_t k_chunks = (k + col_block - 1) / col_block;+ const uint32_t total_batches = (uint32_t) (src0->ne[2] * src0->ne[3]);+ const bool batched = total_batches >= max_threads_sess;+ const uint32_t total_jobs = batched ? total_batches : (total_batches * k_chunks);+ const uint32_t max_threads = (std::max)(1u, total_jobs);+ const uint32_t n_threads = (std::min)(max_threads_sess, max_threads);+ const uint32_t batch_size = (uint32_t) dst->nb[2];+ const uint32_t batches_per_chunk = (batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(batch_size, HEX_L2_LINE_SIZE)) : 1;++ kparams->n_threads = n_threads;+ kparams->total_batches = total_batches;+ kparams->total_jobs = total_jobs;+ kparams->k_chunks = k_chunks;+ kparams->col_block = col_block;+ kparams->batched = batched ? 1 : 0;+ kparams->batches_per_chunk = batches_per_chunk;+ kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]);+ kparams->div_k_chunks = init_fastdiv_values(k_chunks > 0 ? k_chunks : 1);++ return true;+}++template <auto SolveFn>+static inline bool htp_kparams_invoke(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ void * kparams+) {+ using KParams = decltype(htp_kparams_type(SolveFn));+ KParams tmp;+ return SolveFn(sess, op, kparams ? (KParams *) kparams : &tmp);+}++bool htp_kparams_solve(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ void * kparams+) {+ const htp_op_code opcode = htp_op_remap(op);+ switch (opcode) {+ case HTP_OP_MUL_MAT:+ case HTP_OP_MUL_MAT_ID:+ return htp_kparams_invoke<htp_kparams_solve_matmul>(sess, op, kparams);++ case HTP_OP_MUL:+ case HTP_OP_ADD:+ case HTP_OP_ADD_ID:+ case HTP_OP_SUB:+ case HTP_OP_DIV:+ return htp_kparams_invoke<htp_kparams_solve_binary>(sess, op, kparams);++ case HTP_OP_FLASH_ATTN_EXT:+ return htp_kparams_invoke<htp_kparams_solve_flash_attn>(sess, op, kparams);++ case HTP_OP_GET_ROWS:+ return htp_kparams_invoke<htp_kparams_solve_get_rows>(sess, op, kparams);++ case HTP_OP_SET_ROWS:+ return htp_kparams_invoke<htp_kparams_solve_set_rows>(sess, op, kparams);++ case HTP_OP_ROPE:+ return htp_kparams_invoke<htp_kparams_solve_rope>(sess, op, kparams);++ case HTP_OP_SSM_CONV:+ return htp_kparams_invoke<htp_kparams_solve_ssm_conv>(sess, op, kparams);++ case HTP_OP_SOFTMAX:+ return htp_kparams_invoke<htp_kparams_solve_softmax>(sess, op, kparams);++ case HTP_OP_GATED_DELTA_NET:+ return htp_kparams_invoke<htp_kparams_solve_gated_delta_net>(sess, op, kparams);++ case HTP_OP_ARGSORT:+ case HTP_OP_TOP_K:+ return htp_kparams_invoke<htp_kparams_solve_sort>(sess, op, kparams);++ case HTP_OP_POOL_2D:+ case HTP_OP_POOL_1D:+ return htp_kparams_invoke<htp_kparams_solve_pool_2d>(sess, op, kparams);++ case HTP_OP_CONCAT:+ return htp_kparams_invoke<htp_kparams_solve_concat>(sess, op, kparams);++ case HTP_OP_CPY:+ case HTP_OP_CPY_FENCE:+ return htp_kparams_invoke<htp_kparams_solve_cpy>(sess, op, kparams);++ case HTP_OP_IM2COL:+ return htp_kparams_invoke<htp_kparams_solve_im2col>(sess, op, kparams);++ case HTP_OP_IM2COL_3D:+ return htp_kparams_invoke<htp_kparams_solve_im2col_3d>(sess, op, kparams);++ case HTP_OP_SUM_ROWS:+ case HTP_OP_SUM:+ case HTP_OP_ARGMAX:+ return htp_kparams_invoke<htp_kparams_solve_sum_rows>(sess, op, kparams);++ case HTP_OP_REPEAT:+ return htp_kparams_invoke<htp_kparams_solve_repeat>(sess, op, kparams);++ case HTP_OP_PAD:+ return htp_kparams_invoke<htp_kparams_solve_pad>(sess, op, kparams);++ case HTP_OP_CUMSUM:+ return htp_kparams_invoke<htp_kparams_solve_cumsum>(sess, op, kparams);++ case HTP_OP_FILL:+ return htp_kparams_invoke<htp_kparams_solve_fill>(sess, op, kparams);++ case HTP_OP_ROLL:+ return htp_kparams_invoke<htp_kparams_solve_roll>(sess, op, kparams);++ case HTP_OP_DIAG:+ return htp_kparams_invoke<htp_kparams_solve_diag>(sess, op, kparams);++ case HTP_OP_SOLVE_TRI:+ return htp_kparams_invoke<htp_kparams_solve_solve_tri>(sess, op, kparams);++ default:+ if (htp_op_is_unary(opcode)) {+ return htp_kparams_invoke<htp_kparams_solve_unary>(sess, op, kparams);+ }+ if (htp_op_is_glu(opcode)) {+ return htp_kparams_invoke<htp_kparams_solve_glu>(sess, op, kparams);+ }+ return false;+ }+}diff --git a/ggml/src/ggml-hexagon/htp-kparams.h b/ggml/src/ggml-hexagon/htp-kparams.hnew file mode 100644index 0000000000..0871209e7c--- /dev/null+++ b/ggml/src/ggml-hexagon/htp-kparams.h@@ -0,0 +1,123 @@+#ifndef HTP_KPARAMS_H+#define HTP_KPARAMS_H++#include "htp-common.h"++#include "htp/allreduce-ops.h"+#include "htp/argsort-ops.h"+#include "htp/binary-ops.h"+#include "htp/concat-ops.h"+#include "htp/cpy-ops.h"+#include "htp/cumsum-ops.h"+#include "htp/diag-ops.h"+#include "htp/fill-ops.h"+#include "htp/flash-attn-ops.h"+#include "htp/gated-delta-net-ops.h"+#include "htp/get-rows-ops.h"+#include "htp/glu-ops.h"+#include "htp/im2col-ops.h"+#include "htp/matmul-ops.h"+#include "htp/pad-ops.h"+#include "htp/pool-ops.h"+#include "htp/repeat-ops.h"+#include "htp/roll-ops.h"+#include "htp/rope-ops.h"+#include "htp/set-rows-ops.h"+#include "htp/softmax-ops.h"+#include "htp/solve-tri-ops.h"+#include "htp/ssm-conv.h"+#include "htp/sum-rows-ops.h"+#include "htp/unary-ops.h"++#include <type_traits>+#include <utility>++bool htp_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride);+bool htp_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst);+ggml_tensor htp_tensor_collapse_rows(const struct ggml_tensor * t);++template <typename... Args>+auto htp_kparams_type(bool (*)(Args...)) -> std::remove_pointer_t<std::remove_reference_t<decltype((std::declval<Args>(), ...))>>;++template <auto SolveFn, typename... Args>+static inline bool htp_kparams_check(Args &&... args) {+ decltype(htp_kparams_type(SolveFn)) kparams;+ return SolveFn(std::forward<Args>(args)..., &kparams);+}++bool htp_kparams_solve(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op,+ void * kparams = nullptr+);++static inline bool htp_kparams_check(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * op+) {+ return htp_kparams_solve(sess, op, nullptr);+}++bool htp_kparams_solve_allreduce(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * dst,+ uint32_t rank,+ uint32_t n_ranks,+ bool has_add,+ bool is_row_bcast,+ bool is_shard_ok,+ struct htp_allreduce_kernel_params * kparams+);++bool htp_kparams_solve_rms_norm_mul(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * dst,+ struct htp_unary_kernel_params * kparams+);++bool htp_kparams_solve_matmul_add(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * src2,+ const struct ggml_tensor * dst,+ struct htp_mm_kernel_params * kparams+);++bool htp_kparams_solve_mmnx(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * act,+ int32_t n_weights,+ struct htp_mm_kernel_params * kparams+);++bool htp_kparams_solve_mmidnx(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * act,+ const struct ggml_tensor * dst,+ int32_t n_weights,+ struct htp_mm_kernel_params * kparams+);++bool htp_kparams_solve_repeat_get_rows(+ const struct ggml_hexagon_session * sess,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * dst,+ struct htp_get_rows_kernel_params * kparams+);++bool htp_kparams_solve_unary_mul_glu(+ const struct ggml_hexagon_session * sess,+ uint32_t op,+ const struct ggml_tensor * src0,+ const struct ggml_tensor * src1,+ const struct ggml_tensor * dst,+ struct htp_glu_kernel_params * kparams+);++#endif // HTP_KPARAMS_Hdiff --git a/ggml/src/ggml-hexagon/htp-opnode.cpp b/ggml/src/ggml-hexagon/htp-opnode.cppnew file mode 100644index 0000000000..3509ffeeec--- /dev/null+++ b/ggml/src/ggml-hexagon/htp-opnode.cpp@@ -0,0 +1,260 @@+#include "htp-opnode.h"++#include "htp/allreduce-ops.h"+#include "htp/argsort-ops.h"+#include "htp/binary-ops.h"+#include "htp/cumsum-ops.h"+#include "htp/diag-ops.h"+#include "htp/fill-ops.h"+#include "htp/flash-attn-ops.h"+#include "htp/gated-delta-net-ops.h"+#include "htp/get-rows-ops.h"+#include "htp/glu-ops.h"+#include "htp/matmul-ops.h"+#include "htp/pad-ops.h"+#include "htp/repeat-ops.h"+#include "htp/roll-ops.h"+#include "htp/softmax-ops.h"+#include "htp/solve-tri-ops.h"+#include "htp/ssm-conv.h"+#include "htp/sum-rows-ops.h"+#include "htp/unary-ops.h"++#include <stdio.h>++std::string htp_opnode::name() const {+ switch (opcode) {+ case HTP_OP_MDEV_GROUP: return "MDEV_GROUP";+ case HTP_OP_CPY_FENCE: return "CPY+FENCE";+ case HTP_OP_FENCE: return (node && node->op_params[1] == 0) ? "FENCE_WAIT" : "FENCE_SIGNAL";+ case HTP_OP_ALLREDUCE: return "ALLREDUCE";+ case HTP_OP_ALLREDUCE_ADD: return "ALLREDUCE+ADD";+ case HTP_OP_HADAMARD: return "MUL_MAT_HADAMARD";+ case HTP_OP_MUL_MAT_NX: return "MUL_MAT_NX";+ case HTP_OP_MUL_MAT_ID_NX: return "MUL_MAT_ID_NX";+ case HTP_OP_ADD:+ case HTP_OP_SUB:+ case HTP_OP_MUL:+ case HTP_OP_DIV:+ if (!fused.empty()) {+ const auto * kp = kparams<htp_binary_kernel_params>();+ if (kp->kernel_type != HTP_BINARY_KERNEL_SCALAR) {+ switch (opcode) {+ case HTP_OP_ADD: return "ADD_NX";+ case HTP_OP_SUB: return "SUB_NX";+ case HTP_OP_MUL: return "MUL_NX";+ case HTP_OP_DIV: return "DIV_NX";+ default: break;+ }+ }+ }+ break;+ default:+ break;+ }++ if (!node) {+ return "";+ }+ if (htp_is_hadamard(node)) {+ return "MUL_MAT_HADAMARD";+ }++ std::string res = ggml_op_desc(node);+ for (const auto * t : fused) {+ res += "+";+ res += ggml_op_desc(t);+ }+ return res;+}++static const char * format_tensor_name(const struct ggml_tensor * t) {+ return t ? t->name : "NONE";+}++static std::string format_tensor_dims(const struct ggml_tensor * t) {+ if (!t) {+ return "NONE";+ }+ char buf[64];+ if (t->ne[2] == 1 && t->ne[3] == 1) {+ snprintf(buf, sizeof(buf), "%d:%d", (int) t->ne[0], (int) t->ne[1]);+ } else {+ snprintf(buf, sizeof(buf), "%d:%d:%d:%d", (int) t->ne[0], (int) t->ne[1], (int) t->ne[2], (int) t->ne[3]);+ }+ return buf;+}++static const char * format_tensor_type(const struct ggml_tensor * t) {+ return t ? ggml_type_name(t->type) : "NONE";+}++static std::string format_tensor_strides(const struct ggml_tensor * t) {+ if (!t) {+ return "NONE";+ }+ char buf[64];+ const char * c = ggml_is_contiguous(t) ? "" : "!";+ if (t->ne[2] == 1 && t->ne[3] == 1) {+ snprintf(buf, sizeof(buf), "%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], c);+ } else {+ snprintf(buf, sizeof(buf), "%zu:%zu:%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], (size_t) t->nb[2], (size_t) t->nb[3], c);+ }+ return buf;+}++static const char * format_tensor_buff(const struct ggml_tensor * t) {+ if (t && t->buffer) {+ return ggml_backend_buffer_name(t->buffer);+ }+ return "NONE";+}++template <typename Fn>+static std::string format_op_tensors(const htp_opnode & node, Fn fn) {+ std::string res;+ const auto & inputs = node.get_inputs();+ const auto & outputs = node.get_outputs();++ if (!inputs.empty()) {+ for (size_t i = 0; i < inputs.size(); i++) {+ if (i > 0) res += " x ";+ res += fn(inputs[i]);+ }+ res += " -> ";+ }++ if (outputs.empty()) {+ res += fn(node.dst());+ } else {+ for (size_t i = 0; i < outputs.size(); i++) {+ if (i > 0) res += " x ";+ res += fn(outputs[i]);+ }+ }+ return res;+}++static std::string format_kernel_params(const htp_opnode & node) {+ char str[128];+ const size_t max_size = sizeof(str);++ if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID ||+ node.opcode == HTP_OP_MUL_MAT_NX || node.opcode == HTP_OP_MUL_MAT_ID_NX ||+ node.opcode == HTP_OP_MUL_MAT_ADD) {+ const auto * kparams = node.kparams<htp_mm_kernel_params>();+ const char * path = "unknown";+ int32_t type = kparams->kernel_type;+ if (type == HTP_MM_KERNEL_HMX_2D || type == HTP_MM_KERNEL_HMX_BATCHED) {+ path = (kparams->tile_size > 0) ? "hmx-tiled" : "hmx-flat";+ } else if (type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || type == HTP_MM_KERNEL_HVX_F32_F32_VTCM ||+ type == HTP_MM_KERNEL_HVX_BF16_F32_VTCM) {+ path = "hvx-flat";+ } else if (type == HTP_MM_KERNEL_HVX_QUANT_ROW || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {+ path = "hvx-tiled";+ }+ snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);+ } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) {+ const auto * kparams = node.kparams<htp_fa_kernel_params>();+ const char * path = "unknown";+ int32_t type = kparams->kernel_type;+ if (type == HTP_FA_KERNEL_HMX) {+ path = kparams->u.hmx.pipeline ? "hmx-pipe" : "hmx-seq";+ } else if (type == HTP_FA_KERNEL_HVX) {+ path = "hvx";+ }+ snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);+ } else if (htp_op_is_unary(node.opcode)) {+ const auto * kparams = node.kparams<htp_unary_kernel_params>();+ snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size);+ } else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) {+ snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]);+ } else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) {+ snprintf(str, max_size, "seq 0x%x", (uint32_t) node.node->op_params[0]);+ } else if (node.opcode == HTP_OP_ALLREDUCE && node.node) {+ snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]);+ } else if (node.opcode == HTP_OP_SSM_CONV) {+ const auto * kparams = node.kparams<htp_ssm_conv_kernel_params>();+ snprintf(str, max_size, "%s vtcm %d", kparams->n_t == 1 ? "decode" : "prefill", (int) kparams->vtcm_size);+ } else if (node.opcode == HTP_OP_SOFTMAX) {+ const auto * kparams = node.kparams<htp_softmax_kernel_params>();+ snprintf(str, max_size, "k%d nth %d vtcm %d", (int) kparams->kernel_type, (int) kparams->n_threads, (int) kparams->vtcm_size);+ } else if (node.opcode == HTP_OP_GATED_DELTA_NET) {+ const auto * kparams = node.kparams<htp_gdn_kernel_params>();+ const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent";+ snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar",+ (unsigned int) kparams->vtcm_size);+ } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID ||+ node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {+ const auto * kparams = node.kparams<htp_binary_kernel_params>();+ snprintf(str, max_size, "vtcm %u", (unsigned int) kparams->vtcm_size);+ } else if (node.opcode == HTP_OP_ARGSORT || node.opcode == HTP_OP_TOP_K) {+ const auto * kparams = node.kparams<htp_sort_kernel_params>();+ snprintf(str, max_size, "%s nth %d nchk %d chk %d vtcm %d",+ node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort",+ (int) kparams->n_threads, (int) kparams->n_chunks,+ (int) kparams->chunk_elems, (int) kparams->vtcm_size);+ } else if (node.opcode == HTP_OP_GET_ROWS) {+ const auto * kparams = node.kparams<htp_get_rows_kernel_params>();+ const char * ktype_str = "unknown";+ switch (kparams->kernel_type) {+ case HTP_GET_ROWS_KERNEL_SAMETYPE: ktype_str = "sametype"; break;+ case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break;+ case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break;+ }+ snprintf(str, max_size, "%s%s vtcm %u", ktype_str, kparams->n_threads > 1 ? "-multi" : "", (unsigned int) kparams->vtcm_size);+ } else if (htp_op_is_glu(node.opcode)) {+ const auto * kparams = node.kparams<htp_glu_kernel_params>();+ snprintf(str, max_size, "nth %d vtcm %u", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);+ } else if (node.opcode == HTP_OP_PAD) {+ const auto * kparams = node.kparams<htp_pad_kernel_params>();+ snprintf(str, max_size, "k%d nth %d vtcm %u", (int) kparams->kernel_type, (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);+ } else if (node.opcode == HTP_OP_CUMSUM) {+ const auto * kparams = node.kparams<htp_cumsum_kernel_params>();+ snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);+ } else if (node.opcode == HTP_OP_FILL) {+ const auto * kparams = node.kparams<htp_fill_kernel_params>();+ snprintf(str, max_size, "%s nth %d", kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS ? "contig" : "strided", (int) kparams->n_threads);+ } else if (node.opcode == HTP_OP_ROLL) {+ const auto * kparams = node.kparams<htp_roll_kernel_params>();+ snprintf(str, max_size, "k%d nth %d", (int) kparams->kernel_type, (int) kparams->n_threads);+ } else if (node.opcode == HTP_OP_DIAG) {+ const auto * kparams = node.kparams<htp_diag_kernel_params>();+ snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);+ } else if (node.opcode == HTP_OP_SOLVE_TRI) {+ const auto * kparams = node.kparams<htp_solve_tri_kernel_params>();+ snprintf(str, max_size, "%s nth %d", kparams->batched ? "batched" : "chunked", (int) kparams->n_threads);+ } else {+ snprintf(str, max_size, "----");+ }+ return str;+}++static void append_section(std::string & res, const std::string & sec) {+ if (!res.empty()) {+ res += "|";+ }+ res += sec;+}++std::string htp_format_tensor(const ggml_tensor * t, uint32_t flags) {+ std::string res;+ if (flags & HTP_OPFMT_NAMES) append_section(res, format_tensor_name(t));+ if (flags & HTP_OPFMT_DIMS) append_section(res, format_tensor_dims(t));+ if (flags & HTP_OPFMT_TYPES) append_section(res, format_tensor_type(t));+ if (flags & HTP_OPFMT_STRIDES) append_section(res, format_tensor_strides(t));+ if (flags & HTP_OPFMT_BUFFS) append_section(res, format_tensor_buff(t));+ return res;+}++std::string htp_format_op(const htp_opnode & node, uint32_t flags) {+ std::string res;+ if (flags & HTP_OPFMT_OP) append_section(res, node.name());+ if (flags & HTP_OPFMT_NAMES) append_section(res, format_op_tensors(node, format_tensor_name));+ if (flags & HTP_OPFMT_DIMS) append_section(res, format_op_tensors(node, format_tensor_dims));+ if (flags & HTP_OPFMT_TYPES) append_section(res, format_op_tensors(node, format_tensor_type));+ if (flags & HTP_OPFMT_STRIDES) append_section(res, format_op_tensors(node, format_tensor_strides));+ if (flags & HTP_OPFMT_BUFFS) append_section(res, format_op_tensors(node, format_tensor_buff));+ if (flags & HTP_OPFMT_KPARAMS) append_section(res, format_kernel_params(node));+ return res;+}diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.hindex 57cd5f208d..a30c5b600b 100644--- a/ggml/src/ggml-hexagon/htp-opnode.h+++ b/ggml/src/ggml-hexagon/htp-opnode.h@@ -1,38 +1,74 @@#ifndef HTP_OPNODE_H#define HTP_OPNODE_H-#define GGML_COMMON_IMPL_CPP#include "ggml-backend-impl.h"-#include "ggml-common.h"+#include "ggml-impl.h"+#include "htp/htp-ops.h"-#include <algorithm>+#include <cstring>+#include <initializer_list>+#include <memory>#include <string>#include <vector>-#include <memory>-#include <stdio.h>-#include "htp-ops.h"-#include "htp/matmul-ops.h"-#include "htp/flash-attn-ops.h"-#include "htp/unary-ops.h"-#include "htp/binary-ops.h"-#include "htp/allreduce-ops.h"-#include "htp/ssm-conv.h"-#include "htp/gated-delta-net-ops.h"-#include "htp/softmax-ops.h"-#include "htp/argsort-ops.h"-#include "htp/get-rows-ops.h"++#define HTP_OP_MAX_FUSED 16++// local std::inplace_vector replacement+template <typename T, size_t MAX_N>+struct htp_opvec {+ T data[MAX_N] {};+ size_t n = 0;++ htp_opvec() = default;+ htp_opvec(std::initializer_list<T> init) {+ GGML_ASSERT(init.size() <= MAX_N);+ for (const auto & v : init) {+ data[n++] = v;+ }+ }++ size_t size() const { return n; }+ bool empty() const { return n == 0; }+ void clear() { n = 0; }+ T * begin() { return data; }+ T * end() { return data + n; }+ const T * begin() const { return data; }+ const T * end() const { return data + n; }+ T & back() { return data[n - 1]; }+ const T & back() const { return data[n - 1]; }+ T & operator[](size_t i) { return data[i]; }+ const T & operator[](size_t i) const { return data[i]; }++ void push_back(T val) {+ GGML_ASSERT(n < MAX_N);+ data[n++] = val;+ }++ void insert(T * pos, T val) {+ GGML_ASSERT(n < MAX_N);+ for (T * p = data + n; p > pos; --p) {+ *p = *(p - 1);+ }+ *pos = val;+ n++;+ }+};++static inline bool htp_is_hadamard(const struct ggml_tensor * t) {+ return t && t->op == GGML_OP_MUL_MAT && ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD;+}struct htp_opnode {- ggml_tensor * node { nullptr };- htp_op_code opcode { HTP_OP_INVALID };- int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS] {0};+ const ggml_tensor * node { nullptr };+ htp_op_code opcode { HTP_OP_INVALID };+ int32_t params[HTP_OP_MAX_PARAMS] {0};+ int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS] {0};- std::vector<ggml_tensor *> fused;- std::vector<std::shared_ptr<ggml_tensor>> dummy;+ std::vector<std::shared_ptr<ggml_tensor>> dummy;- std::vector<const ggml_tensor *> inputs;- std::vector<const ggml_tensor *> outputs;- std::string name;+ htp_opvec<const ggml_tensor *, HTP_OP_MAX_FUSED> fused;+ htp_opvec<const ggml_tensor *, HTP_OP_MAX_INPUTS> inputs;+ htp_opvec<const ggml_tensor *, HTP_OP_MAX_OUTPUTS> outputs;int n_active_src(const ggml_tensor * t) const {if (!t) return 0;@@ -44,34 +80,36 @@ struct htp_opnode {return 0;}- void init(ggml_tensor * node) {+ void init(const ggml_tensor * node) {+ static_assert(sizeof(this->params) == sizeof(this->node->op_params), "params size mismatch");+this->node = node;- if (this->node) {- if (this->node->op == GGML_OP_MUL_MAT &&- ggml_get_op_params_i32(this->node, 1) == GGML_HINT_SRC0_IS_HADAMARD) {- this->name = "MUL_MAT_HADAMARD";- this->inputs.push_back(this->node->src[1]);- } else {- this->name = ggml_op_desc(this->node);-- // Build inputs (preserving optional nullptrs)- int n_inputs = n_active_src(this->node);- this->inputs.resize(n_inputs, nullptr);- for (int i = 0; i < n_inputs; i++) {- this->inputs[i] = this->node->src[i];- }- }+ if (!node) return;++ memcpy(this->params, this->node->op_params, sizeof(this->params));- // Build outputs- this->outputs.push_back(this->dst());+ // Build inputs (preserving optional nullptrs)+ if (htp_is_hadamard(this->node)) {+ this->inputs.push_back(this->node->src[1]);+ } else {+ int n_inputs = n_active_src(this->node);+ for (int i = 0; i < n_inputs; i++) {+ this->inputs.push_back(this->node->src[i]);+ }}++ // Build outputs+ this->outputs.push_back(this->dst());}- htp_opnode(htp_op_code opcode = HTP_OP_INVALID, ggml_tensor * node = nullptr) : opcode(opcode) {+ htp_opnode(htp_op_code opcode = HTP_OP_INVALID, const ggml_tensor * node = nullptr) : opcode(opcode) {+ init(node);+ }++ htp_opnode(const ggml_tensor * node) : opcode(HTP_OP_INVALID) {init(node);}- ggml_op op() const { return node->op; }const ggml_tensor * src0() const { return node->src[0]; }const ggml_tensor * src1() const { return node->src[1]; }const ggml_tensor * dst() const { return outputs.empty() ? node : outputs.back(); }@@ -81,343 +119,54 @@ struct htp_opnode {return dummy.back().get();}- void add_fused(ggml_tensor * t, bool extra_dst = false) {- fused.push_back(t);-- name += "+";- name += ggml_op_desc(t);-- if (extra_dst) {- outputs.push_back(t);- } else {- outputs.clear();- outputs.push_back(t);- }-- // Remove the newly fused intermediate output tensor t from inputs (if it was there)- inputs.erase(std::remove(inputs.begin(), inputs.end(), t), inputs.end());-- // Append new inputs from t, preserving middle nullptrs- int n_inputs = n_active_src(t);- for (int i = 0; i < n_inputs; i++) {- const auto * src = t->src[i];- if (!src) {- inputs.push_back(nullptr);- } else if (src != node &&- std::find(fused.begin(), fused.end(), src) == fused.end() &&- std::find(inputs.begin(), inputs.end(), src) == inputs.end()) {- inputs.push_back(src);- }- }- }-- const std::vector<const ggml_tensor *> & get_inputs() const {+ const htp_opvec<const ggml_tensor *, HTP_OP_MAX_INPUTS> & get_inputs() const {return inputs;}- const std::vector<const ggml_tensor *> & get_outputs() const {+ const htp_opvec<const ggml_tensor *, HTP_OP_MAX_OUTPUTS> & get_outputs() const {return outputs;}- std::string op_name() const {- return name;+ template <typename T>+ T * kparams() {+ static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS");+ static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t");+ return reinterpret_cast<T *>(kernel_params);}- bool is_empty() const {- return ggml_op_is_empty(node->op);+ template <typename T>+ const T * kparams() const {+ static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS");+ static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t");+ return reinterpret_cast<const T *>(kernel_params);}- bool stackable() const {- switch (this->op()) {- case GGML_OP_MUL_MAT:- if (ggml_get_op_params_i32(this->node, 1) == GGML_HINT_SRC0_IS_HADAMARD) {- return false;- }- return ggml_is_quantized(this->src0()->type);- case GGML_OP_MUL_MAT_ID:- return ggml_is_quantized(this->src0()->type);- default:- return false;- }+ template <typename T>+ void set_kparams(const T & kp) {+ static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS");+ static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t");+ memcpy(kernel_params, &kp, sizeof(T));}- bool same_input(const htp_opnode& n) const {- return n.src1() == this->src1();- }+ std::string name() const;};-struct htp_opformat {- char strides[64 * GGML_MAX_SRC];- char dims[64 * GGML_MAX_SRC];- char types[16 * GGML_MAX_SRC];- char buffs[64 * GGML_MAX_SRC];- char names[64 * GGML_MAX_SRC];- char kparams[128];-- int format_tensor_dims(char * str, size_t max_size, const struct ggml_tensor * t) {- if (!t) {- return snprintf(str, max_size, "NONE");- }- if (t->ne[2] == 1 && t->ne[3] == 1) {- return snprintf(str, max_size, "%d:%d", (int) t->ne[0], (int) t->ne[1]);- } else {- return snprintf(str, max_size, "%d:%d:%d:%d", (int) t->ne[0], (int) t->ne[1], (int) t->ne[2], (int) t->ne[3]);- }- }-- void format_op_dims(char * str, size_t max_size, const htp_opnode & node) {- char * p = str;- char * p_end = str + max_size;- auto inputs = node.get_inputs();-- if (!inputs.empty()) {- p += std::min((size_t)format_tensor_dims(p, p_end - p, inputs[0]), (size_t)(p_end - p));-- for (size_t i = 1; i < inputs.size(); i++) {- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));- }- if (p < p_end) {- p += std::min((size_t)format_tensor_dims(p, p_end - p, inputs[i]), (size_t)(p_end - p));- }- }-- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));- }- }-- char self[64];- format_tensor_dims(self, sizeof(self), node.dst());- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, "%s", self), (size_t)(p_end - p));- }- }-- int format_tensor_strides(char * str, size_t max_size, const struct ggml_tensor * t) {- if (!t) {- return snprintf(str, max_size, "NONE");- }- const char * c = ggml_is_contiguous(t) ? "" : "!";-- if (t->ne[2] == 1 && t->ne[3] == 1) {- return snprintf(str, max_size, "%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], c);- } else {- return snprintf(str, max_size, "%zu:%zu:%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], (size_t) t->nb[2], (size_t) t->nb[3], c);- }- }-- void format_op_strides(char * str, size_t max_size, const htp_opnode & node) {- char * p = str;- char * p_end = str + max_size;- auto inputs = node.get_inputs();-- if (!inputs.empty()) {- p += std::min((size_t)format_tensor_strides(p, p_end - p, inputs[0]), (size_t)(p_end - p));-- for (size_t i = 1; i < inputs.size(); i++) {- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));- }- if (p < p_end) {- p += std::min((size_t)format_tensor_strides(p, p_end - p, inputs[i]), (size_t)(p_end - p));- }- }-- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));- }- }-- char self[64];- format_tensor_strides(self, sizeof(self), node.dst());- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, "%s", self), (size_t)(p_end - p));- }- }-- void format_op_types(char * str, size_t max_size, const htp_opnode & node) {- char * p = str;- char * p_end = str + max_size;- auto inputs = node.get_inputs();-- if (!inputs.empty()) {- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[0] ? ggml_type_name(inputs[0]->type) : "NONE"), (size_t)(p_end - p));- }-- for (size_t i = 1; i < inputs.size(); i++) {- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));- }- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[i] ? ggml_type_name(inputs[i]->type) : "NONE"), (size_t)(p_end - p));- }- }-- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));- }- }-- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, "%s", ggml_type_name(node.dst()->type)), (size_t)(p_end - p));- }- }-- const char * tensor_buff_name(const struct ggml_tensor * t) {- if (t && t->buffer) {- return ggml_backend_buffer_name(t->buffer);- }- return "NONE";- }-- void format_op_buffs(char * str, size_t max_size, const htp_opnode & node) {- char * p = str;- char * p_end = str + max_size;- auto inputs = node.get_inputs();-- if (!inputs.empty()) {- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, "%s", tensor_buff_name(inputs[0])), (size_t)(p_end - p));- }-- for (size_t i = 1; i < inputs.size(); i++) {- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));- }- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, "%s", tensor_buff_name(inputs[i])), (size_t)(p_end - p));- }- }-- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));- }- }-- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, "%s", tensor_buff_name(node.dst())), (size_t)(p_end - p));- }- }-- void format_op_names(char * str, size_t max_size, const htp_opnode & node) {- char * p = str;- char * p_end = str + max_size;- auto inputs = node.get_inputs();-- if (!inputs.empty()) {- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[0] ? inputs[0]->name : "NONE"), (size_t)(p_end - p));- }-- for (size_t i = 1; i < inputs.size(); i++) {- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, " x "), (size_t)(p_end - p));- }- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, "%s", inputs[i] ? inputs[i]->name : "NONE"), (size_t)(p_end - p));- }- }-- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, " -> "), (size_t)(p_end - p));- }- }-- if (p < p_end) {- p += std::min((size_t)snprintf(p, p_end - p, "%s", node.dst()->name), (size_t)(p_end - p));- }- }- void format_kernel_params(char * str, size_t max_size, const htp_opnode & node) {- if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID ||- node.opcode == HTP_OP_MUL_MAT_NX || node.opcode == HTP_OP_MUL_MAT_ID_NX ||- node.opcode == HTP_OP_MUL_MAT_ADD) {- const auto * kparams = (const struct htp_mm_kernel_params *) node.kernel_params;- const char * path = "unknown";- int32_t type = kparams->kernel_type;- if (type == HTP_MM_KERNEL_HMX_2D || type == HTP_MM_KERNEL_HMX_BATCHED) {- path = (kparams->tile_size > 0) ? "hmx-tiled" : "hmx-flat";- } else if (type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || type == HTP_MM_KERNEL_HVX_F32_F32_VTCM ||- type == HTP_MM_KERNEL_HVX_BF16_F32_VTCM) {- path = "hvx-flat";- } else if (type == HTP_MM_KERNEL_HVX_QUANT_ROW || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {- path = "hvx-tiled";- }- snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);- } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) {- const auto * kparams = (const struct htp_fa_kernel_params *) node.kernel_params;- const char * path = "unknown";- int32_t type = kparams->kernel_type;- if (type == HTP_FA_KERNEL_HMX) {- path = kparams->u.hmx.pipeline ? "hmx-pipe" : "hmx-seq";- } else if (type == HTP_FA_KERNEL_HVX) {- path = "hvx";- }- snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);- } else if (htp_op_is_unary(node.opcode)) {- const auto * kparams = (const struct htp_unary_kernel_params *) node.kernel_params;- snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size);- } else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) {- snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]);- } else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) {- snprintf(str, max_size, "seq 0x%x", (uint32_t) node.node->op_params[0]);- } else if (node.opcode == HTP_OP_ALLREDUCE && node.node) {- snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]);- } else if (node.opcode == HTP_OP_SSM_CONV) {- const auto * kparams = (const struct htp_ssm_conv_kernel_params *) node.kernel_params;- snprintf(str, max_size, "%s vtcm %d", kparams->n_t == 1 ? "decode" : "prefill", (int) kparams->vtcm_size);- } else if (node.opcode == HTP_OP_SOFTMAX) {- const auto * kparams = (const struct htp_softmax_kernel_params *) node.kernel_params;- snprintf(str, max_size, "k%d nth %d vtcm %d", (int) kparams->kernel_id, (int) kparams->n_threads, (int) kparams->vtcm_size);- } else if (node.opcode == HTP_OP_GATED_DELTA_NET) {- const auto * kparams = (const struct htp_gdn_kernel_params *) node.kernel_params;- const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent";- snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar",- (unsigned int) (kparams->vtcm_size ? kparams->vtcm_size : kparams->vtcm_per_thread * kparams->n_threads));- } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID ||- node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {- const auto * kparams = (const struct htp_binary_kernel_params *) node.kernel_params;- snprintf(str, max_size, "vtcm %u", (unsigned int) kparams->vtcm_size);- } else if (node.opcode == HTP_OP_ARGSORT || node.opcode == HTP_OP_TOP_K) {- const auto * kparams = (const struct htp_sort_kernel_params *) node.kernel_params;- snprintf(str, max_size, "%s nth %d nchk %d chk %d vtcm %d",- node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort",- (int) kparams->n_threads, (int) kparams->n_chunks,- (int) kparams->chunk_elems, (int) kparams->vtcm_size);- } else if (node.opcode == HTP_OP_GET_ROWS) {- const auto * kparams = (const struct htp_get_rows_kernel_params *) node.kernel_params;- const char * ktype_str = "unknown";- switch (kparams->kernel_type) {- case HTP_GET_ROWS_KERNEL_SAMETYPE: ktype_str = "sametype"; break;- case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break;- case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break;- }- snprintf(str, max_size, "%s%s vtcm %u", ktype_str, kparams->n_threads > 1 ? "-multi" : "", (unsigned int) kparams->vtcm_size);- } else {- snprintf(str, max_size, "----");- }-- }-- void format(const htp_opnode & node) {- format_op_dims(dims, sizeof(dims), node);- format_op_strides(strides, sizeof(strides), node);- format_op_types(types, sizeof(types), node);- format_op_buffs(buffs, sizeof(buffs), node);- format_op_names(names, sizeof(names), node);- format_kernel_params(kparams, sizeof(kparams), node);- }-- htp_opformat() {- strides[0] = '\0';- dims[0] = '\0';- types[0] = '\0';- buffs[0] = '\0';- names[0] = '\0';- kparams[0] = '\0';- }- htp_opformat(const htp_opnode & node) { format(node); }+enum htp_opformat_flags {+ HTP_OPFMT_OP = (1 << 0),+ HTP_OPFMT_NAMES = (1 << 1),+ HTP_OPFMT_DIMS = (1 << 2),+ HTP_OPFMT_TYPES = (1 << 3),+ HTP_OPFMT_STRIDES = (1 << 4),+ HTP_OPFMT_BUFFS = (1 << 5),+ HTP_OPFMT_KPARAMS = (1 << 6),++ HTP_OPFMT_EXEC = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_BUFFS | HTP_OPFMT_KPARAMS,+ HTP_OPFMT_SUPP = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_BUFFS,+ HTP_OPFMT_PROF = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_KPARAMS,+ HTP_OPFMT_TENSOR = HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES,};+std::string htp_format_op(const htp_opnode & node, uint32_t flags = HTP_OPFMT_EXEC);+std::string htp_format_tensor(const ggml_tensor * t, uint32_t flags = HTP_OPFMT_TENSOR);+#endif // HTP_OPNODE_Hdiff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txtindex 28fae4fe83..2edd653ce3 100644--- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt+++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt@@ -28,7 +28,7 @@ add_library(${HTP_LIB} SHAREDunary-ops.csum-rows-ops.csoftmax-ops.c- act-ops.c+ glu-ops.crope-ops.cset-rows-ops.cget-rows-ops.cdiff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.h b/ggml/src/ggml-hexagon/htp/allreduce-ops.hindex 54ef60378f..1264c1cbbc 100644--- a/ggml/src/ggml-hexagon/htp/allreduce-ops.h+++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.h@@ -32,6 +32,7 @@ static inline size_t htp_allreduce_vtcm_buffer_count(}struct htp_allreduce_kernel_params {+ uint8_t kernel_type;int32_t rank;int32_t n_ranks;int32_t n_threads;@@ -39,7 +40,6 @@ struct htp_allreduce_kernel_params {int32_t elems_per_thread; // 1D: nelem_per_thread, 2D: nrows_per_threadint32_t vtcm_size_per_thread;int32_t vtcm_size;- int32_t kernel_type;int32_t ne0;int32_t ne1;int32_t row_size_aligned;@@ -50,6 +50,12 @@ struct htp_allreduce_kernel_params {int32_t mode; // enum htp_allreduce_mode: FULL or SHARDED_FANOUT};+#if defined(__cplusplus)+static_assert(sizeof(struct htp_allreduce_kernel_params) <= 128, "htp_allreduce_kernel_params is too large for kernel_params blob");+#else+_Static_assert(sizeof(struct htp_allreduce_kernel_params) <= 128, "htp_allreduce_kernel_params is too large for kernel_params blob");+#endif+#ifdef __cplusplus}#endifdiff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.c b/ggml/src/ggml-hexagon/htp/argsort-ops.cindex 6f24d95f90..7aa7ca8e41 100644--- a/ggml/src/ggml-hexagon/htp/argsort-ops.c+++ b/ggml/src/ggml-hexagon/htp/argsort-ops.c@@ -453,7 +453,10 @@ static void merge_runs(struct htp_sort_chunk_ctx {struct htp_ops_context * octx;const struct htp_sort_kernel_params * kparams;- uint8_t * vtcm_base;+ uint8_t * phase1_base;+ size_t phase1_slot_size;+ float * merge_values;+ int32_t * merge_indices;uint32_t real_count[HTP_MAX_NTHREADS];};@@ -462,26 +465,27 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) {struct htp_ops_context * octx = cctx->octx;const struct htp_sort_kernel_params * kparams = cctx->kparams;const struct htp_tensor * src0 = octx->src[0];+ const struct htp_tensor * dst = octx->dst;uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;- uint32_t ne00 = (uint32_t) kparams->ne00;+ uint32_t ne00 = src0->ne[0];uint32_t chunk_base = i * chunk_elems;uint32_t real_count = (chunk_base < ne00) ? MIN(chunk_elems, ne00 - chunk_base) : 0;cctx->real_count[i] = real_count;- uint8_t * spad = cctx->vtcm_base + (size_t) kparams->phase1_slot_size * i;+ uint8_t * spad = cctx->phase1_base + cctx->phase1_slot_size * i;float * values_buf = (float *) spad;int32_t * indices_buf = (int32_t *) (spad + chunk_elems * sizeof(float));- float * merge_values = (float *) (cctx->vtcm_base + kparams->merge_values_off);- int32_t * merge_indices = (int32_t *) (cctx->vtcm_base + kparams->merge_indices_off);+ float * merge_values = cctx->merge_values;+ int32_t * merge_indices = cctx->merge_indices;struct htp_thread_trace * tr = &octx->ctx->trace[i];const dma_addr_t src_addr = src0->data + (size_t) chunk_base * sizeof(float);if (kparams->is_top_k) {- uint32_t k = (uint32_t) kparams->k;+ uint32_t k = dst->ne[0];uint32_t local_k = MIN(k, chunk_elems);if (real_count == 0) {@@ -559,17 +563,17 @@ static void htp_argsort_merge_l1_job(unsigned int n, unsigned int i, void * datauint32_t count0 = cctx->real_count[c0];uint32_t count1 = cctx->real_count[c1];- size_t slot_size = (size_t) kparams->phase1_slot_size;+ size_t slot_size = cctx->phase1_slot_size;uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;- const float * val0 = (const float *) (cctx->vtcm_base + slot_size * c0);- const int32_t * idx0 = (const int32_t *) (cctx->vtcm_base + slot_size * c0 + chunk_elems * sizeof(float));- const float * val1 = (const float *) (cctx->vtcm_base + slot_size * c1);- const int32_t * idx1 = (const int32_t *) (cctx->vtcm_base + slot_size * c1 + chunk_elems * sizeof(float));+ const float * val0 = (const float *) (cctx->phase1_base + slot_size * c0);+ const int32_t * idx0 = (const int32_t *) (cctx->phase1_base + slot_size * c0 + chunk_elems * sizeof(float));+ const float * val1 = (const float *) (cctx->phase1_base + slot_size * c1);+ const int32_t * idx1 = (const int32_t *) (cctx->phase1_base + slot_size * c1 + chunk_elems * sizeof(float));uint32_t out_offset = (2 * i) * chunk_elems;- float * out_val = (float *) (cctx->vtcm_base + kparams->merge_values_off) + out_offset;- int32_t * out_idx = (int32_t *) (cctx->vtcm_base + kparams->merge_indices_off) + out_offset;+ float * out_val = cctx->merge_values + out_offset;+ int32_t * out_idx = cctx->merge_indices + out_offset;struct htp_thread_trace * tr = &cctx->octx->ctx->trace[i];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);@@ -587,21 +591,21 @@ static void htp_argsort_merge_l2_job(unsigned int n, unsigned int i, void * datastruct htp_sort_chunk_ctx * cctx = mctx->cctx;const struct htp_sort_kernel_params * kparams = cctx->kparams;- size_t slot_size = (size_t) kparams->phase1_slot_size;+ size_t slot_size = cctx->phase1_slot_size;uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;uint32_t off0 = (4 * i) * chunk_elems;uint32_t off1 = (4 * i + 2) * chunk_elems;- const float * val0 = (const float *) (cctx->vtcm_base + kparams->merge_values_off) + off0;- const int32_t * idx0 = (const int32_t *) (cctx->vtcm_base + kparams->merge_indices_off) + off0;- const float * val1 = (const float *) (cctx->vtcm_base + kparams->merge_values_off) + off1;- const int32_t * idx1 = (const int32_t *) (cctx->vtcm_base + kparams->merge_indices_off) + off1;+ const float * val0 = cctx->merge_values + off0;+ const int32_t * idx0 = cctx->merge_indices + off0;+ const float * val1 = cctx->merge_values + off1;+ const int32_t * idx1 = cctx->merge_indices + off1;uint32_t count0 = cctx->real_count[4*i + 0] + cctx->real_count[4*i + 1];uint32_t count1 = cctx->real_count[4*i + 2] + cctx->real_count[4*i + 3];- uint8_t * slot_base = cctx->vtcm_base + (4 * i) * slot_size;+ uint8_t * slot_base = cctx->phase1_base + (4 * i) * slot_size;float * out_val = (float *) slot_base;int32_t * out_idx = (int32_t *) (slot_base + 4 * chunk_elems * sizeof(float));@@ -614,7 +618,10 @@ static void htp_argsort_merge_l2_job(unsigned int n, unsigned int i, void * datastruct htp_sort_multi_row_ctx {struct htp_ops_context * octx;const struct htp_sort_kernel_params * kparams;- uint8_t * vtcm_base;+ uint8_t * phase1_base;+ size_t phase1_slot_size;+ uint32_t row_start;+ uint32_t row_end;uint32_t nrows_per_thread;};@@ -624,12 +631,18 @@ struct htp_tensor_row_coord {uint32_t i03;};-static inline struct htp_tensor_row_coord init_row_coord(uint32_t r, uint32_t ne01, uint32_t ne02) {+static inline struct htp_tensor_row_coord init_row_coord(+ uint32_t r,+ uint32_t ne01,+ uint32_t ne02,+ const struct fastdiv_values * div_ne01,+ const struct fastdiv_values * div_ne01_ne02+) {uint32_t ne01_ne02 = ne01 * ne02;- uint32_t i03 = (ne01_ne02 > 0) ? (r / ne01_ne02) : 0;- uint32_t rem = (ne01_ne02 > 0) ? (r % ne01_ne02) : 0;- uint32_t i02 = (ne01 > 0) ? (rem / ne01) : 0;- uint32_t i01 = (ne01 > 0) ? (rem % ne01) : 0;+ uint32_t i03 = (ne01_ne02 > 0) ? fastdiv(r, div_ne01_ne02) : 0;+ uint32_t rem = (ne01_ne02 > 0) ? (r - i03 * ne01_ne02) : 0;+ uint32_t i02 = (ne01 > 0) ? fastdiv(rem, div_ne01) : 0;+ uint32_t i01 = (ne01 > 0) ? (rem - i02 * ne01) : 0;return (struct htp_tensor_row_coord){ i01, i02, i03 };}@@ -701,7 +714,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * dataconst struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * dst = octx->dst;- uint32_t ne00 = (uint32_t) kparams->ne00;+ uint32_t ne00 = src0->ne[0];uint32_t ne01 = src0->ne[1];uint32_t ne02 = src0->ne[2];@@ -713,16 +726,16 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * datauint32_t nb2 = dst->nb[2];uint32_t nb3 = dst->nb[3];- uint32_t row_start = (uint32_t) kparams->row_start;- uint32_t row_end = (uint32_t) kparams->row_end;+ uint32_t row_start = mctx->row_start;+ uint32_t row_end = mctx->row_end;uint32_t r_start = row_start + ith * mctx->nrows_per_thread;uint32_t r_end = MIN(r_start + mctx->nrows_per_thread, row_end);if (r_start >= r_end) return;- size_t spad_offset = ith * (size_t) (kparams->phase1_slot_size * kparams->n_slots);- uint8_t * thread_spad = mctx->vtcm_base + spad_offset;- size_t slot_size = (size_t) kparams->phase1_slot_size;+ size_t slot_size = mctx->phase1_slot_size;+ size_t spad_offset = ith * slot_size * (size_t) kparams->n_slots;+ uint8_t * thread_spad = mctx->phase1_base + spad_offset;uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;float * values_buf = (float *) thread_spad;@@ -732,14 +745,14 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * datastruct htp_thread_trace * tr = &octx->ctx->trace[ith];bool is_top_k = kparams->is_top_k != 0;- uint32_t k = (uint32_t) kparams->k;+ uint32_t k = dst->ne[0];enum ggml_sort_order order = (enum ggml_sort_order) kparams->order;const uint32_t src_bytes = ne00 * sizeof(float);const uint32_t dst_bytes = (is_top_k ? k : ne00) * sizeof(int32_t);if (kparams->n_slots == 2) {- struct htp_tensor_row_coord prime_coord = init_row_coord(r_start, ne01, ne02);+ struct htp_tensor_row_coord prime_coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02);for (uint32_t r = r_start, spad_idx = 0; r < r_end && spad_idx < 2; r++, spad_idx++) {uint8_t * cur_spad = thread_spad + spad_idx * slot_size;float * cur_values = (float *) cur_spad;@@ -759,8 +772,8 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * datastep_row_coord(&prime_coord, ne01, ne02);}- struct htp_tensor_row_coord dst_coord = init_row_coord(r_start, ne01, ne02);- struct htp_tensor_row_coord next_src_coord = init_row_coord(r_start + 2, ne01, ne02);+ struct htp_tensor_row_coord dst_coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02);+ struct htp_tensor_row_coord next_src_coord = init_row_coord(r_start + 2, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02);for (uint32_t r = r_start; r < r_end; r++) {uint32_t cur_slot = (r - r_start) & 1;@@ -796,7 +809,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * datadma_queue_flush(dma_q);} else {- struct htp_tensor_row_coord coord = init_row_coord(r_start, ne01, ne02);+ struct htp_tensor_row_coord coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02);for (uint32_t r = r_start; r < r_end; r++) {dma_addr_t src_addr = src0->data + coord_offset(coord, nb01, nb02, nb03);dma_addr_t dst_addr = dst->data + coord_offset(coord, nb1, nb2, nb3);@@ -847,52 +860,39 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {uint32_t ne00 = src0->ne[0];uint32_t k = dst->ne[0];- struct htp_sort_kernel_params kparams_local;const struct htp_sort_kernel_params * kparams = (const struct htp_sort_kernel_params *) octx->kernel_params;- if (kparams->n_threads == 0) {- struct htp_sort_vtcm_layout layout;- if (!htp_sort_solve_layout(&layout, ne00, total_rows, k, octx->n_threads, octx->ctx->vtcm_size, is_top_k)) {- return HTP_STATUS_VTCM_TOO_SMALL;- }- memset(&kparams_local, 0, sizeof(kparams_local));- kparams_local.n_threads = (int32_t) layout.n_threads;- kparams_local.total_rows = (int32_t) total_rows;- kparams_local.row_start = (int32_t) row_start;- kparams_local.row_end = (int32_t) row_end;- kparams_local.ne00 = (int32_t) ne00;- kparams_local.k = (int32_t) k;- kparams_local.order = is_top_k ? GGML_SORT_ORDER_DESC : (int32_t) octx->op_params[0];- kparams_local.is_top_k = is_top_k ? 1 : 0;- kparams_local.use_dma = 1;- kparams_local.chunk_elems = (int32_t) layout.chunk_elems;- kparams_local.n_chunks = (int32_t) layout.n_chunks;- kparams_local.vtcm_size = (int32_t) layout.total_bytes;- kparams_local.phase1_slot_size = (int32_t) layout.phase1_slot_size;- kparams_local.merge_values_off = (int32_t) layout.merge_values_off;- kparams_local.merge_indices_off = (int32_t) layout.merge_indices_off;- kparams_local.merge_elems = (int32_t) layout.merge_elems;- kparams_local.n_slots = (int32_t) layout.n_slots;- kparams = &kparams_local;- } else if (octx->ctx->mdev.count > 1) {- memcpy(&kparams_local, kparams, sizeof(kparams_local));- kparams_local.row_start = (int32_t) row_start;- kparams_local.row_end = (int32_t) row_end;- kparams = &kparams_local;+ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {+ return HTP_STATUS_INVAL_PARAMS;}- if (octx->ctx->vtcm_size < (size_t) kparams->vtcm_size) {- FARF(ERROR, "sort: VTCM size too small. Needed %d, have %zu", kparams->vtcm_size, octx->ctx->vtcm_size);+ struct htp_sort_vtcm_layout layout;+ htp_sort_vtcm_layout_build(&layout,+ total_rows,+ (uint32_t) kparams->chunk_elems,+ (uint32_t) kparams->n_chunks,+ (uint32_t) kparams->merge_elems,+ (uint32_t) kparams->n_threads,+ (uint32_t) kparams->n_slots);++ if (octx->ctx->vtcm_size < layout.total_bytes) {+ FARF(ERROR, "sort: VTCM size too small. Needed %zu, have %zu", layout.total_bytes, octx->ctx->vtcm_size);return HTP_STATUS_VTCM_TOO_SMALL;}- uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;+ uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;+ uint8_t * phase1_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_phase1);+ float * merge_values = VTCM_LAYOUT_PTR(float, vtcm_base, layout.off_merge_values);+ int32_t * merge_indices = VTCM_LAYOUT_PTR(int32_t, vtcm_base, layout.off_merge_indices);if (total_rows == 1 && kparams->n_chunks > 1) {struct htp_sort_chunk_ctx cctx;- cctx.octx = octx;- cctx.kparams = kparams;- cctx.vtcm_base = vtcm_base;+ cctx.octx = octx;+ cctx.kparams = kparams;+ cctx.phase1_base = phase1_base;+ cctx.phase1_slot_size = layout.phase1_slot_size;+ cctx.merge_values = merge_values;+ cctx.merge_indices = merge_indices;memset(cctx.real_count, 0, sizeof(cctx.real_count));work_queue_run(octx->ctx->work_queue, htp_sort_chunk_job, &cctx, (uint32_t) kparams->n_chunks);@@ -902,8 +902,8 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {uint32_t local_k = MIN(k, chunk_elems);if (k == 1 && ne00 >= 128*1024) {- float * cand_vals = (float *) (vtcm_base + kparams->merge_values_off);- int32_t * cand_idxs = (int32_t *) (vtcm_base + kparams->merge_indices_off);+ float * cand_vals = merge_values;+ int32_t * cand_idxs = merge_indices;float best_val = cand_vals[0];int32_t best_idx = cand_idxs[0];for (int32_t c = 1; c < kparams->n_chunks; c++) {@@ -918,8 +918,8 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {}uint32_t total_candidates = (uint32_t) kparams->n_chunks * local_k;- float * cand_vals = (float *) (vtcm_base + kparams->merge_values_off);- int32_t * cand_idxs = (int32_t *) (vtcm_base + kparams->merge_indices_off);+ float * cand_vals = merge_values;+ int32_t * cand_idxs = merge_indices;uint32_t merge_elems = (uint32_t) kparams->merge_elems;if (merge_elems > total_candidates) {@@ -943,18 +943,18 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {struct htp_argsort_merge_l2_ctx l2ctx = { &cctx, asc };work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l2_job, &l2ctx, 2);- size_t slot_size = (size_t) kparams->phase1_slot_size;+ size_t slot_size = layout.phase1_slot_size;uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;- const float * val0 = (const float *) vtcm_base;- const int32_t * idx0 = (const int32_t *) (vtcm_base + 4 * chunk_elems * sizeof(float));- const float * val1 = (const float *) (vtcm_base + 4 * slot_size);- const int32_t * idx1 = (const int32_t *) (vtcm_base + 4 * slot_size + 4 * chunk_elems * sizeof(float));+ const float * val0 = (const float *) phase1_base;+ const int32_t * idx0 = (const int32_t *) (phase1_base + 4 * chunk_elems * sizeof(float));+ const float * val1 = (const float *) (phase1_base + 4 * slot_size);+ const int32_t * idx1 = (const int32_t *) (phase1_base + 4 * slot_size + 4 * chunk_elems * sizeof(float));uint32_t n01 = cctx.real_count[0] + cctx.real_count[1] + cctx.real_count[2] + cctx.real_count[3];uint32_t n23 = cctx.real_count[4] + cctx.real_count[5] + cctx.real_count[6] + cctx.real_count[7];- int32_t * final_idx = (int32_t *) (vtcm_base + kparams->merge_indices_off);+ int32_t * final_idx = merge_indices;struct htp_thread_trace * tr = &octx->ctx->trace[0];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0);merge_runs(val0, idx0, n01, val1, idx1, n23, NULL, final_idx, asc);@@ -968,27 +968,27 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {uint32_t n01 = cctx.real_count[0] + cctx.real_count[1];uint32_t n23 = cctx.real_count[2] + cctx.real_count[3];- const float * val01 = (const float *) (vtcm_base + kparams->merge_values_off);- const int32_t * idx01 = (const int32_t *) (vtcm_base + kparams->merge_indices_off);- const float * val23 = (const float *) (vtcm_base + kparams->merge_values_off) + 2 * chunk_elems;- const int32_t * idx23 = (const int32_t *) (vtcm_base + kparams->merge_indices_off) + 2 * chunk_elems;+ const float * val01 = merge_values;+ const int32_t * idx01 = merge_indices;+ const float * val23 = merge_values + 2 * chunk_elems;+ const int32_t * idx23 = merge_indices + 2 * chunk_elems;- int32_t * final_idx = (int32_t *) vtcm_base;+ int32_t * final_idx = (int32_t *) phase1_base;struct htp_thread_trace * tr = &octx->ctx->trace[0];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0);merge_runs(val01, idx01, n01, val23, idx23, n23, NULL, final_idx, asc);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, 0);dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t));} else if (kparams->n_chunks == 2) {- size_t slot_size = (size_t) kparams->phase1_slot_size;+ size_t slot_size = layout.phase1_slot_size;uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;- const float * val0 = (const float *) (vtcm_base);- const int32_t * idx0 = (const int32_t *) (vtcm_base + chunk_elems * sizeof(float));- const float * val1 = (const float *) (vtcm_base + slot_size);- const int32_t * idx1 = (const int32_t *) (vtcm_base + slot_size + chunk_elems * sizeof(float));+ const float * val0 = (const float *) (phase1_base);+ const int32_t * idx0 = (const int32_t *) (phase1_base + chunk_elems * sizeof(float));+ const float * val1 = (const float *) (phase1_base + slot_size);+ const int32_t * idx1 = (const int32_t *) (phase1_base + slot_size + chunk_elems * sizeof(float));- int32_t * final_idx = (int32_t *) (vtcm_base + kparams->merge_indices_off);+ int32_t * final_idx = merge_indices;struct htp_thread_trace * tr = &octx->ctx->trace[0];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0);merge_runs(val0, idx0, cctx.real_count[0], val1, idx1, cctx.real_count[1], NULL, final_idx, asc);@@ -996,7 +996,7 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t));} else {uint32_t chunk_elems = (uint32_t) kparams->chunk_elems;- int32_t * final_idx = (int32_t *) (vtcm_base + chunk_elems * sizeof(float));+ int32_t * final_idx = (int32_t *) (phase1_base + chunk_elems * sizeof(float));dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t));}}@@ -1006,7 +1006,10 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {struct htp_sort_multi_row_ctx mctx;mctx.octx = octx;mctx.kparams = kparams;- mctx.vtcm_base = vtcm_base;+ mctx.phase1_base = phase1_base;+ mctx.phase1_slot_size = layout.phase1_slot_size;+ mctx.row_start = row_start;+ mctx.row_end = row_end;uint32_t dst_row_bytes = (uint32_t) dst_row_size;uint32_t rows_per_chunk = 0;@@ -1014,13 +1017,13 @@ static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) {if (rows_per_chunk == 0) {rows_per_chunk = 1;}- uint32_t dr = (nrows + (uint32_t) kparams->n_threads - 1) / (uint32_t) kparams->n_threads;+ uint32_t dr = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div);if (rows_per_chunk > 1) {dr = hex_round_up(dr, rows_per_chunk);}mctx.nrows_per_thread = dr;- work_queue_run(octx->ctx->work_queue, htp_sort_multi_row_job, &mctx, (uint32_t) kparams->n_threads);+ work_queue_run(octx->ctx->work_queue, htp_sort_multi_row_job, &mctx, octx->n_threads);return HTP_STATUS_OK;}diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.h b/ggml/src/ggml-hexagon/htp/argsort-ops.hindex 05320a2e91..f74189fba5 100644--- a/ggml/src/ggml-hexagon/htp/argsort-ops.h+++ b/ggml/src/ggml-hexagon/htp/argsort-ops.h@@ -7,33 +7,27 @@#include <string.h>#include "hex-fastdiv.h"+#include "htp-vtcm.h"struct htp_sort_kernel_params {- int32_t n_threads;- int32_t total_rows;- int32_t row_start;- int32_t row_end;- int32_t ne00;- int32_t k;- int32_t order; // GGML_SORT_ORDER_ASC (0) or GGML_SORT_ORDER_DESC (1)- int32_t is_top_k; // 1 if TOP_K, 0 if ARGSORT- int32_t use_dma; // 1 if DMA enabled- int32_t chunk_elems;- int32_t n_chunks;- int32_t vtcm_size;- int32_t phase1_slot_size;- int32_t merge_values_off;- int32_t merge_indices_off;- int32_t merge_elems;- int32_t n_slots;- int32_t pad[15];+ uint32_t n_threads;+ uint32_t order; // GGML_SORT_ORDER_ASC (0) or GGML_SORT_ORDER_DESC (1)+ uint32_t is_top_k; // 1 if TOP_K, 0 if ARGSORT+ uint32_t chunk_elems;+ uint32_t n_chunks;+ uint32_t vtcm_size;+ uint32_t merge_elems;+ uint32_t n_slots;+ struct fastdiv_values div_ne01;+ struct fastdiv_values div_ne01_ne02;};struct htp_sort_vtcm_layout {size_t total_bytes;size_t phase1_slot_size;- size_t merge_values_off;- size_t merge_indices_off;+ size_t off_phase1;+ size_t off_merge_values;+ size_t off_merge_indices;uint32_t chunk_elems;uint32_t n_chunks;uint32_t merge_elems;@@ -41,6 +35,39 @@ struct htp_sort_vtcm_layout {uint32_t n_slots;};+static inline void htp_sort_vtcm_layout_build(+ struct htp_sort_vtcm_layout * L,+ uint32_t total_rows,+ uint32_t chunk_elems,+ uint32_t n_chunks,+ uint32_t merge_elems,+ uint32_t n_threads,+ uint32_t n_slots) {++ memset(L, 0, sizeof(*L));+ L->chunk_elems = chunk_elems;+ L->n_chunks = n_chunks;+ L->merge_elems = merge_elems;+ L->n_threads = n_threads;+ L->n_slots = n_slots;++ const size_t values_size = ((chunk_elems * sizeof(float)) + 127) & ~127;+ const size_t indices_size = ((chunk_elems * sizeof(int32_t)) + 127) & ~127;+ L->phase1_slot_size = ((values_size + indices_size) + 255) & ~255;++ size_t off = 0;+ if (total_rows > 1) {+ VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_slots * n_threads);+ } else {+ VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_chunks * n_slots);+ const size_t merge_values_size = ((merge_elems * sizeof(float)) + 127) & ~127;+ const size_t merge_indices_size = ((merge_elems * sizeof(int32_t)) + 127) & ~127;+ VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_merge_values, merge_values_size, n_chunks > 1);+ VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_merge_indices, merge_indices_size, n_chunks > 1);+ }+ L->total_bytes = off;+}+static inline bool htp_sort_solve_layout(struct htp_sort_vtcm_layout * layout,uint32_t ne00,@@ -75,18 +102,8 @@ static inline bool htp_sort_solve_layout(n_threads--;}- size_t total_bytes = spad_per_thread * n_threads;- if (total_bytes > vtcm_budget) {- return false;- }-- layout->total_bytes = total_bytes;- layout->phase1_slot_size = spad_per_slot;- layout->chunk_elems = ne00_padded;- layout->n_chunks = 1;- layout->n_threads = n_threads;- layout->n_slots = n_slots;- return true;+ htp_sort_vtcm_layout_build(layout, total_rows, ne00_padded, 1, 0, n_threads, n_slots);+ return layout->total_bytes <= vtcm_budget;}uint32_t n_vec = (ne00 + 31) / 32;@@ -102,17 +119,7 @@ static inline bool htp_sort_solve_layout(uint32_t chunk_n_vec = n_vec_pow2 / n_chunks;uint32_t chunk_elems = chunk_n_vec * 32;-- size_t phase1_values_size = ((chunk_elems * sizeof(float)) + 127) & ~127;- size_t phase1_indices_size = ((chunk_elems * sizeof(int32_t)) + 127) & ~127;- size_t phase1_slot_size = ((phase1_values_size + phase1_indices_size) + 255) & ~255;- size_t phase1_total_size = phase1_slot_size * n_chunks;-- size_t merge_values_size = 0;- size_t merge_indices_size = 0;- size_t merge_values_off = phase1_total_size;- size_t merge_indices_off = 0;- uint32_t merge_elems = 0;+ uint32_t merge_elems = 0;if (n_chunks > 1) {if (is_top_k) {@@ -125,47 +132,25 @@ static inline bool htp_sort_solve_layout(} else {merge_elems = n_vec_pow2 * 32;}- merge_values_size = ((merge_elems * sizeof(float)) + 127) & ~127;- merge_indices_size = ((merge_elems * sizeof(int32_t)) + 127) & ~127;- merge_indices_off = merge_values_off + merge_values_size;}- size_t total_bytes = phase1_total_size + merge_values_size + merge_indices_size;+ htp_sort_vtcm_layout_build(layout, 1, chunk_elems, n_chunks, merge_elems, n_chunks, 1);- if (total_bytes > vtcm_budget && n_chunks > 1) {- n_chunks = 1;+ if (layout->total_bytes > vtcm_budget && n_chunks > 1) {+ n_chunks = 1;chunk_elems = n_vec_pow2 * 32;- phase1_values_size = ((chunk_elems * sizeof(float)) + 127) & ~127;- phase1_indices_size = ((chunk_elems * sizeof(int32_t)) + 127) & ~127;- phase1_slot_size = ((phase1_values_size + phase1_indices_size) + 255) & ~255;- phase1_total_size = phase1_slot_size;- merge_values_size = 0;- merge_indices_size = 0;- merge_values_off = phase1_total_size;- merge_indices_off = 0;- merge_elems = 0;- total_bytes = phase1_total_size;+ merge_elems = 0;+ htp_sort_vtcm_layout_build(layout, 1, chunk_elems, n_chunks, merge_elems, 1, 1);}- if (total_bytes > vtcm_budget) {+ if (layout->total_bytes > vtcm_budget) {return false;}- uint32_t n_slots = 1;- if (n_chunks == 1 && phase1_slot_size * 2 <= vtcm_budget) {- n_slots = 2;- total_bytes = phase1_slot_size * 2;+ if (n_chunks == 1 && layout->phase1_slot_size * 2 <= vtcm_budget) {+ htp_sort_vtcm_layout_build(layout, 1, chunk_elems, 1, 0, 1, 2);}- layout->total_bytes = total_bytes;- layout->phase1_slot_size = phase1_slot_size;- layout->merge_values_off = merge_values_off;- layout->merge_indices_off = merge_indices_off;- layout->chunk_elems = chunk_elems;- layout->n_chunks = n_chunks;- layout->merge_elems = merge_elems;- layout->n_threads = n_chunks;- layout->n_slots = n_slots;return true;}diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.cindex 78ce4bb63d..48f461e466 100644--- a/ggml/src/ggml-hexagon/htp/binary-ops.c+++ b/ggml/src/ggml-hexagon/htp/binary-ops.c@@ -42,6 +42,7 @@ struct htp_binary_context {uint32_t nrows_per_thread;uint32_t total_rows;uint32_t row_start;+ uint32_t n_srcs;size_t src0_row_size_aligned;size_t src1_row_size_aligned;size_t dst_row_size_aligned;@@ -380,6 +381,7 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith,size_t dst_spad_half = layout->dst_spad_half_size;dma_queue * dma_q = octx->ctx->dma[ith];+ const uint32_t n_srcs = bctx->n_srcs;uint32_t ir_prefetch = start_row;int spad_idx = 0;@@ -391,12 +393,7 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith,i02 = fastdiv(rem, &bctx->src0_dim1_div);i01 = rem - i02 * ne01;- uint32_t i13 = (ne13 == 1) ? 0 : i03;- uint32_t i12 = (ne12 == 1) ? 0 : i02;- uint32_t i11 = (ne11 == 1) ? 0 : i01;-dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01;- dma_addr_t src1_curr = src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11;dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half;@@ -405,7 +402,15 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith,dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0);dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size);- dma_queue_push(dma_q, dma_make_data(s1_spad, src1_curr), bctx->src1_row_size_aligned, nb11, row_size_bytes, current_block_size);+ for (uint32_t s = 1; s < n_srcs; s++) {+ const struct htp_tensor * src_s = octx->src[s];+ uint32_t is3 = (src_s->ne[3] == 1) ? 0 : i03;+ uint32_t is2 = (src_s->ne[2] == 1) ? 0 : i02;+ uint32_t is1 = (src_s->ne[1] == 1) ? 0 : i01;+ dma_addr_t s_curr = src_s->data + is3 * src_s->nb[3] + is2 * src_s->nb[2] + is1 * src_s->nb[1];+ uint8_t * s_spad = s1_spad + (s - 1) * 2 * src1_spad_half;+ dma_queue_push(dma_q, dma_make_data(s_spad, s_curr), bctx->src1_row_size_aligned, src_s->nb[1], row_size_bytes, current_block_size);+ }ir_prefetch += current_block_size;spad_idx ^= 1;}@@ -422,6 +427,11 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith,htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);compute(d_spad, s0_spad, s1_spad, current_block_size,bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, bctx->src1_row_size_aligned, ne00);+ for (uint32_t s = 2; s < n_srcs; s++) {+ uint8_t * ss_spad = (uint8_t *) dma_queue_pop(dma_q).dst;+ compute(d_spad, d_spad, ss_spad, current_block_size,+ bctx->dst_row_size_aligned, bctx->dst_row_size_aligned, bctx->src1_row_size_aligned, ne00);+ }htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);uint32_t i03, i02, i01, rem;@@ -440,15 +450,18 @@ static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith,p02 = fastdiv(prem, &bctx->src0_dim1_div);p01 = prem - p02 * ne01;- uint32_t p13 = (ne13 == 1) ? 0 : p03;- uint32_t p12 = (ne12 == 1) ? 0 : p02;- uint32_t p11 = (ne11 == 1) ? 0 : p01;-dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01;- dma_addr_t s1_next = src1->data + p13 * nb13 + p12 * nb12 + p11 * nb11;-dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size);- dma_queue_push(dma_q, dma_make_data(s1_spad, s1_next), bctx->src1_row_size_aligned, nb11, row_size_bytes, next_block_size);++ for (uint32_t s = 1; s < n_srcs; s++) {+ const struct htp_tensor * src_s = octx->src[s];+ uint32_t ps3 = (src_s->ne[3] == 1) ? 0 : p03;+ uint32_t ps2 = (src_s->ne[2] == 1) ? 0 : p02;+ uint32_t ps1 = (src_s->ne[1] == 1) ? 0 : p01;+ dma_addr_t s_next = src_s->data + ps3 * src_s->nb[3] + ps2 * src_s->nb[2] + ps1 * src_s->nb[1];+ uint8_t * s_spad = s1_spad + (s - 1) * 2 * src1_spad_half;+ dma_queue_push(dma_q, dma_make_data(s_spad, s_next), bctx->src1_row_size_aligned, src_s->nb[1], row_size_bytes, next_block_size);+ }ir_prefetch += next_block_size;}@@ -859,6 +872,7 @@ struct binary_chunked_context {uint32_t chunk_size;uint32_t chunks_per_thread;uint32_t total_chunks;+ uint32_t n_srcs;bool is_scalar;float scalar_f32;_Float16 scalar_f16;@@ -872,7 +886,6 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * datstruct binary_chunked_context * ctx = (struct binary_chunked_context *) data;struct htp_ops_context * octx = ctx->octx;const struct htp_tensor * src0 = octx->src[0];- const struct htp_tensor * src1 = octx->src[1];const struct htp_tensor * dst = octx->dst;const uint32_t start_chunk = ctx->chunks_per_thread * ith;@@ -895,6 +908,7 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * datuint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread);dma_queue * dma_q = octx->ctx->dma[ith];+ const uint32_t n_srcs = ctx->n_srcs;uint32_t prefetch_chunk = start_chunk;int spad_idx = 0;@@ -913,9 +927,12 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * datdma_queue_push(dma_q, dma_make_data(d_curr, d_spad), chunk_bytes, chunk_bytes, cur_bytes, 0);dma_queue_push(dma_q, dma_make_data(s0_spad, s0_curr), chunk_bytes, chunk_bytes, cur_bytes, 1);if (!ctx->is_scalar) {- dma_addr_t s1_curr = src1->data + (size_t) c_start * elem_size;uint8_t * s1_spad = src1_spad_base + spad_idx * chunk_bytes;- dma_queue_push(dma_q, dma_make_data(s1_spad, s1_curr), chunk_bytes, chunk_bytes, cur_bytes, 1);+ for (uint32_t s = 1; s < n_srcs; s++) {+ dma_addr_t s_curr = octx->src[s]->data + (size_t) c_start * elem_size;+ uint8_t * s_spad = s1_spad + (s - 1) * 2 * chunk_bytes;+ dma_queue_push(dma_q, dma_make_data(s_spad, s_curr), chunk_bytes, chunk_bytes, cur_bytes, 1);+ }}prefetch_chunk++;@@ -943,6 +960,10 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * dat}} else {ctx->compute(d_spad, s0_spad, s1_spad, cur_elems);+ for (uint32_t s = 2; s < n_srcs; s++) {+ uint8_t * s_spad = (uint8_t *) dma_queue_pop(dma_q).dst;+ ctx->compute(d_spad, d_spad, s_spad, cur_elems);+ }}htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) c);@@ -958,8 +979,11 @@ static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * datdma_addr_t s0_next = src0->data + (size_t) pc_start * elem_size;dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), chunk_bytes, chunk_bytes, p_bytes, 1);if (!ctx->is_scalar) {- dma_addr_t s1_next = src1->data + (size_t) pc_start * elem_size;- dma_queue_push(dma_q, dma_make_data(s1_spad, s1_next), chunk_bytes, chunk_bytes, p_bytes, 1);+ for (uint32_t s = 1; s < n_srcs; s++) {+ dma_addr_t s_next = octx->src[s]->data + (size_t) pc_start * elem_size;+ uint8_t * s_spad = s1_spad + (s - 1) * 2 * chunk_bytes;+ dma_queue_push(dma_q, dma_make_data(s_spad, s_next), chunk_bytes, chunk_bytes, p_bytes, 1);+ }}prefetch_chunk++;@@ -974,6 +998,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {const struct htp_tensor * src1 = octx->src[1];const struct htp_tensor * dst = octx->dst;const struct htp_binary_kernel_params * kparams = (const struct htp_binary_kernel_params *) octx->kernel_params;+ const uint32_t n_srcs = kparams->n_srcs > 2 ? kparams->n_srcs : 2;const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];@@ -985,7 +1010,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {const size_t dst_row_size = dst->ne[0] * elem_size;if (kparams->kernel_type == HTP_BINARY_KERNEL_CHUNKED) {- const bool is_scalar = kparams->is_scalar || (src1->ne[0] == 1 && src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1);+ const bool is_scalar = kparams->is_scalar;compute_binary_chunked_t compute = NULL;compute_binary_scalar_chunked_f32_t compute_scalar_f32 = NULL;@@ -1045,12 +1070,16 @@ static int execute_op_binary(struct htp_ops_context * octx) {const uint32_t elems_per_line = (src0_type == HTP_TYPE_F32) ? 32 : 64;if (octx->ctx->mdev.count > 1) {- const bool can_split = htp_tensor_mdev_data_aligned(dst) &&- htp_tensor_mdev_data_aligned(src0) &&- (is_scalar || htp_tensor_mdev_data_aligned(src1)) &&- htp_tensor_is_contiguous(dst, elem_size) &&- htp_tensor_is_contiguous(src0, elem_size) &&- (is_scalar || htp_tensor_is_contiguous(src1, elem_size));+ bool can_split = htp_tensor_mdev_data_aligned(dst) &&+ htp_tensor_mdev_data_aligned(src0) &&+ (is_scalar || htp_tensor_mdev_data_aligned(src1)) &&+ htp_tensor_is_contiguous(dst, elem_size) &&+ htp_tensor_is_contiguous(src0, elem_size) &&+ (is_scalar || htp_tensor_is_contiguous(src1, elem_size));+ for (uint32_t s = 2; can_split && s < n_srcs; s++) {+ can_split = htp_tensor_mdev_data_aligned(octx->src[s]) &&+ htp_tensor_is_contiguous(octx->src[s], elem_size);+ }const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_elems, can_split ? elems_per_line : 0,octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);@@ -1072,7 +1101,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {return HTP_STATUS_VTCM_TOO_SMALL;}- const uint32_t chunk_size = kparams->chunk_size > 0 ? kparams->chunk_size : (32768 / elem_size);+ const uint32_t chunk_size = kparams->chunk_size;const uint32_t total_chunks = (nelem + chunk_size - 1) / chunk_size;const uint32_t n_threads = (total_chunks >= 2) ? MIN(octx->n_threads, total_chunks) : 1;const uint32_t chunks_per_thread = (total_chunks + n_threads - 1) / n_threads;@@ -1101,6 +1130,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {.chunk_size = chunk_size,.chunks_per_thread = chunks_per_thread,.total_chunks = total_chunks,+ .n_srcs = n_srcs,.is_scalar = is_scalar,.scalar_f32 = scalar_f32,.scalar_f16 = scalar_f16,@@ -1146,65 +1176,6 @@ static int execute_op_binary(struct htp_ops_context * octx) {return HTP_STATUS_VTCM_TOO_SMALL;}- dma_queue * dma_q = octx->ctx->dma[0];- uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src1);- if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {- dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[0] * elem_size, 1);- } else if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR ||- kparams->kernel_type == HTP_BINARY_KERNEL_REPEAT ||- kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX) {- if (!kparams->src1_strided) {- const size_t src1_bytes = (size_t) src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3] * elem_size;- dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1_bytes, 1);- } else {- const uint32_t ne11 = src1->ne[1];- const uint32_t ne12 = src1->ne[2];- const uint32_t ne13 = src1->ne[3];- const size_t row_bytes = src1->ne[0] * elem_size;- for (uint32_t i13 = 0; i13 < ne13; i13++) {- for (uint32_t i12 = 0; i12 < ne12; i12++) {- dma_addr_t src_addr = src1->data + i13 * src1->nb[3] + i12 * src1->nb[2];- uint8_t * dst_ptr = vtcm_src1 + (i13 * ne12 * ne11 + i12 * ne11) * row_bytes;- while (!dma_queue_push(dma_q, dma_make_data(dst_ptr, src_addr),- row_bytes, src1->nb[1], row_bytes, ne11)) {- dma_queue_pop(dma_q);- }- }- }- }- } else if (kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) {- const struct htp_tensor * src2 = octx->src[2];- uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2);- dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data),- kparams->src1_row_size_aligned, src1->nb[1],- src1->ne[0] * elem_size, src1->ne[1]);- const size_t src2_row_bytes = src2->ne[0] * sizeof(int32_t);- dma_queue_push(dma_q, dma_make_data(vtcm_src2, src2->data),- src2_row_bytes, src2->nb[1],- src2_row_bytes, src2->ne[1]);- }-- bctx.octx = octx;- bctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);- bctx.total_rows = nrows;- bctx.row_start = row_start;- bctx.block_max = bctx.vtcm_layout.rows_per_buffer;- bctx.src0_row_size_aligned = src0_row_size_aligned;- bctx.src1_row_size_aligned = src1_row_size_aligned;- bctx.dst_row_size_aligned = dst_row_size_aligned;- bctx.row_size_bytes = src0_row_size;-- bctx.src0_dim1_div = init_fastdiv_values(src0->ne[1]);- bctx.src0_dim2_div = init_fastdiv_values(src0->ne[2]);- bctx.src0_dim12_div = init_fastdiv_values(src0->ne[1] * src0->ne[2]);-- bctx.src1_dim1_div = init_fastdiv_values(src1->ne[1]);- bctx.src1_dim2_div = init_fastdiv_values(src1->ne[2]);- bctx.src1_dim3_div = init_fastdiv_values(src1->ne[3]);-- bctx.split_at_ne01 = kparams->split_at_ne01;- bctx.split_at_ne02 = kparams->split_at_ne02;-worker_callback_t worker_func = NULL;void * compute_func = NULL;@@ -1322,7 +1293,70 @@ static int execute_op_binary(struct htp_ops_context * octx) {return HTP_STATUS_NO_SUPPORT;}- bctx.compute = compute_func;+ dma_queue * dma_q = octx->ctx->dma[0];+ uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src1);+ if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) {+ dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[0] * elem_size, 1);+ } else if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR ||+ kparams->kernel_type == HTP_BINARY_KERNEL_REPEAT ||+ kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX) {+ if (!kparams->src1_strided) {+ const size_t src1_bytes = (size_t) src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3] * elem_size;+ dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1_bytes, 1);+ if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR && octx->src[2] != NULL) {+ uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2);+ dma_queue_push(dma_q, dma_make_data(vtcm_src2, octx->src[2]->data), bctx.vtcm_layout.src2_size, 0, src1_bytes, 1);+ }+ } else {+ const uint32_t ne11 = src1->ne[1];+ const uint32_t ne12 = src1->ne[2];+ const uint32_t ne13 = src1->ne[3];+ const size_t row_bytes = src1->ne[0] * elem_size;+ for (uint32_t i13 = 0; i13 < ne13; i13++) {+ for (uint32_t i12 = 0; i12 < ne12; i12++) {+ dma_addr_t src_addr = src1->data + i13 * src1->nb[3] + i12 * src1->nb[2];+ uint8_t * dst_ptr = vtcm_src1 + (i13 * ne12 * ne11 + i12 * ne11) * row_bytes;+ while (!dma_queue_push(dma_q, dma_make_data(dst_ptr, src_addr),+ row_bytes, src1->nb[1], row_bytes, ne11)) {+ dma_queue_pop(dma_q);+ }+ }+ }+ }+ } else if (kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) {+ const struct htp_tensor * src2 = octx->src[2];+ uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2);+ dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data),+ kparams->src1_row_size_aligned, src1->nb[1],+ src1->ne[0] * elem_size, src1->ne[1]);+ const size_t src2_row_bytes = src2->ne[0] * sizeof(int32_t);+ dma_queue_push(dma_q, dma_make_data(vtcm_src2, src2->data),+ src2_row_bytes, src2->nb[1],+ src2_row_bytes, src2->ne[1]);+ }++ bctx.octx = octx;+ bctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);+ bctx.total_rows = nrows;+ bctx.row_start = row_start;+ bctx.n_srcs = n_srcs;+ bctx.block_max = bctx.vtcm_layout.rows_per_buffer;+ bctx.src0_row_size_aligned = src0_row_size_aligned;+ bctx.src1_row_size_aligned = src1_row_size_aligned;+ bctx.dst_row_size_aligned = dst_row_size_aligned;+ bctx.row_size_bytes = src0_row_size;++ bctx.src0_dim1_div = kparams->div_ne01;+ bctx.src0_dim2_div = kparams->div_ne02;+ bctx.src0_dim12_div = kparams->div_ne012;++ bctx.src1_dim1_div = kparams->div_ne11;+ bctx.src1_dim2_div = kparams->div_ne12;+ bctx.src1_dim3_div = kparams->div_ne13;++ bctx.split_at_ne01 = kparams->split_at_ne01;+ bctx.split_at_ne02 = kparams->split_at_ne02;+ bctx.compute = compute_func;if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST ||kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR ||@@ -1330,6 +1364,15 @@ static int execute_op_binary(struct htp_ops_context * octx) {kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX ||kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) {dma_queue_flush(dma_q);+ if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR && octx->src[2] != NULL && octx->op == HTP_OP_MUL) {+ uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2);+ const uint32_t n_scalars = src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3];+ if (src0_type == HTP_TYPE_F32) {+ hvx_mul_f32_aaa(vtcm_src1, vtcm_src1, vtcm_src2, n_scalars);+ } else if (src0_type == HTP_TYPE_F16) {+ hvx_mul_f16_aaa(vtcm_src1, vtcm_src1, vtcm_src2, n_scalars);+ }+ }}work_queue_run(octx->ctx->work_queue, worker_func, &bctx, n_threads);diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.h b/ggml/src/ggml-hexagon/htp/binary-ops.hindex db3d514895..8c5e683e81 100644--- a/ggml/src/ggml-hexagon/htp/binary-ops.h+++ b/ggml/src/ggml-hexagon/htp/binary-ops.h@@ -6,6 +6,7 @@#include <string.h>#include "hex-common.h"+#include "hex-fastdiv.h"#include "htp-ops.h"#include "htp-vtcm.h"@@ -20,7 +21,15 @@ enum htp_binary_kernel_type {};struct htp_binary_kernel_params {- uint32_t kernel_type;+ uint8_t kernel_type;+ uint8_t split_at_ne01;+ uint8_t split_at_ne02;+ uint8_t src1_strided;++ uint8_t n_srcs;+ uint8_t is_scalar;+ uint8_t pad[2];+uint32_t n_threads;uint32_t rows_per_buffer;@@ -34,12 +43,14 @@ struct htp_binary_kernel_params {uint32_t chunk_size;uint32_t chunk_bytes;- uint32_t is_scalar;- uint8_t split_at_ne01;- uint8_t split_at_ne02;- uint8_t src1_strided;- uint8_t pad;+ struct fastdiv_values div_ne01;+ struct fastdiv_values div_ne02;+ struct fastdiv_values div_ne012;++ struct fastdiv_values div_ne11;+ struct fastdiv_values div_ne12;+ struct fastdiv_values div_ne13;};#if defined(__cplusplus)@@ -100,6 +111,8 @@ static inline void htp_binary_vtcm_layout_build(return;}+ const uint32_t n_srcs = kparams->n_srcs > 2 ? kparams->n_srcs : 2;+if (kparams->kernel_type == HTP_BINARY_KERNEL_CHUNKED) {const size_t chunk_bytes = kparams->chunk_bytes;if (chunk_bytes == 0) {@@ -107,7 +120,7 @@ static inline void htp_binary_vtcm_layout_build(}L->src0_bytes_per_thread = 2 * chunk_bytes;- L->src1_bytes_per_thread = kparams->is_scalar ? 0 : (2 * chunk_bytes);+ L->src1_bytes_per_thread = kparams->is_scalar ? 0 : (2 * (n_srcs - 1) * chunk_bytes);L->dst_bytes_per_thread = 2 * chunk_bytes;L->src0_spad_half_size = chunk_bytes;@@ -135,7 +148,7 @@ static inline void htp_binary_vtcm_layout_build(}const size_t spad_row_total = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE)- ? 2 * (kparams->src0_row_size_aligned + kparams->src1_row_size_aligned + kparams->dst_row_size_aligned)+ ? 2 * (kparams->src0_row_size_aligned + (n_srcs - 1) * kparams->src1_row_size_aligned + kparams->dst_row_size_aligned): 2 * (kparams->src0_row_size_aligned + kparams->dst_row_size_aligned);const size_t static_vtcm_size = kparams->src1_size + kparams->src2_size;@@ -143,7 +156,9 @@ static inline void htp_binary_vtcm_layout_build(return;}- const size_t rows_per_buffer = (vtcm_size - static_vtcm_size) / (n_threads * spad_row_total);+ const size_t rows_per_buffer = (kparams->rows_per_buffer > 0)+ ? kparams->rows_per_buffer+ : (vtcm_size - static_vtcm_size) / (n_threads * spad_row_total);if (rows_per_buffer == 0) {return;}@@ -155,11 +170,13 @@ static inline void htp_binary_vtcm_layout_build(L->src0_bytes_per_thread = rows_per_buffer * 2 * kparams->src0_row_size_aligned;L->dst_bytes_per_thread = rows_per_buffer * 2 * kparams->dst_row_size_aligned;L->src1_bytes_per_thread = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE)- ? rows_per_buffer * 2 * kparams->src1_row_size_aligned+ ? rows_per_buffer * 2 * (n_srcs - 1) * kparams->src1_row_size_aligned: 0;L->src0_spad_half_size = L->src0_bytes_per_thread / 2;- L->src1_spad_half_size = L->src1_bytes_per_thread / 2;+ L->src1_spad_half_size = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE)+ ? rows_per_buffer * kparams->src1_row_size_aligned+ : 0;L->dst_spad_half_size = L->dst_bytes_per_thread / 2;const size_t src0_total = n_threads * L->src0_bytes_per_thread;@@ -178,4 +195,53 @@ static inline void htp_binary_vtcm_layout_build(L->total_bytes = off;}+static inline bool htp_binary_solve_chunked(+ struct htp_binary_vtcm_layout * L,+ struct htp_binary_kernel_params * kparams,+ uint32_t total_elems,+ uint32_t elem_size,+ uint32_t max_threads,+ uint32_t n_srcs,+ bool is_scalar,+ size_t vtcm_size+) {+ const size_t n_bufs = is_scalar ? 2 : (size_t) n_srcs + 1;+ const size_t scalar_bytes = is_scalar ? 128 : 0;+ const size_t budget_per_buf = (max_threads > 0 && vtcm_size > scalar_bytes)+ ? hex_align_down((vtcm_size - scalar_bytes) / (2 * max_threads * n_bufs), 128)+ : 0;+ const uint32_t max_chunk_bytes = (budget_per_buf > 0 && budget_per_buf < 32768) ? (uint32_t) budget_per_buf : 32768u;+ const uint32_t max_chunk_elems = max_chunk_bytes / elem_size;+ if (max_chunk_elems == 0) {+ return false;+ }++ const uint32_t min_chunk_elems = 256 < max_chunk_elems ? 256 : max_chunk_elems;+ const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_threads) - 1) / (2 * max_threads), 32);+ const uint32_t chunk_size = (target_chunk_elems < min_chunk_elems) ? min_chunk_elems+ : (target_chunk_elems > max_chunk_elems) ? max_chunk_elems+ : target_chunk_elems;+ const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128);+ const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size;+ const uint32_t n_threads = (total_chunks >= 2) ? (max_threads < total_chunks ? max_threads : total_chunks) : 1;++ kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED;+ kparams->n_threads = n_threads;+ kparams->n_srcs = (uint8_t) n_srcs;+ kparams->rows_per_buffer = 1;+ kparams->src1_row_size_aligned = is_scalar ? 0 : kparams->src0_row_size_aligned;+ kparams->src1_size = 0;+ kparams->chunk_size = chunk_size;+ kparams->chunk_bytes = chunk_bytes;+ kparams->is_scalar = is_scalar ? 1 : 0;++ htp_binary_vtcm_layout_build(L, kparams, vtcm_size);+ if (L->total_bytes == 0 || L->total_bytes > vtcm_size) {+ return false;+ }++ kparams->vtcm_size = (uint32_t) L->total_bytes;+ return true;+}+#endifdiff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.cindex 40fa09151b..20369884b4 100644--- a/ggml/src/ggml-hexagon/htp/concat-ops.c+++ b/ggml/src/ggml-hexagon/htp/concat-ops.c@@ -294,22 +294,21 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_conconst uint32_t n_threads = octx->n_threads;- // layout precomputed on host; kept for reference:- // struct htp_concat_transposed_vtcm_layout layout;- // htp_concat_transposed_vtcm_layout_build(&layout, octx->src[0]->ne[0], octx->src[1]->ne[0], type_size, n_threads);+ struct htp_concat_transposed_vtcm_layout layout;+ htp_concat_transposed_vtcm_layout_build(&layout, octx->src[0]->ne[0], octx->src[1]->ne[0], type_size, n_threads);uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;struct htp_concat_context cctx;cctx.octx = octx;- cctx.spad0_base = vtcm_base;- cctx.spad1_base = vtcm_base + n_threads * kparams->u.transposed.spad0_size_per_thread;- cctx.spad0_size_per_thread = kparams->u.transposed.spad0_size_per_thread;- cctx.spad1_size_per_thread = kparams->u.transposed.spad1_size_per_thread;+ cctx.spad0_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad0);+ cctx.spad1_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad1);+ cctx.spad0_size_per_thread = layout.src0_spad_size_per_thread;+ cctx.spad1_size_per_thread = layout.src1_spad_size_per_thread;cctx.row_start = row_start;cctx.nrows = nrows;cctx.nplanes = dst->ne[2] * dst->ne[3];- cctx.div_ne2 = init_fastdiv_values(dst->ne[2]);+ cctx.div_ne2 = kparams->u.transposed.div_ne2;cctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);work_queue_func_t worker_func = (type_size == 4) ? concat_2d_f32_transposed : concat_2d_f16_transposed;diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.h b/ggml/src/ggml-hexagon/htp/concat-ops.hindex 4bfba9894a..a5d93284fe 100644--- a/ggml/src/ggml-hexagon/htp/concat-ops.h+++ b/ggml/src/ggml-hexagon/htp/concat-ops.h@@ -2,6 +2,8 @@#define HTP_CONCAT_OPS_H#include "hex-common.h"+#include "hex-fastdiv.h"+#include "htp-vtcm.h"#include "gather-rows.h"#include <stdint.h>@@ -13,8 +15,7 @@ enum htp_concat_kernel_type {};struct htp_concat_transposed_params {- uint32_t spad0_size_per_thread;- uint32_t spad1_size_per_thread;+ struct fastdiv_values div_ne2;};struct htp_concat_kernel_params {@@ -40,11 +41,13 @@ _Static_assert(sizeof(struct htp_concat_kernel_params) <= 128, "htp_concat_kernestruct htp_concat_transposed_vtcm_layout {uint32_t src0_spad_size_per_thread;uint32_t src1_spad_size_per_thread;+ size_t off_spad0;+ size_t off_spad1;uint32_t total_bytes;};static inline void htp_concat_transposed_vtcm_layout_build(- struct htp_concat_transposed_vtcm_layout * layout,+ struct htp_concat_transposed_vtcm_layout * L,uint32_t src0_ne0,uint32_t src1_ne0,uint32_t type_size,@@ -55,9 +58,13 @@ static inline void htp_concat_transposed_vtcm_layout_build(uint32_t src1_ne0_padded = hex_round_up(src1_ne0, block_i);uint32_t spad0_row_bytes = hex_round_up(src0_ne0 * type_size, 128) + src1_ne0_padded * type_size;- layout->src0_spad_size_per_thread = block_i * spad0_row_bytes;- layout->src1_spad_size_per_thread = src1_ne0_padded * spad1_stride;- layout->total_bytes = n_threads * (layout->src0_spad_size_per_thread + layout->src1_spad_size_per_thread);+ L->src0_spad_size_per_thread = block_i * spad0_row_bytes;+ L->src1_spad_size_per_thread = src1_ne0_padded * spad1_stride;++ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_spad0, (size_t) n_threads * L->src0_spad_size_per_thread);+ VTCM_LAYOUT_ALLOC(off, off_spad1, (size_t) n_threads * L->src1_spad_size_per_thread);+ L->total_bytes = (uint32_t) off;}#endif // HTP_CONCAT_OPS_Hdiff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.cindex 63ebb38e68..fce45c2d47 100644--- a/ggml/src/ggml-hexagon/htp/cpy-ops.c+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c@@ -32,6 +32,7 @@ struct htp_copy_context {uint32_t nelem;uint32_t elem_per_thread;+ struct htp_copy_convert_vtcm_layout layout;uint8_t * vtcm_src0;uint8_t * vtcm_dst;};@@ -176,13 +177,14 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vostruct htp_thread_trace * tr = &octx->ctx->trace[ith]; \\const struct htp_copy_convert_params * cvt = &ct->kparams->u.convert; \- const uint32_t src0_buf_size = cvt->src0_buf_size; \- const uint32_t dst_buf_size = cvt->dst_buf_size; \- const uint32_t src0_row_stride = cvt->src0_row_stride; \- const uint32_t dst_row_stride = cvt->dst_row_stride; \- const uint32_t blk_rows = cvt->blk_rows; \- uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * cvt->spad0_size_per_thread; \- uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * cvt->spad1_size_per_thread; \+ const struct htp_copy_convert_vtcm_layout * L = &ct->layout; \+ const uint32_t src0_buf_size = L->src0_buf_size; \+ const uint32_t dst_buf_size = L->dst_buf_size; \+ const uint32_t src0_row_stride = L->src0_row_stride; \+ const uint32_t dst_row_stride = L->dst_row_stride; \+ const uint32_t blk_rows = L->blk_rows; \+ uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * L->spad0_size_per_thread; \+ uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * L->spad1_size_per_thread; \const uint32_t src0_row_size = ne00 * ct->kparams->src0_type_size; \const uint32_t dst_row_size = ne00 * ct->kparams->dst_type_size; \\@@ -254,17 +256,16 @@ static int cpy_scalar(struct htp_ops_context * octx, const struct htp_copy_kerne}dma_queue * dma_q = octx->ctx->dma[0];- dma_addr_t s_vtcm = (dma_addr_t)(uintptr_t) octx->ctx->vtcm_base;- dma_addr_t d_vtcm = s_vtcm + VLEN;+ uint8_t * s_ptr = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, 0);+ uint8_t * d_ptr = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, VLEN);+ dma_addr_t s_vtcm = (dma_addr_t)(uintptr_t) s_ptr;+ dma_addr_t d_vtcm = (dma_addr_t)(uintptr_t) d_ptr;const uint32_t s_size = kparams->src0_type_size;const uint32_t d_size = kparams->dst_type_size;dma_queue_push(dma_q, dma_make_data(s_vtcm, src0->data), s_size, s_size, s_size, 1);dma_queue_pop(dma_q);- uint8_t * s_ptr = (uint8_t *) octx->ctx->vtcm_base;- uint8_t * d_ptr = s_ptr + VLEN;-const HVX_Vector v_src = hvx_vmem(s_ptr);HVX_Vector v_dst;@@ -366,6 +367,8 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htpconst struct htp_copy_convert_params * cvt = &kparams->u.convert;struct htp_copy_context ct;+ htp_copy_convert_vtcm_layout_build(&ct.layout, src0->ne[0], kparams->src0_type_size, kparams->dst_type_size, cvt->blk_rows, n_threads);+ct.octx = octx;ct.kparams = kparams;ct.row_start = row_start;@@ -373,8 +376,8 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htpct.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;- ct.vtcm_src0 = vtcm_base;- ct.vtcm_dst = vtcm_base + (size_t) n_threads * cvt->spad0_size_per_thread;+ ct.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, ct.layout.off_src0);+ ct.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, ct.layout.off_dst);work_queue_func_t copy_fun = NULL;if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) {diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.h b/ggml/src/ggml-hexagon/htp/cpy-ops.hindex 16695c2ab6..c3b97c0cab 100644--- a/ggml/src/ggml-hexagon/htp/cpy-ops.h+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.h@@ -3,6 +3,7 @@#include "hex-common.h"#include "hex-fastdiv.h"+#include "htp-vtcm.h"#include "gather-rows.h"#include <stdint.h>@@ -17,12 +18,6 @@ enum htp_copy_kernel_type {};struct htp_copy_convert_params {- uint32_t src0_buf_size;- uint32_t dst_buf_size;- uint32_t spad0_size_per_thread;- uint32_t spad1_size_per_thread;- uint32_t src0_row_stride; // row pitch in the VTCM buffers- uint32_t dst_row_stride;uint32_t blk_rows; // rows moved per DMA descriptorstruct fastdiv_values div_ne01;struct fastdiv_values div_ne02_ne01;@@ -60,6 +55,9 @@ struct htp_copy_kernel_params {#define HTP_COPY_VTCM_BLK_BYTES (32 * 1024)struct htp_copy_convert_vtcm_layout {+ size_t total_bytes;+ size_t off_src0;+ size_t off_dst;uint32_t src0_row_stride;uint32_t dst_row_stride;uint32_t blk_rows;@@ -67,11 +65,35 @@ struct htp_copy_convert_vtcm_layout {uint32_t dst_buf_size;uint32_t spad0_size_per_thread;uint32_t spad1_size_per_thread;- uint32_t total_bytes;};-// ne01 bounds a block: rows of one block share the dim-1 stride.static inline void htp_copy_convert_vtcm_layout_build(+ struct htp_copy_convert_vtcm_layout * L,+ uint32_t ne00,+ uint32_t src_type_size,+ uint32_t dst_type_size,+ uint32_t blk_rows,+ uint32_t n_threads) {++ const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256);+ const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256);++ L->src0_row_stride = src0_row_stride;+ L->dst_row_stride = dst_row_stride;+ L->blk_rows = blk_rows;+ L->src0_buf_size = blk_rows * src0_row_stride;+ L->dst_buf_size = blk_rows * dst_row_stride;+ L->spad0_size_per_thread = 2 * L->src0_buf_size;+ L->spad1_size_per_thread = 2 * L->dst_buf_size;++ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_src0, (size_t) n_threads * L->spad0_size_per_thread);+ VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) n_threads * L->spad1_size_per_thread);+ L->total_bytes = off;+}++// ne01 bounds a block: rows of one block share the dim-1 stride.+static inline void htp_copy_convert_solve_layout(struct htp_copy_convert_vtcm_layout * layout,uint32_t ne00,uint32_t ne01,@@ -79,10 +101,10 @@ static inline void htp_copy_convert_vtcm_layout_build(uint32_t dst_type_size,uint32_t n_threads) {- layout->src0_row_stride = hex_round_up(ne00 * src_type_size, 256);- layout->dst_row_stride = hex_round_up(ne00 * dst_type_size, 256);+ const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256);+ const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256);- const uint32_t max_row = layout->src0_row_stride > layout->dst_row_stride ? layout->src0_row_stride : layout->dst_row_stride;+ const uint32_t max_row = src0_row_stride > dst_row_stride ? src0_row_stride : dst_row_stride;uint32_t blk_rows = HTP_COPY_VTCM_BLK_BYTES / max_row;if (blk_rows > ne01) {blk_rows = ne01;@@ -90,13 +112,8 @@ static inline void htp_copy_convert_vtcm_layout_build(if (blk_rows == 0) {blk_rows = 1;}- layout->blk_rows = blk_rows;- layout->src0_buf_size = blk_rows * layout->src0_row_stride;- layout->dst_buf_size = blk_rows * layout->dst_row_stride;- layout->spad0_size_per_thread = 2 * layout->src0_buf_size;- layout->spad1_size_per_thread = 2 * layout->dst_buf_size;- layout->total_bytes = n_threads * (layout->spad0_size_per_thread + layout->spad1_size_per_thread);+ htp_copy_convert_vtcm_layout_build(layout, ne00, src_type_size, dst_type_size, blk_rows, n_threads);}#if defined(__cplusplus)diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.c b/ggml/src/ggml-hexagon/htp/cumsum-ops.cindex eaab7d7e51..36fc812760 100644--- a/ggml/src/ggml-hexagon/htp/cumsum-ops.c+++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.c@@ -12,9 +12,11 @@#include "htp-ctx.h"#include "htp-ops.h"#include "htp-tensor.h"+#include "htp-vtcm.h"#include "hvx-types.h"#include "hvx-utils.h"#include "dma-queue.h"+#include "cumsum-ops.h"#define htp_cumsum_tensors_preamble \const struct htp_tensor * restrict src0 = octx->src[0]; \@@ -89,8 +91,8 @@ static inline HVX_Vector hvx_splat_last_f32(HVX_Vector v) {}static inline void hvx_cumsum_row_f32(const float * restrict src, float * restrict dst, uint32_t n) {- const uint32_t nvec = n / VLEN_FP32;- const uint32_t nloe = n % VLEN_FP32;+ const uint32_t nvec = n >> 5;+ const uint32_t nloe = n & (VLEN_FP32 - 1);HVX_Vector carry = Q6_V_vsplat_R(0);@@ -210,16 +212,19 @@ static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) {int op_cumsum_f32(struct htp_ops_context * octx) {const struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * dst = octx->dst;+ const struct htp_cumsum_kernel_params * kparams =+ (const struct htp_cumsum_kernel_params *) octx->kernel_params;- const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];- const size_t dst_data_row_size = dst->ne[0] * sizeof(float);+ const uint32_t total_rows = kparams->total_rows;+ const size_t src_row_size = src0->nb[1];+ const size_t dst_row_size = dst->nb[1];uint32_t row_start = 0;uint32_t nrows = total_rows;if (octx->ctx->mdev.count > 1) {uint32_t rows_per_chunk = 0;- htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk);+ htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_row_size, &rows_per_chunk);const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);row_start = range.start;nrows = range.count;@@ -229,43 +234,45 @@ int op_cumsum_f32(struct htp_ops_context * octx) {return HTP_STATUS_OK;}- const uint32_t n_threads = octx->n_threads;-- const size_t src_row_size = src0->nb[1];- const size_t dst_row_size = dst->nb[1];- const size_t src_row_size_aligned = hex_round_up(src_row_size, VLEN);- const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);-- // 2 ping-pong buffers per thread for src and dst- const size_t spad_per_thread = 2 * (src_row_size_aligned + dst_row_size_aligned);+ if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {+ return HTP_STATUS_INVAL_PARAMS;+ }- octx->src0_spad.size_per_thread = src_row_size_aligned * 2;- octx->dst_spad.size_per_thread = dst_row_size_aligned * 2;+ const uint32_t n_threads = octx->n_threads;- octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread;- octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread;+ if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {+ return HTP_STATUS_NO_SUPPORT;+ }- octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL;- octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->dst_spad.src = NULL;+ struct htp_cumsum_vtcm_layout layout = { 0 };+ if (kparams->use_dma) {+ htp_cumsum_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_threads);++ octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread;+ octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread;+ octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread;+ octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread;+ octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0);+ octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst);+ octx->src0_spad.src = NULL;+ octx->dst_spad.src = NULL;+ }struct htp_cumsum_context cctx = {.octx = octx,.src_row_size = src_row_size,.dst_row_size = dst_row_size,- .src_row_size_aligned = src_row_size_aligned,- .dst_row_size_aligned = dst_row_size_aligned,+ .src_row_size_aligned = layout.src_row_size_aligned,+ .dst_row_size_aligned = layout.dst_row_size_aligned,.rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),.total_rows = nrows,.row_start = row_start,};- if (octx->ctx->vtcm_size < spad_per_thread * n_threads) {- if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {- return HTP_STATUS_NO_SUPPORT;- }- work_queue_run(octx->ctx->work_queue, cumsum_thread_f32, &cctx, n_threads);- } else {+ if (kparams->use_dma) {work_queue_run(octx->ctx->work_queue, cumsum_thread_f32_dma, &cctx, n_threads);+ } else {+ work_queue_run(octx->ctx->work_queue, cumsum_thread_f32, &cctx, n_threads);}return HTP_STATUS_OK;diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.h b/ggml/src/ggml-hexagon/htp/cumsum-ops.hnew file mode 100644index 0000000000..6e1ac316fe--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.h@@ -0,0 +1,49 @@+#ifndef HTP_CUMSUM_OPS_H+#define HTP_CUMSUM_OPS_H++#include "hex-common.h"+#include "htp-vtcm.h"+#include <stdint.h>++struct htp_cumsum_vtcm_layout {+ size_t total_bytes;+ size_t off_src0;+ size_t off_dst;+ size_t src_row_size_aligned;+ size_t dst_row_size_aligned;+ size_t src0_bytes_per_thread;+ size_t dst_bytes_per_thread;+};++static inline void htp_cumsum_vtcm_layout_build(+ struct htp_cumsum_vtcm_layout * L,+ size_t src_row_size,+ size_t dst_row_size,+ uint32_t n_threads+) {+ L->src_row_size_aligned = hex_round_up(src_row_size, 128);+ L->dst_row_size_aligned = hex_round_up(dst_row_size, 128);+ L->src0_bytes_per_thread = 2 * L->src_row_size_aligned;+ L->dst_bytes_per_thread = 2 * L->dst_row_size_aligned;++ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);++ L->total_bytes = off;+}++struct htp_cumsum_kernel_params {+ uint32_t n_threads;+ uint32_t total_rows;+ uint32_t use_dma;+ uint32_t vtcm_size;+};++#if defined(__cplusplus)+static_assert(sizeof(struct htp_cumsum_kernel_params) <= 128, "htp_cumsum_kernel_params is too large for kernel_params blob");+#else+_Static_assert(sizeof(struct htp_cumsum_kernel_params) <= 128, "htp_cumsum_kernel_params is too large for kernel_params blob");+#endif++#endif // HTP_CUMSUM_OPS_Hdiff --git a/ggml/src/ggml-hexagon/htp/diag-ops.c b/ggml/src/ggml-hexagon/htp/diag-ops.cindex 162214d3e4..7f03e85501 100644--- a/ggml/src/ggml-hexagon/htp/diag-ops.c+++ b/ggml/src/ggml-hexagon/htp/diag-ops.c@@ -10,10 +10,12 @@#include "htp-ctx.h"#include "htp-ops.h"#include "htp-tensor.h"+#include "htp-vtcm.h"#include "hvx-types.h"#include "hex-utils.h"#include "hvx-copy.h"#include "dma-queue.h"+#include "diag-ops.h"#define htp_diag_tensors_preamble \const struct htp_tensor * restrict src0 = octx->src[0]; \@@ -33,6 +35,7 @@struct htp_diag_context {struct htp_ops_context * octx;+ const struct htp_diag_kernel_params * kparams;size_t src_batch_size;size_t dst_row_size;size_t src_batch_size_aligned;@@ -83,8 +86,8 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data)struct htp_thread_trace * tr = &octx->ctx->trace[ith];for (uint32_t ib = ib0; ib < ib1; ib++) {- const uint32_t i3 = ib / ne02;- const uint32_t i2 = ib % ne02;+ const uint32_t i3 = fastdiv(ib, &dctx->kparams->div_ne02);+ const uint32_t i2 = ib - i3 * ne02;const dma_addr_t src_batch = src_data + i3 * nb03 + i2 * nb02;@@ -134,8 +137,8 @@ static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) {htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0);for (uint32_t ib = ib0; ib < ib1; ib++) {- const uint32_t i3 = ib / ne02;- const uint32_t i2 = ib % ne02;+ const uint32_t i3 = fastdiv(ib, &dctx->kparams->div_ne02);+ const uint32_t i2 = ib - i3 * ne02;const float * restrict src_batch = (const float *)(src_data + i3 * nb03 + i2 * nb02);@@ -155,29 +158,17 @@ static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) {int op_diag_f32(struct htp_ops_context * octx) {const struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * dst = octx->dst;+ const struct htp_diag_kernel_params * kparams =+ (const struct htp_diag_kernel_params *) octx->kernel_params;- const uint32_t total_batches = src0->ne[2] * src0->ne[3];- const size_t dst_batch_size = dst->ne[1] * dst->nb[1];+ const uint32_t total_batches = kparams->total_batches;uint32_t batch_start = 0;uint32_t nbatches = total_batches;if (octx->ctx->mdev.count > 1) {- bool can_split = htp_tensor_mdev_data_aligned(dst) && (dst->ne[0] == 1 || dst->nb[0] == sizeof(float)) && !htp_tensor_is_permuted(dst);- uint32_t batches_per_chunk = 1;- if (can_split) {- if (dst->ne[2] > 1 && (dst->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 &&- (dst->ne[3] <= 1 || (dst->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0)) {- batches_per_chunk = 1;- } else if (dst->nb[2] == dst_batch_size &&- (dst->ne[3] <= 1 || dst->nb[3] == dst->nb[2] * dst->ne[2])) {- batches_per_chunk = (dst_batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_batch_size, HEX_L2_LINE_SIZE)) : 1;- } else {- can_split = false;- }- }-- const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, can_split ? batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);+ const bool can_split = (kparams->can_split != 0) && htp_tensor_mdev_data_aligned(dst);+ const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, can_split ? kparams->batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);batch_start = range.start;nbatches = range.count;}@@ -186,43 +177,49 @@ int op_diag_f32(struct htp_ops_context * octx) {return HTP_STATUS_OK;}- const uint32_t n_threads = octx->n_threads;-- const size_t src_batch_size = src0->ne[0] * sizeof(float);- const size_t dst_row_size = dst->ne[0] * sizeof(float);- const size_t src_batch_size_aligned = hex_round_up(src_batch_size, VLEN);- const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);-- // 1 src buffer + 1 dst row buffer per thread- const size_t spad_per_thread = src_batch_size_aligned + dst_row_size_aligned;+ if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nbatches))) {+ return HTP_STATUS_INVAL_PARAMS;+ }- octx->src0_spad.size_per_thread = src_batch_size_aligned;- octx->dst_spad.size_per_thread = dst_row_size_aligned;+ const uint32_t n_threads = octx->n_threads;- octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread;- octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread;+ if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {+ return HTP_STATUS_NO_SUPPORT;+ }- octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL;- octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->dst_spad.src = NULL;+ const size_t src_batch_size = src0->ne[0] * sizeof(float);+ const size_t dst_row_size = dst->ne[0] * sizeof(float);++ struct htp_diag_vtcm_layout layout = { 0 };+ if (kparams->use_dma) {+ htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_threads);++ octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread;+ octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread;+ octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread;+ octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread;+ octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0);+ octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst);+ octx->src0_spad.src = NULL;+ octx->dst_spad.src = NULL;+ }struct htp_diag_context dctx = {.octx = octx,+ .kparams = kparams,.src_batch_size = src_batch_size,.dst_row_size = dst_row_size,- .src_batch_size_aligned = src_batch_size_aligned,- .dst_row_size_aligned = dst_row_size_aligned,+ .src_batch_size_aligned = layout.src_batch_size_aligned,+ .dst_row_size_aligned = layout.dst_row_size_aligned,.batches_per_thread = fastdiv(nbatches + n_threads - 1, &octx->n_threads_div),.total_batches = nbatches,.batch_start = batch_start,};- if (octx->ctx->vtcm_size < spad_per_thread * n_threads) {- if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {- return HTP_STATUS_NO_SUPPORT;- }- work_queue_run(octx->ctx->work_queue, diag_thread_f32, &dctx, n_threads);- } else {+ if (kparams->use_dma) {work_queue_run(octx->ctx->work_queue, diag_thread_f32_dma, &dctx, n_threads);+ } else {+ work_queue_run(octx->ctx->work_queue, diag_thread_f32, &dctx, n_threads);}return HTP_STATUS_OK;diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.h b/ggml/src/ggml-hexagon/htp/diag-ops.hnew file mode 100644index 0000000000..983bc02423--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/diag-ops.h@@ -0,0 +1,54 @@+#ifndef HTP_DIAG_OPS_H+#define HTP_DIAG_OPS_H++#include "hex-common.h"+#include "hex-fastdiv.h"+#include "htp-vtcm.h"+#include <stdint.h>++struct htp_diag_vtcm_layout {+ size_t total_bytes;+ size_t off_src0;+ size_t off_dst;+ size_t src_batch_size_aligned;+ size_t dst_row_size_aligned;+ size_t src0_bytes_per_thread;+ size_t dst_bytes_per_thread;+};++static inline void htp_diag_vtcm_layout_build(+ struct htp_diag_vtcm_layout * L,+ size_t src_batch_size,+ size_t dst_row_size,+ uint32_t n_threads+) {+ L->src_batch_size_aligned = hex_round_up(src_batch_size, 128);+ L->dst_row_size_aligned = hex_round_up(dst_row_size, 128);+ L->src0_bytes_per_thread = L->src_batch_size_aligned;+ L->dst_bytes_per_thread = L->dst_row_size_aligned;++ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);++ L->total_bytes = off;+}++struct htp_diag_kernel_params {+ uint32_t n_threads;+ uint32_t total_batches;+ uint32_t use_dma;+ uint32_t can_split;+ uint32_t batches_per_chunk;+ uint32_t vtcm_size;++ struct fastdiv_values div_ne02;+};++#if defined(__cplusplus)+static_assert(sizeof(struct htp_diag_kernel_params) <= 128, "htp_diag_kernel_params is too large for kernel_params blob");+#else+_Static_assert(sizeof(struct htp_diag_kernel_params) <= 128, "htp_diag_kernel_params is too large for kernel_params blob");+#endif++#endif // HTP_DIAG_OPS_Hdiff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.hindex 9b774bdd9c..05f37305d9 100644--- a/ggml/src/ggml-hexagon/htp/dma-queue.h+++ b/ggml/src/ggml-hexagon/htp/dma-queue.h@@ -436,10 +436,10 @@ static inline void dma_sync_write(dma_queue * dma_q, dma_addr_t dst, const void}}-#define DMA_CACHE_MAX_SIZE 256U+#define DMA_CACHE_MAX_SIZE 128U// Fully assoc LRU cache-typedef struct {+typedef struct __attribute__((aligned(128))) {uint8_t *base;uint32_t line_size;uint32_t capacity;@@ -485,7 +485,7 @@ static inline bool dma_cache_fa_push(dma_queue *q, dma_cache_fa *c, dma_addr_t s}// Direct mapped cache-typedef struct {+typedef struct __attribute__((aligned(128))) {uint8_t *base;uint32_t line_size;uint32_t capacity;diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.c b/ggml/src/ggml-hexagon/htp/fill-ops.cindex 212104a23b..2cab31b5fd 100644--- a/ggml/src/ggml-hexagon/htp/fill-ops.c+++ b/ggml/src/ggml-hexagon/htp/fill-ops.c@@ -16,6 +16,7 @@#include "htp-ctx.h"#include "htp-ops.h"#include "htp-tensor.h"+#include "fill-ops.h"// ggml op_params layout for FILL:// op_params[0] (as float) - the scalar fill value@@ -30,21 +31,19 @@\const uint32_t nb1 = dst->nb[1]; \const uint32_t nb2 = dst->nb[2]; \- const uint32_t nb3 = dst->nb[3]; \- \- const uint32_t nr = ne1 * ne2 * ne3;+ const uint32_t nb3 = dst->nb[3];struct htp_fill_context {struct htp_ops_context * octx;+ const struct htp_fill_kernel_params * kparams;uint32_t nrows_per_thread;uint32_t total_rows; // ne1 * ne2 * ne3uint32_t row_start;- bool opt_path;HVX_Vector splat_vec;uint32_t elem_size;};-static void fill_thread(unsigned int nth, unsigned int ith, void * data) {+static void fill_thread_contiguous(unsigned int nth, unsigned int ith, void * data) {const struct htp_fill_context * fctx = (const struct htp_fill_context *) data;struct htp_ops_context * octx = fctx->octx;fill_preamble;@@ -60,45 +59,59 @@ static void fill_thread(unsigned int nth, unsigned int ith, void * data) {struct htp_thread_trace * tr = &octx->ctx->trace[ith];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);- if (fctx->opt_path) {- // Opt path: tensor is fully contiguous, treat as flat array- const uint32_t elem_start = ir0 * ne0;- const uint32_t elem_end = ir1 * ne0;- uint8_t * dst_ptr = (uint8_t *) dst->data + elem_start * fctx->elem_size;- hvx_splat_u(dst_ptr, fctx->splat_vec, elem_end - elem_start, fctx->elem_size);- } else {- // Non-contiguous path: must respect strides- for (uint32_t ir = ir0; ir < ir1; ++ir) {- const uint32_t i1 = ir % ne1;- const uint32_t i2 = (ir / ne1) % ne2;- const uint32_t i3 = ir / (ne1 * ne2);- uint8_t * dst_ptr = (uint8_t *) dst->data + i1*nb1 + i2*nb2 + i3*nb3;- hvx_splat_u(dst_ptr, fctx->splat_vec, ne0, fctx->elem_size);- }- }+ const uint32_t elem_start = ir0 * ne0;+ const uint32_t elem_end = ir1 * ne0;+ uint8_t * dst_ptr = (uint8_t *) dst->data + elem_start * fctx->elem_size;+ hvx_splat_u(dst_ptr, fctx->splat_vec, elem_end - elem_start, fctx->elem_size);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1);- FARF(HIGH, "fill %u/%u: rows %u:%u\n", ith, nth, ir0, ir1);+ FARF(HIGH, "fill-contig %u/%u: rows %u:%u\n", ith, nth, ir0, ir1);}-int op_fill(struct htp_ops_context * octx) {+static void fill_thread_strided(unsigned int nth, unsigned int ith, void * data) {+ const struct htp_fill_context * fctx = (const struct htp_fill_context *) data;+ struct htp_ops_context * octx = fctx->octx;+ const struct htp_fill_kernel_params * kparams = fctx->kparams;fill_preamble;- if (dst->type != HTP_TYPE_F32 && dst->type != HTP_TYPE_F16) {- return HTP_STATUS_NO_SUPPORT;+ // Parallelise over the flat row index spanning ne1*ne2*ne3+ const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith;+ const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows);++ if (ir0 >= ir1) {+ return;+ }++ struct htp_thread_trace * tr = &octx->ctx->trace[ith];+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);++ for (uint32_t ir = ir0; ir < ir1; ++ir) {+ const uint32_t i3 = fastdiv(ir, &kparams->div_ne1_ne2);+ const uint32_t rem = ir - i3 * (ne1 * ne2);+ const uint32_t i2 = fastdiv(rem, &kparams->div_ne1);+ const uint32_t i1 = rem - i2 * ne1;+ uint8_t * dst_ptr = (uint8_t *) dst->data + i1*nb1 + i2*nb2 + i3*nb3;+ hvx_splat_u(dst_ptr, fctx->splat_vec, ne0, fctx->elem_size);}+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1);+ FARF(HIGH, "fill-strided %u/%u: rows %u:%u\n", ith, nth, ir0, ir1);+}++int op_fill(struct htp_ops_context * octx) {+ const struct htp_tensor * dst = octx->dst;+ const struct htp_fill_kernel_params * kparams =+ (const struct htp_fill_kernel_params *) octx->kernel_params;+if (htp_tensor_is_extended(dst)) {return HTP_STATUS_NO_SUPPORT;}uint32_t row_start = 0;- uint32_t nrows = nr;+ uint32_t nrows = kparams->total_rows;if (octx->ctx->mdev.count > 1) {- const uint32_t row_size = nb1;- const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;- const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(nr, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);+ const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);row_start = range.start;nrows = range.count;}@@ -107,40 +120,43 @@ int op_fill(struct htp_ops_context * octx) {return HTP_STATUS_OK;}- // nr = ne1*ne2*ne3 (flat row count across all outer dims); parallelise over it.- const uint32_t n_threads = octx->n_threads;+ if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {+ return HTP_STATUS_INVAL_PARAMS;+ }- // Optimize if fully contiguous: skip stride arithmetic, treat as flat array- const bool opt_path = (nb2 == nb1 * ne1) && (nb3 == nb2 * ne2);+ const uint32_t n_threads = octx->n_threads;- FARF(HIGH, "fill: (%ux%ux%ux%u) type=%u opt=%d\n",- dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], dst->type, (int) opt_path);+ FARF(HIGH, "fill: (%ux%ux%ux%u) type=%u ktype=%u\n",+ dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], dst->type, kparams->kernel_type);float val_f32 = 0.f;memcpy(&val_f32, &octx->op_params[0], sizeof(float));struct htp_fill_context fctx = {.octx = octx,+ .kparams = kparams,.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),.total_rows = nrows,.row_start = row_start,- .opt_path = opt_path,+ .elem_size = kparams->elem_size,};switch (dst->type) {case HTP_TYPE_F32:fctx.splat_vec = hvx_vec_splat_f32(val_f32);- fctx.elem_size = sizeof(float);break;case HTP_TYPE_F16:fctx.splat_vec = hvx_vec_splat_f16((_Float16) val_f32);- fctx.elem_size = sizeof(_Float16);break;default:return HTP_STATUS_NO_SUPPORT;}- work_queue_run(octx->ctx->work_queue, fill_thread, &fctx, n_threads);+ work_queue_func_t worker = (kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS)+ ? fill_thread_contiguous+ : fill_thread_strided;++ work_queue_run(octx->ctx->work_queue, worker, &fctx, n_threads);return HTP_STATUS_OK;}diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.h b/ggml/src/ggml-hexagon/htp/fill-ops.hnew file mode 100644index 0000000000..d62f997513--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/fill-ops.h@@ -0,0 +1,29 @@+#ifndef HTP_FILL_OPS_H+#define HTP_FILL_OPS_H++#include "hex-fastdiv.h"+#include <stdint.h>++enum htp_fill_kernel_type {+ HTP_FILL_KERNEL_CONTIGUOUS = 0,+ HTP_FILL_KERNEL_STRIDED = 1,+};++struct htp_fill_kernel_params {+ uint8_t kernel_type;+ uint32_t n_threads;+ uint32_t total_rows;+ uint32_t elem_size;+ uint32_t rows_per_chunk;++ struct fastdiv_values div_ne1;+ struct fastdiv_values div_ne1_ne2;+};++#if defined(__cplusplus)+static_assert(sizeof(struct htp_fill_kernel_params) <= 128, "htp_fill_kernel_params is too large for kernel_params blob");+#else+_Static_assert(sizeof(struct htp_fill_kernel_params) <= 128, "htp_fill_kernel_params is too large for kernel_params blob");+#endif++#endif // HTP_FILL_OPS_Hdiff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.cindex 2cbd16e8a5..fc56641850 100644--- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c+++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c@@ -39,6 +39,26 @@// Must be multiple of 32#define FLASH_ATTN_BLOCK_SIZE (32 * 2)+// Large scratch lives in ctx->ddr_spad to keep stack usage low.+// Ops can run on the dspqueue callback thread, which has a small stack.+struct hvx_fa_ddr_spad {+ __fp16 slopes[512];+ uint16_t active_blocks[FA_MAX_KV_BLOCKS];+ uint16_t thread_active_blocks[HTP_MAX_NTHREADS][FA_MAX_KV_BLOCKS] __attribute__((aligned(128)));+};++struct hmx_fa_ddr_spad {+ uint16_t active_blocks[FA_MAX_KV_BLOCKS];+ __fp16 slopes[] __attribute__((aligned(128))); // [g_br]+};++static inline void * fa_get_ddr_spad(const struct htp_ops_context * octx, size_t size) {+ if (!octx->ctx->ddr_spad_base || octx->ctx->ddr_spad_size < size) {+ return NULL;+ }+ return octx->ctx->ddr_spad_base;+}+struct htp_fa_context {const struct htp_ops_context * octx;@@ -61,7 +81,6 @@ struct htp_fa_context {uint32_t n_head_log2;float m0;float m1;- __fp16 slopes[512];uint32_t n_blocks;uint32_t n_active_blocks;@@ -90,7 +109,9 @@ struct htp_fa_context {uint8_t * spad_m;uint8_t * spad_a;float * spad_sinks;- uint16_t active_blocks[FA_MAX_KV_BLOCKS];++ struct hvx_fa_ddr_spad * ddr;+ dma_cache_dm * m_cache; // [n_threads]const struct htp_tensor * k;const struct htp_tensor * v;@@ -156,6 +177,7 @@ struct hmx_fa_context {__fp16 * vtcm_mask_buf; // VTCM mask buffer [Br * m_line], DMA'd per KV block__fp16 * vtcm_slopes; // ALiBi slopes [g_br]float * vtcm_sinks; // Attention sinks+ __fp16 * ddr_slopes; // ALiBi slopes staging buffer in DDR [g_br]size_t row_buf_stride; // HVX vectors per row buffer (Bc/64)size_t mask_buf_row_stride; // elements (__fp16) per row in mask buffersize_t q_tile_bytes;@@ -164,7 +186,7 @@ struct hmx_fa_context {size_t col_vec_bytes;size_t d_tile_bytes;bool mask_broadcast; // true when mask->ne[2] == 1 (head-independent, single 2D DMA)- dma_cache_fa m_cache;+ dma_cache_fa * m_cache;};// Scan mask rows [q_start, q_start + n_rows_q) and build compact list of KV blocks with any entry != -inf.@@ -308,11 +330,11 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *uint8_t * spad_m = factx->spad_m + (mask ? factx->size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * ith;uint8_t * spad_a = factx->spad_a + factx->size_vkq_acc * ith;- dma_cache_dm m_cache;- dma_cache_dm_init(&m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE);+ dma_cache_dm * m_cache = &factx->m_cache[ith];+ dma_cache_dm_init(m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE);- uint16_t thread_active_blocks[FA_MAX_KV_BLOCKS];- const uint16_t * active_blocks = factx->shared_active_blocks ? factx->active_blocks : thread_active_blocks;+ uint16_t * thread_active_blocks = factx->ddr->thread_active_blocks[ith];+ const uint16_t * active_blocks = factx->shared_active_blocks ? factx->ddr->active_blocks : thread_active_blocks;uint32_t n_active_blocks = factx->n_active_blocks;dma_addr_t prev_mp_base = 0;@@ -385,7 +407,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *const uint32_t r_iq1 = (r - r_iq3*neq2*neq1 - r_iq2 * neq1);heads[g] = r_iq2;- const __fp16 slope = factx->slopes[r_iq2];+ const __fp16 slope = factx->ddr->slopes[r_iq2];slope_vecs[g] = hvx_vec_splat_f16(slope);S_vec[g] = hvx_vec_splat_f32(0.0f);@@ -422,7 +444,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *// Maskif (mask) {const dma_addr_t m_src = mp_base + ic_start * sizeof(__fp16);- dma_cache_dm_push(dma_q, &m_cache, ib, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1);+ dma_cache_dm_push(dma_q, m_cache, ib, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1);}}@@ -682,7 +704,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *// Maskif (mask) {const dma_addr_t m_src = mp_base + next_ic_start * sizeof(__fp16);- dma_cache_dm_push(dma_q, &m_cache, next_ib, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1);+ dma_cache_dm_push(dma_q, m_cache, next_ib, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1);}}} // end for g@@ -1887,7 +1909,7 @@ static __attribute__((noinline)) void fa_compute_slopes(const float m0 = factx->m0;const float m1 = factx->m1;- __fp16 temp_slopes[512] __attribute__((aligned(128)));+ __fp16 * temp_slopes = factx->ddr_slopes;if (G <= 32) {// Fast path: Compute G unique slope values in vector registersHVX_Vector v_val = hvx_alibi_slopes(kv_head, G, n_head_log2, m0, m1);@@ -1905,14 +1927,13 @@ static __attribute__((noinline)) void fa_compute_slopes(}}- // Allocate stack buffer to avoid scalar writes to VTCM (which generates L2 misses)- __fp16 local_slopes[n_rows_g] __attribute__((aligned(128)));- for (size_t r = 0; r < n_rows_g; ++r) {- local_slopes[r] = temp_slopes[fastmodulo(r, G, &factx->div_G)];+ // Replicate the G values in DDR to avoid scalar writes to VTCM (which generates L2 misses)+ for (size_t r = G; r < n_rows_g; ++r) {+ temp_slopes[r] = temp_slopes[r - G];}// Copy to VTCM slopes using HVX block copy (both are aligned to 128 bytes)- hvx_copy_f16_aa((uint8_t *)slopes, (const uint8_t *)local_slopes, n_rows_g);+ hvx_copy_f16_aa((uint8_t *)slopes, (const uint8_t *)temp_slopes, n_rows_g);}static void fa_push_mask_dma_gqa(@@ -1958,7 +1979,7 @@ static inline void fa_prefetch_block(dma_queue * dma_q, const struct htp_tensorif (mask) {if (__builtin_expect(factx->mask_broadcast, true)) {const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + prefetch_start * sizeof(__fp16);- dma_cache_fa_push(dma_q, &factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q);+ dma_cache_fa_push(dma_q, factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q);} else {fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, prefetch_start, kv_head, G, m_line_bytes, prefetch_rows, n_rows_q, factx);}@@ -2151,7 +2172,14 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {const size_t m_line_bytes = L.m_line_bytes; // used by the mask DMAs in the KV loop- dma_cache_fa_init(&factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE);+ struct hmx_fa_ddr_spad * ddr = (struct hmx_fa_ddr_spad *) fa_get_ddr_spad(octx, sizeof(struct hmx_fa_ddr_spad) + factx.g_br * sizeof(__fp16));+ if (!ddr) {+ return HTP_STATUS_INTERNAL_ERR;+ }++ factx.ddr_slopes = ddr->slopes;+ factx.m_cache = &octx->dma_cache_fa[0];+ dma_cache_fa_init(factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE);// Head-dim padding: the K/V DMA staging buffers and the flat-Q buffer are laid out// with padded row strides (size_{k,v,q}_row_padded, covering D_pad columns) but the@@ -2210,7 +2238,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {const size_t g_br_actual = hex_align_up(n_rows_g, HMX_FP16_TILE_N_ROWS);const size_t n_row_tiles = g_br_actual / HMX_FP16_TILE_N_ROWS;- uint16_t active_blocks[FA_MAX_KV_BLOCKS];+ uint16_t * active_blocks = ddr->active_blocks;uint32_t n_active_blocks = factx.n_kv_blocks;if (mask && factx.mask_broadcast && factx.n_kv_blocks > 1) {const size_t scratch_bytes = factx.k_tile_bytes * (factx.pipeline ? 2 : 1);@@ -2252,7 +2280,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {if (factx.pipeline && mask) {if (__builtin_expect(factx.mask_broadcast, true)) {const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + b0_start * sizeof(__fp16);- dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], b0_rows * sizeof(__fp16), n_rows_q);+ dma_cache_fa_push(dma_q, factx.m_cache, ms_src, m_line_bytes, mask->nb[1], b0_rows * sizeof(__fp16), n_rows_q);} else {fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, b0_start, kv_head, G, m_line_bytes, b0_rows, n_rows_q, &factx);}@@ -2453,7 +2481,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {if (mask) {if (__builtin_expect(factx.mask_broadcast, true)) {const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + kv_start * sizeof(__fp16);- dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q);+ dma_cache_fa_push(dma_q, factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q);} else {fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, kv_start, kv_head, G, m_line_bytes, kv_rows, n_rows_q, &factx);}@@ -2577,7 +2605,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {if (factx.pipeline && mask) {if (__builtin_expect(factx.mask_broadcast, true)) {const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + b0_start * sizeof(__fp16);- dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], b0_rows * sizeof(__fp16), n_rows_q);+ dma_cache_fa_push(dma_q, factx.m_cache, ms_src, m_line_bytes, mask->nb[1], b0_rows * sizeof(__fp16), n_rows_q);} else {fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, b0_start, next_kv_head, G, m_line_bytes, b0_rows, n_rows_q, &factx);}@@ -2680,8 +2708,15 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {if (n_head > 512) {return HTP_STATUS_NO_SUPPORT;}++ factx.ddr = (struct hvx_fa_ddr_spad *) fa_get_ddr_spad(octx, sizeof(struct hvx_fa_ddr_spad));+ if (!factx.ddr) {+ return HTP_STATUS_INTERNAL_ERR;+ }+ factx.m_cache = octx->dma_cache_dm;+for (uint32_t h = 0; h < n_head; ++h) {- factx.slopes[h] = (__fp16) ((kparams->max_bias > 0.0f) ? alibi_slope(h, factx.n_head_log2, factx.m0, factx.m1) : 1.0f);+ factx.ddr->slopes[h] = (__fp16) ((kparams->max_bias > 0.0f) ? alibi_slope(h, factx.n_head_log2, factx.m0, factx.m1) : 1.0f);}// total rows in q@@ -2754,7 +2789,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {if (has_mask && neq1 == 1 && neq3 == 1 && mask->ne[2] == 1 && factx.n_blocks > 1 && factx.n_blocks <= FA_MAX_KV_BLOCKS) {factx.n_active_blocks = fa_scan_active_blocks(octx->ctx->dma[0], mask, 0, 1, 0, 0, k->ne[1], FLASH_ATTN_BLOCK_SIZE,- factx.n_blocks, factx.spad_k, factx.size_k_block * 2, factx.active_blocks);+ factx.n_blocks, factx.spad_k, factx.size_k_block * 2, factx.ddr->active_blocks);factx.shared_active_blocks = true;}diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h b/ggml/src/ggml-hexagon/htp/flash-attn-ops.hindex 5f6a1e3e68..a3d3805016 100644--- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h+++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h@@ -35,7 +35,6 @@ enum htp_fa_kernel_type {struct htp_fa_kernel_params {uint8_t kernel_type; // enum htp_fa_kernel_typeuint8_t head_split; // 1 = partition by KV heads in multicore, 0 = token partition- uint8_t flags; // reserveduint8_t n_threads; // Number of threads to run// Common parameters@@ -49,9 +48,6 @@ struct htp_fa_kernel_params {float logit_softcap;uint32_t vtcm_size;- uint32_t qrows;- uint32_t qrows_per_thread;- uint32_t qrow_start;float m0;float m1;uint32_t n_head_log2;@@ -67,8 +63,6 @@ struct htp_fa_kernel_params {union {struct {uint32_t g_br;- uint32_t row_buf_stride;- uint32_t mask_buf_row_stride;int32_t mask_broadcast;int32_t pipeline;struct fastdiv_values div_G;@@ -85,6 +79,8 @@ struct htp_fa_kernel_params {#if defined(__cplusplus)static_assert(sizeof(struct htp_fa_kernel_params) <= 128, "htp_fa_kernel_params is too large for kernel_params blob");+#else+_Static_assert(sizeof(struct htp_fa_kernel_params) <= 128, "htp_fa_kernel_params is too large for kernel_params blob");#endif// VTCM region layout for the HMX flash-attention kernel.diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.cindex 1dd828db7b..5252ac3526 100644--- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c+++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c@@ -2270,46 +2270,9 @@ static int gated_delta_net_f32_hmx_chunked(}int op_gated_delta_net(struct htp_ops_context * octx) {- const struct htp_tensor * q = octx->src[0];- const struct htp_tensor * k = octx->src[1];- const struct htp_tensor * v = octx->src[2];- const struct htp_tensor * g = octx->src[3];- const struct htp_tensor * beta = octx->src[4];- const struct htp_tensor * state = octx->src[5];- const struct htp_tensor * dst = octx->dst;-- if (q->type != HTP_TYPE_F32 || k->type != HTP_TYPE_F32 || v->type != HTP_TYPE_F32 ||- g->type != HTP_TYPE_F32 || beta->type != HTP_TYPE_F32 || state->type != HTP_TYPE_F32 ||- dst->type != HTP_TYPE_F32) {- return HTP_STATUS_NO_SUPPORT;- }-- const uint32_t S_v = v->ne[0];- const uint32_t H = v->ne[1];- const uint32_t n_tokens = v->ne[2];- const uint32_t n_seqs = v->ne[3];- const uint32_t K = octx->op_params[0];-- if (S_v == 0 || S_v > HTP_GDN_MAX_SV || H == 0 || n_tokens == 0 || n_seqs == 0) {- return HTP_STATUS_NO_SUPPORT;- }- if ((g->ne[0] != 1 && g->ne[0] != S_v) || beta->ne[0] != 1) {- return HTP_STATUS_NO_SUPPORT;- }- if (q->ne[0] != S_v || k->ne[0] != S_v || q->ne[1] == 0 || k->ne[1] == 0 ||- q->ne[2] != n_tokens || k->ne[2] != n_tokens || q->ne[3] == 0 || k->ne[3] == 0 ||- (n_seqs % q->ne[3]) != 0 || (n_seqs % k->ne[3]) != 0) {- return HTP_STATUS_NO_SUPPORT;- }- // state holds s0 only: [S_v, S_v, H, n_seqs]- if (state->ne[0] != S_v || state->ne[1] != S_v || state->ne[2] != H || state->ne[3] != n_seqs) {- return HTP_STATUS_NO_SUPPORT;- }- if (dst->ne[0] != S_v * H || dst->ne[1] != n_tokens * n_seqs + S_v * n_seqs * K) {- return HTP_STATUS_NO_SUPPORT;- }+ const struct htp_tensor * dst = octx->dst;- for (int i = 0; i < 5; i++) {+ for (int i = 0; i < 6; i++) {if (htp_tensor_is_extended(octx->src[i])) {return HTP_STATUS_NO_SUPPORT;}@@ -2325,68 +2288,12 @@ int op_gated_delta_net(struct htp_ops_context * octx) {}const struct htp_gdn_kernel_params * kparams = (const struct htp_gdn_kernel_params *) octx->kernel_params;- struct htp_gdn_kernel_params kparams_local;- if (!kparams || kparams->S_v == 0) {- const uint32_t rq3 = n_seqs / q->ne[3];- const uint32_t rk3 = n_seqs / k->ne[3];- const uint32_t total_rows = H * n_seqs;- uint32_t n_threads = (total_rows < octx->n_threads) ? total_rows : octx->n_threads;- if (n_threads == 0) {- n_threads = 1;- }-- memset(&kparams_local, 0, sizeof(kparams_local));- kparams_local.n_threads = n_threads;- kparams_local.S_v = S_v;- kparams_local.H = H;- kparams_local.n_tokens = n_tokens;- kparams_local.n_seqs = n_seqs;- kparams_local.K = K;- kparams_local.total_rows = total_rows;- kparams_local.rows_per_thread = (total_rows + n_threads - 1) / n_threads;- const bool can_use_hmx = (octx->ctx->hmx_enabled) &&- (S_v % 64 == 0) &&- (n_tokens >= HTP_GDN_MIN_TOKENS) &&- (g->ne[0] == 1) &&- (K == 1);-- struct htp_gdn_hmx_vtcm_layout hmx_layout_local;- struct htp_gdn_vtcm_layout hvx_layout_local;- uint32_t n_heads_batch = 1;-- if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout_local, S_v, HTP_GDN_CHUNK_SIZE, total_rows, octx->ctx->vtcm_size, n_threads, true, &n_heads_batch)) {- kparams_local.kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED;- kparams_local.pipeline = hmx_layout_local.pipeline ? 1 : 0;- kparams_local.chunk_size = HTP_GDN_CHUNK_SIZE;- kparams_local.n_chunks = (n_tokens + HTP_GDN_CHUNK_SIZE - 1) / HTP_GDN_CHUNK_SIZE;- kparams_local.n_heads_batch = (uint16_t) n_heads_batch;- kparams_local.vtcm_size = (uint32_t) hmx_layout_local.total_bytes;- kparams_local.state_aligned = (uint32_t) hmx_layout_local.state_f32_bytes;- kparams_local.vtcm_per_thread = (uint32_t) (hmx_layout_local.total_bytes / (n_threads > 0 ? n_threads : 1));- } else {- htp_gdn_vtcm_layout_build(&hvx_layout_local, S_v, n_threads);- kparams_local.kernel_type = HTP_GDN_KERNEL_HVX_RECURRENT;- kparams_local.pipeline = 0;- kparams_local.n_heads_batch = 1;- kparams_local.state_aligned = (uint32_t) hvx_layout_local.state_aligned;- kparams_local.vtcm_per_thread = (uint32_t) hvx_layout_local.bytes_per_thread;- kparams_local.vtcm_size = (uint32_t) hvx_layout_local.total_bytes;- }- kparams_local.kda = (g->ne[0] == S_v) ? 1 : 0;- kparams_local.scale = 1.0f / sqrtf((float) S_v);- kparams_local.state_seq_stride = (uint32_t) (state->nb[3] / sizeof(float));- kparams_local.state_size_per_snap = S_v * S_v * H * n_seqs;-- kparams_local.div_H = init_fastdiv_values(H);- kparams_local.div_q1 = init_fastdiv_values(q->ne[1]);- kparams_local.div_k1 = init_fastdiv_values(k->ne[1]);- kparams_local.div_rq3 = init_fastdiv_values(rq3);- kparams_local.div_rk3 = init_fastdiv_values(rk3);- kparams_local.div_n_threads = init_fastdiv_values(n_threads);-- kparams = &kparams_local;+ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {+ return HTP_STATUS_INVAL_PARAMS;}+ const uint32_t S_v = kparams->S_v;+ const uint32_t n_tokens = kparams->n_tokens;const uint32_t total_rows = kparams->total_rows;uint32_t row_start = 0;uint32_t nrows = total_rows;@@ -2416,16 +2323,16 @@ int op_gated_delta_net(struct htp_ops_context * octx) {return gated_delta_net_f32_hmx_chunked(octx, kparams, row_start, nrows);}- const uint32_t n_threads = (nrows < kparams->n_threads) ? nrows : kparams->n_threads;+ const uint32_t n_threads = (nrows < octx->n_threads) ? nrows : octx->n_threads;struct htp_gdn_context gctx;gctx.octx = octx;gctx.kparams = kparams;gctx.row_start = row_start;gctx.nrows = nrows;- gctx.vtcm_base = octx->ctx->vtcm_base;htp_gdn_vtcm_layout_build(&gctx.layout, S_v, n_threads);+ gctx.vtcm_base = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, gctx.layout.off_state);if (gctx.layout.total_bytes > octx->ctx->vtcm_size) {return HTP_STATUS_VTCM_TOO_SMALL;@@ -2433,10 +2340,10 @@ int op_gated_delta_net(struct htp_ops_context * octx) {FARF(HIGH, "gated-delta-net-f32: q(%ux%ux%ux%u) k(%ux%ux%ux%u) v(%ux%ux%ux%u) state(%ux%ux%ux%u) -> (%ux%ux%ux%u) : ""vtcm-size %zu n_threads %u\n",- q->ne[0], q->ne[1], q->ne[2], q->ne[3],- k->ne[0], k->ne[1], k->ne[2], k->ne[3],- v->ne[0], v->ne[1], v->ne[2], v->ne[3],- state->ne[0], state->ne[1], state->ne[2], state->ne[3],+ octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3],+ octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3],+ octx->src[2]->ne[0], octx->src[2]->ne[1], octx->src[2]->ne[2], octx->src[2]->ne[3],+ octx->src[5]->ne[0], octx->src[5]->ne[1], octx->src[5]->ne[2], octx->src[5]->ne[3],dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],gctx.layout.total_bytes, n_threads);diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.hindex 32fb7d24bf..0625c4d44b 100644--- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h+++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h@@ -37,13 +37,7 @@ struct htp_gdn_kernel_params {uint32_t K;uint32_t total_rows;- uint32_t row_start;- uint32_t nrows;- uint32_t rows_per_thread;-uint32_t kda;- uint32_t state_aligned;- uint32_t vtcm_per_thread;uint32_t vtcm_size;uint32_t state_seq_stride;uint32_t state_size_per_snap;@@ -55,7 +49,6 @@ struct htp_gdn_kernel_params {struct fastdiv_values div_k1;struct fastdiv_values div_rq3;struct fastdiv_values div_rk3;- struct fastdiv_values div_n_threads;};#if defined(__cplusplus)@@ -67,18 +60,21 @@ _Static_assert(sizeof(struct htp_gdn_kernel_params) <= 128, "htp_gdn_kernel_parastruct htp_gdn_vtcm_layout {size_t state_aligned;size_t bytes_per_thread;+ size_t off_state;size_t total_bytes;};static inline void htp_gdn_vtcm_layout_build(- struct htp_gdn_vtcm_layout * layout,+ struct htp_gdn_vtcm_layout * L,uint32_t S_v,uint32_t n_threads) {size_t state_bytes = (size_t) S_v * S_v * sizeof(float);- layout->state_aligned = hex_round_up(state_bytes, 128);- layout->bytes_per_thread = 2 * layout->state_aligned;- layout->total_bytes = layout->bytes_per_thread * n_threads;+ L->state_aligned = hex_round_up(state_bytes, 128);+ L->bytes_per_thread = 2 * L->state_aligned;+ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_state, L->bytes_per_thread * n_threads);+ L->total_bytes = off;}struct htp_gdn_hmx_vtcm_layout {diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.c b/ggml/src/ggml-hexagon/htp/gather-rows.cindex 1db48e59cd..1e88445f1c 100644--- a/ggml/src/ggml-hexagon/htp/gather-rows.c+++ b/ggml/src/ggml-hexagon/htp/gather-rows.c@@ -206,14 +206,22 @@ static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * dastruct htp_thread_trace * tr = &task->octx->ctx->trace[ith];- uint8_t * base = task->octx->ctx->vtcm_base + ith * kparams->slice;- HVX_Vector * tab = (HVX_Vector *) base;- HVX_Vector * inc = tab + kparams->vg;- HVX_Vector * cur = inc + kparams->vg;- uint8_t * region_base = (uint8_t *) (cur + kparams->vg);-- uint8_t * region[2] = { region_base, region_base + kparams->buf_stride };- uint8_t * out[2] = { region[0] + kparams->out_off, region[1] + kparams->out_off };+ struct htp_gather_rows_vtcm_layout layout;+ htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 4);++ uint8_t * base = VTCM_LAYOUT_PTR(uint8_t, task->octx->ctx->vtcm_base, (size_t) ith * kparams->slice);+ HVX_Vector * tab = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_tab);+ HVX_Vector * inc = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_inc);+ HVX_Vector * cur = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_cur);++ uint8_t * region[2] = {+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region0),+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region1),+ };+ uint8_t * out[2] = {+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out0),+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out1),+ };htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);gather_rows_init_tables_f32(tab, inc, kparams, a, b);@@ -279,14 +287,22 @@ static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * dastruct htp_thread_trace * tr = &task->octx->ctx->trace[ith];- uint8_t * base = task->octx->ctx->vtcm_base + ith * kparams->slice;- HVX_Vector * tab = (HVX_Vector *) base;- HVX_Vector * inc = tab + kparams->vg;- HVX_Vector * cur = inc + kparams->vg;- uint8_t * region_base = (uint8_t *) (cur + kparams->vg);-- uint8_t * region[2] = { region_base, region_base + kparams->buf_stride };- uint8_t * out[2] = { region[0] + kparams->out_off, region[1] + kparams->out_off };+ struct htp_gather_rows_vtcm_layout layout;+ htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 2);++ uint8_t * base = VTCM_LAYOUT_PTR(uint8_t, task->octx->ctx->vtcm_base, (size_t) ith * kparams->slice);+ HVX_Vector * tab = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_tab);+ HVX_Vector * inc = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_inc);+ HVX_Vector * cur = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_cur);++ uint8_t * region[2] = {+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region0),+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region1),+ };+ uint8_t * out[2] = {+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out0),+ VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out1),+ };htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);gather_rows_init_tables_f16(tab, inc, kparams, a, b);diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.h b/ggml/src/ggml-hexagon/htp/gather-rows.hindex 3bbb1c6357..c08ef4d38f 100644--- a/ggml/src/ggml-hexagon/htp/gather-rows.h+++ b/ggml/src/ggml-hexagon/htp/gather-rows.h@@ -2,6 +2,7 @@#define HTP_GATHER_ROWS_H#include "hex-common.h"+#include "htp-vtcm.h"#include <stdint.h>#include <stdbool.h>@@ -19,8 +20,6 @@ struct htp_gather_rows_params {uint32_t v1;uint32_t span1;uint32_t region_size;- uint32_t out_off;- uint32_t buf_stride;uint32_t slice;uint32_t total_rows;uint32_t rows_per_thread;@@ -28,6 +27,40 @@ struct htp_gather_rows_params {uint8_t pad[3];};+struct htp_gather_rows_vtcm_layout {+ size_t off_tab;+ size_t off_inc;+ size_t off_cur;+ size_t off_region0;+ size_t off_out0;+ size_t off_region1;+ size_t off_out1;+ size_t total_per_thread;+};++static inline void htp_gather_rows_vtcm_layout_build(+ struct htp_gather_rows_vtcm_layout * L,+ uint32_t vg,+ uint32_t region_size,+ uint32_t nr_max,+ uint32_t ne,+ uint32_t elem_size) {++ const size_t vec_bytes = (size_t) vg * 128;+ const size_t region_aligned = hex_round_up(region_size, 128);+ const size_t out_aligned = hex_round_up(region_aligned + (size_t) nr_max * ne * elem_size, 128) - region_aligned;++ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_tab, vec_bytes);+ VTCM_LAYOUT_ALLOC(off, off_inc, vec_bytes);+ VTCM_LAYOUT_ALLOC(off, off_cur, vec_bytes);+ VTCM_LAYOUT_ALLOC(off, off_region0, region_aligned);+ VTCM_LAYOUT_ALLOC(off, off_out0, out_aligned);+ VTCM_LAYOUT_ALLOC(off, off_region1, region_aligned);+ VTCM_LAYOUT_ALLOC(off, off_out1, out_aligned);+ L->total_per_thread = off;+}+static inline bool htp_gather_rows_solve_layout(struct htp_gather_rows_params * kparams,uint32_t na, uint32_t a_nb1,@@ -80,8 +113,6 @@ static inline bool htp_gather_rows_solve_layout(uint32_t v1 = 0;uint32_t span1 = 0;uint32_t region_size = 0;- uint32_t out_off = 0;- uint32_t buf_stride = 0;while (nr_max > 0) {v1 = hex_round_up((nr_max - 1) * a_nb1 + na * elem_size, 128);@@ -98,10 +129,10 @@ static inline bool htp_gather_rows_solve_layout(region_size = v1;}- out_off = hex_round_up(region_size, 128);- buf_stride = hex_round_up(out_off + nr_max * ne * elem_size, 128);+ struct htp_gather_rows_vtcm_layout layout;+ htp_gather_rows_vtcm_layout_build(&layout, vg, region_size, nr_max, ne, elem_size);- if ((elem_size != 2 || region_size <= 32768) && fixed + 2 * buf_stride <= slice) {+ if ((elem_size != 2 || region_size <= 32768) && layout.total_per_thread <= slice) {break;}@@ -121,8 +152,6 @@ static inline bool htp_gather_rows_solve_layout(kparams->v1 = v1;kparams->span1 = span1;kparams->region_size = region_size;- kparams->out_off = out_off;- kparams->buf_stride = buf_stride;kparams->slice = slice;kparams->total_rows = total_rows;kparams->rows_per_thread = rows_per_thread;diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.cindex 78eb1bb500..39acdfda66 100644--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c@@ -404,7 +404,7 @@ int op_get_rows(struct htp_ops_context * octx) {if (octx->ctx->mdev.count > 1) {uint32_t tasks_per_chunk = 1;- htp_tensor_mdev_rows_per_chunk(dst, dst_row_size / dst->ne[0], dst_row_size, &tasks_per_chunk);+ htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) dst->nb[0], dst_row_size, &tasks_per_chunk);const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, tasks_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);task_start = range.start;tasks = range.count;@@ -414,7 +414,7 @@ int op_get_rows(struct htp_ops_context * octx) {return HTP_STATUS_OK;}- if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) {+ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {return HTP_STATUS_INVAL_PARAMS;}@@ -428,7 +428,7 @@ int op_get_rows(struct htp_ops_context * octx) {grctx.compute_tiled = NULL;grctx.task_start = task_start;grctx.tasks = tasks;- grctx.tasks_per_thread = octx->ctx->mdev.count == 1 ? kparams->tasks_per_thread : fastdiv(tasks + n_threads - 1, &octx->n_threads_div);+ grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div);grctx.tile_size = 0;grctx.tile_stride = 0;grctx.src0_row_size = src0_row_size;diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.hindex 94f3c15f29..4634be2e2e 100644--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h@@ -15,11 +15,10 @@ enum htp_get_rows_kernel_type {};struct htp_get_rows_kernel_params {- int32_t n_threads;- int32_t kernel_type;- int32_t total_tasks;- int32_t tasks_per_thread;- int32_t vtcm_size;+ uint8_t kernel_type;+ uint32_t n_threads;+ uint32_t total_tasks;+ uint32_t vtcm_size;// Fastdiv helpersstruct fastdiv_values div_ne10;diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/glu-ops.csimilarity index 69%rename from ggml/src/ggml-hexagon/htp/act-ops.crename to ggml/src/ggml-hexagon/htp/glu-ops.cindex 59a0575c4b..b4edddf3fe 100644--- a/ggml/src/ggml-hexagon/htp/act-ops.c+++ b/ggml/src/ggml-hexagon/htp/glu-ops.c@@ -17,11 +17,12 @@#include "hex-common.h"#include "htp-tensor.h"#include "htp-vtcm.h"+#include "glu-ops.h"-#define htp_act_preamble \- const struct htp_tensor * src0 = actx->octx->src[0]; \- const struct htp_tensor * src1 = actx->octx->src[1]; \- const struct htp_tensor * dst = actx->octx->dst; \+#define htp_glu_preamble \+ const struct htp_tensor * src0 = gctx->octx->src[0]; \+ const struct htp_tensor * src1 = gctx->octx->src[1]; \+ const struct htp_tensor * dst = gctx->octx->dst; \\const uint32_t ne00 = src0->ne[0]; \const uint32_t ne01 = src0->ne[1]; \@@ -53,21 +54,21 @@const uint32_t nb2 = dst->nb[2]; \const uint32_t nb3 = dst->nb[3];-struct htp_act_context;+struct htp_glu_context;typedef void (*glu_compute_f32_fn_t)(const float * restrict src0,const float * restrict src1,float * restrict dst,const uint32_t num_rows,- const struct htp_act_context * actx);+ const struct htp_glu_context * gctx);typedef void (*glu_compute_f16_fn_t)(const _Float16 * restrict src0,const _Float16 * restrict src1,_Float16 * restrict dst,const uint32_t num_rows,- const struct htp_act_context * actx);+ const struct htp_glu_context * gctx);-struct htp_act_context {+struct htp_glu_context {struct htp_ops_context * octx;void * compute;@@ -108,53 +109,18 @@ struct htp_act_context {size_t vtcm_dst_size_per_thread;};-struct htp_act_vtcm_layout {- size_t total_bytes;- size_t off_src0;- size_t off_src1;- size_t off_dst;-- size_t src0_bytes_per_thread;- size_t src1_bytes_per_thread;- size_t dst_bytes_per_thread;-- uint32_t vtcm_row_per_thread;-};--static inline void htp_act_vtcm_layout_build(struct htp_act_vtcm_layout * L,- size_t src0_row_size_aligned,- size_t src1_row_size_aligned,- size_t dst_row_size_aligned,- uint32_t n_threads,- size_t vtcm_size) {- const size_t spad_size_per_row = src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned;- const uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * spad_size_per_row));-- L->vtcm_row_per_thread = vtcm_row_per_thread;-- L->src0_bytes_per_thread = src0_row_size_aligned * vtcm_row_per_thread;- L->src1_bytes_per_thread = src1_row_size_aligned * vtcm_row_per_thread;- L->dst_bytes_per_thread = dst_row_size_aligned * vtcm_row_per_thread;-- L->off_src0 = 0;- L->off_src1 = L->off_src0 + L->src0_bytes_per_thread * n_threads;- L->off_dst = L->off_src1 + L->src1_bytes_per_thread * n_threads;-- L->total_bytes = L->off_dst + L->dst_bytes_per_thread * n_threads;-}-#define htp_glu_op_preamble \- const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \- const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \- const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \- const int nc = actx->nc;+ const size_t src0_row_size_aligned = gctx->src0_row_size_aligned; \+ const size_t src1_row_size_aligned = gctx->src1_row_size_aligned; \+ const size_t dst_row_size_aligned = gctx->dst_row_size_aligned; \+ const int nc = gctx->nc;// swiglu(x) = x1 * sigmoid(x0)static void swiglu_f32(const float * restrict src0,const float * restrict src1,float * restrict dst,const uint32_t num_rows,- const struct htp_act_context * actx) {+ const struct htp_glu_context * gctx) {htp_glu_op_preamble;for (uint32_t ib = 0; ib < num_rows; ib++) {@@ -172,10 +138,10 @@ static void swiglu_oai_f32(const float * restrict src0,const float * restrict src1,float * restrict dst,const uint32_t num_rows,- const struct htp_act_context * actx) {+ const struct htp_glu_context * gctx) {htp_glu_op_preamble;- const float alpha = ((const float *) (actx->octx->op_params))[2];- const float limit = ((const float *) (actx->octx->op_params))[3];+ const float alpha = ((const float *) (gctx->octx->op_params))[2];+ const float limit = ((const float *) (gctx->octx->op_params))[3];for (uint32_t ib = 0; ib < num_rows; ib++) {const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);@@ -201,9 +167,9 @@ static void swiglu_clamp_f32(const float * restrict src0,const float * restrict src1,float * restrict dst,const uint32_t num_rows,- const struct htp_act_context * actx) {+ const struct htp_glu_context * gctx) {htp_glu_op_preamble;- const float limit = ((const float *) (actx->octx->op_params))[3];+ const float limit = ((const float *) (gctx->octx->op_params))[3];for (uint32_t ib = 0; ib < num_rows; ib++) {const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);@@ -455,7 +421,7 @@ static void geglu_f32(const float * restrict src0,const float * restrict src1,float * restrict dst,const uint32_t num_rows,- const struct htp_act_context * actx) {+ const struct htp_glu_context * gctx) {htp_glu_op_preamble;for (uint32_t ib = 0; ib < num_rows; ib++) {@@ -472,7 +438,7 @@ static void geglu_quick_f32(const float * restrict src0,const float * restrict src1,float * restrict dst,const uint32_t num_rows,- const struct htp_act_context * actx) {+ const struct htp_glu_context * gctx) {htp_glu_op_preamble;for (uint32_t ib = 0; ib < num_rows; ib++) {@@ -489,7 +455,7 @@ static void geglu_erf_f32(const float * restrict src0,const float * restrict src1,float * restrict dst,const uint32_t num_rows,- const struct htp_act_context * actx) {+ const struct htp_glu_context * gctx) {htp_glu_op_preamble;for (uint32_t ib = 0; ib < num_rows; ib++) {@@ -501,12 +467,124 @@ static void geglu_erf_f32(const float * restrict src0,}}+static inline void hvx_reglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {+ assert((unsigned long) dst % 128 == 0);+ assert((unsigned long) src0 % 128 == 0);+ assert((unsigned long) src1 % 128 == 0);++ HVX_Vector * restrict vdst = (HVX_Vector *) dst;+ const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0;+ const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1;++ const uint32_t epv = 128 / sizeof(float);+ const uint32_t nvec = n / epv;+ const uint32_t nloe = n % epv;++ const HVX_Vector v_zero = Q6_V_vzero();++ uint32_t i = 0;++ _Pragma("unroll(4)")+ for (; i < nvec; i++) {+ HVX_Vector x = Q6_Vsf_vmax_VsfVsf(vsrc0[i], v_zero);+ vdst[i] = hvx_vec_mul_f32_f32(x, vsrc1[i]);+ }++ if (nloe) {+ HVX_Vector x = Q6_Vsf_vmax_VsfVsf(vsrc0[i], v_zero);+ HVX_Vector res = hvx_vec_mul_f32_f32(x, vsrc1[i]);+ hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(float), res);+ }+}++// reglu(x, g) = relu(x) * g+static void reglu_f32(const float * restrict src0,+ const float * restrict src1,+ float * restrict dst,+ const uint32_t num_rows,+ const struct htp_glu_context * gctx) {+ htp_glu_op_preamble;++ for (uint32_t ib = 0; ib < num_rows; ib++) {+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);++ hvx_reglu_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc);+ }+}++static __attribute__((noinline)) void hvx_sigglu_f32_vec(HVX_Vector * restrict vdst,+ const HVX_Vector * restrict vsrc0,+ const HVX_Vector * restrict vsrc1,+ uint32_t nvec) {+ const HVX_Vector v_one = hvx_vec_splat_f32(1.0f);+ const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f);+ const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f);++ for (uint32_t i = 0; i < nvec; i++) {+ HVX_Vector sig = hvx_vec_fast_sigmoid_f32_guard(vsrc0[i], v_one, v_max_exp, v_min_exp);+ vdst[i] = hvx_vec_mul_f32_f32(sig, vsrc1[i]);+ }+}++static __attribute__((noinline)) void hvx_sigglu_f32_tail(HVX_Vector * restrict vdst,+ const HVX_Vector * restrict vsrc0,+ const HVX_Vector * restrict vsrc1,+ uint32_t nloe) {+ const HVX_Vector v_one = hvx_vec_splat_f32(1.0f);+ const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f);+ const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f);++ HVX_Vector sig = hvx_vec_fast_sigmoid_f32_guard(vsrc0[0], v_one, v_max_exp, v_min_exp);+ HVX_Vector res = hvx_vec_mul_f32_f32(sig, vsrc1[0]);+ hvx_vec_store_a((void *) vdst, nloe * sizeof(float), res);+}++static inline void hvx_sigglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {+ assert((unsigned long) dst % 128 == 0);+ assert((unsigned long) src0 % 128 == 0);+ assert((unsigned long) src1 % 128 == 0);++ HVX_Vector * restrict vdst = (HVX_Vector *) dst;+ const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0;+ const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1;++ const uint32_t epv = 128 / sizeof(float);+ const uint32_t nvec = n / epv;+ const uint32_t nloe = n % epv;++ if (nvec) {+ hvx_sigglu_f32_vec(vdst, vsrc0, vsrc1, nvec);+ }+ if (nloe) {+ hvx_sigglu_f32_tail(vdst + nvec, vsrc0 + nvec, vsrc1 + nvec, nloe);+ }+}++// sigglu(x, g) = sigmoid(x) * g+static void sigglu_f32(const float * restrict src0,+ const float * restrict src1,+ float * restrict dst,+ const uint32_t num_rows,+ const struct htp_glu_context * gctx) {+ htp_glu_op_preamble;++ for (uint32_t ib = 0; ib < num_rows; ib++) {+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);++ hvx_sigglu_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc);+ }+}+// swiglu(x) = x1 * sigmoid(x0)static void swiglu_f16(const _Float16 * restrict src0,const _Float16 * restrict src1,_Float16 * restrict dst,const uint32_t num_rows,- const struct htp_act_context * actx) {+ const struct htp_glu_context * gctx) {htp_glu_op_preamble;for (uint32_t ib = 0; ib < num_rows; ib++) {@@ -524,10 +602,10 @@ static void swiglu_oai_f16(const _Float16 * restrict src0,const _Float16 * restrict src1,_Float16 * restrict dst,const uint32_t num_rows,- const struct htp_act_context * actx) {+ const struct htp_glu_context * gctx) {htp_glu_op_preamble;- const _Float16 alpha = (_Float16) ((const float *) (actx->octx->op_params))[2];- const _Float16 limit = (_Float16) ((const float *) (actx->octx->op_params))[3];+ const _Float16 alpha = (_Float16) ((const float *) (gctx->octx->op_params))[2];+ const _Float16 limit = (_Float16) ((const float *) (gctx->octx->op_params))[3];for (uint32_t ib = 0; ib < num_rows; ib++) {const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);@@ -553,9 +631,9 @@ static void swiglu_clamp_f16(const _Float16 * restrict src0,const _Float16 * restrict src1,_Float16 * restrict dst,const uint32_t num_rows,- const struct htp_act_context * actx) {+ const struct htp_glu_context * gctx) {htp_glu_op_preamble;- const _Float16 limit = (_Float16) ((const float *) (actx->octx->op_params))[3];+ const _Float16 limit = (_Float16) ((const float *) (gctx->octx->op_params))[3];for (uint32_t ib = 0; ib < num_rows; ib++) {const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);@@ -574,7 +652,7 @@ static void geglu_f16(const _Float16 * restrict src0,const _Float16 * restrict src1,_Float16 * restrict dst,const uint32_t num_rows,- const struct htp_act_context * actx) {+ const struct htp_glu_context * gctx) {htp_glu_op_preamble;for (uint32_t ib = 0; ib < num_rows; ib++) {@@ -586,56 +664,155 @@ static void geglu_f16(const _Float16 * restrict src0,}}+static inline void hvx_reglu_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {+ assert((unsigned long) dst % 128 == 0);+ assert((unsigned long) src0 % 128 == 0);+ assert((unsigned long) src1 % 128 == 0);++ HVX_Vector * restrict vdst = (HVX_Vector *) dst;+ const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0;+ const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1;++ const uint32_t epv = 128 / sizeof(_Float16);+ const uint32_t nvec = n / epv;+ const uint32_t nloe = n % epv;++ const HVX_Vector v_zero = Q6_V_vzero();++ uint32_t i = 0;++ _Pragma("unroll(4)")+ for (; i < nvec; i++) {+ HVX_Vector x = Q6_Vhf_vmax_VhfVhf(vsrc0[i], v_zero);+ vdst[i] = hvx_vec_mul_f16_f16(x, vsrc1[i]);+ }++ if (nloe) {+ HVX_Vector x = Q6_Vhf_vmax_VhfVhf(vsrc0[i], v_zero);+ HVX_Vector res = hvx_vec_mul_f16_f16(x, vsrc1[i]);+ hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(_Float16), res);+ }+}++// reglu(x, g) = relu(x) * g+static void reglu_f16(const _Float16 * restrict src0,+ const _Float16 * restrict src1,+ _Float16 * restrict dst,+ const uint32_t num_rows,+ const struct htp_glu_context * gctx) {+ htp_glu_op_preamble;++ for (uint32_t ib = 0; ib < num_rows; ib++) {+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);++ hvx_reglu_f16_aa(dst_ptr, src0_ptr, src1_ptr, nc);+ }+}++static __attribute__((noinline)) void hvx_sigglu_f16_vec(HVX_Vector * restrict vdst,+ const HVX_Vector * restrict vsrc0,+ const HVX_Vector * restrict vsrc1,+ uint32_t nvec) {+ for (uint32_t i = 0; i < nvec; i++) {+ HVX_Vector sig = hvx_vec_fast_sigmoid_f16(vsrc0[i]);+ vdst[i] = hvx_vec_mul_f16_f16(sig, vsrc1[i]);+ }+}++static __attribute__((noinline)) void hvx_sigglu_f16_tail(HVX_Vector * restrict vdst,+ const HVX_Vector * restrict vsrc0,+ const HVX_Vector * restrict vsrc1,+ uint32_t nloe) {+ HVX_Vector sig = hvx_vec_fast_sigmoid_f16(vsrc0[0]);+ HVX_Vector res = hvx_vec_mul_f16_f16(sig, vsrc1[0]);+ hvx_vec_store_a((void *) vdst, nloe * sizeof(_Float16), res);+}++static inline void hvx_sigglu_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {+ assert((unsigned long) dst % 128 == 0);+ assert((unsigned long) src0 % 128 == 0);+ assert((unsigned long) src1 % 128 == 0);++ HVX_Vector * restrict vdst = (HVX_Vector *) dst;+ const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0;+ const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1;++ const uint32_t epv = 128 / sizeof(_Float16);+ const uint32_t nvec = n / epv;+ const uint32_t nloe = n % epv;++ if (nvec) {+ hvx_sigglu_f16_vec(vdst, vsrc0, vsrc1, nvec);+ }+ if (nloe) {+ hvx_sigglu_f16_tail(vdst + nvec, vsrc0 + nvec, vsrc1 + nvec, nloe);+ }+}++// sigglu(x, g) = sigmoid(x) * g+static void sigglu_f16(const _Float16 * restrict src0,+ const _Float16 * restrict src1,+ _Float16 * restrict dst,+ const uint32_t num_rows,+ const struct htp_glu_context * gctx) {+ htp_glu_op_preamble;++ for (uint32_t ib = 0; ib < num_rows; ib++) {+ const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);+ const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);+ uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);++ hvx_sigglu_f16_aa(dst_ptr, src0_ptr, src1_ptr, nc);+ }+}+#define DEFINE_GLU_PER_THREAD(TYPE, SUFFIX) \static void glu_##SUFFIX##_per_thread(unsigned int nth, unsigned int ith, void * data) { \- struct htp_act_context * actx = (struct htp_act_context *) data; \- htp_act_preamble; \+ struct htp_glu_context * gctx = (struct htp_glu_context *) data; \+ htp_glu_preamble; \\- struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \+ struct htp_thread_trace * tr = gctx->octx->ctx ? &gctx->octx->ctx->trace[ith] : NULL; \\- size_t src0_row_size = actx->src0_row_size; \- size_t src1_row_size = actx->src1_row_size; \- size_t dst_row_size = actx->dst_row_size; \+ size_t src0_row_size = gctx->src0_row_size; \+ size_t src1_row_size = gctx->src1_row_size; \+ size_t dst_row_size = gctx->dst_row_size; \\- size_t src0_row_stride = actx->src0_row_stride; \- size_t src1_row_stride = actx->src1_row_stride; \+ size_t src0_row_stride = gctx->src0_row_stride; \+ size_t src1_row_stride = gctx->src1_row_stride; \\- const uint32_t src0_nrows = actx->src0_nrows; \- const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \+ const uint32_t src0_nrows = gctx->src0_nrows; \+ const uint32_t src0_nrows_per_thread = gctx->src0_nrows_per_thread; \\- const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith; \- const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows); \+ const uint32_t src0_start_row = gctx->row_start + src0_nrows_per_thread * ith; \+ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, gctx->row_start + src0_nrows); \\/* no work for this thread */ \if (src0_start_row >= src0_end_row) { \return; \} \\- const dma_addr_t data_src0 = actx->data_src0; \- const dma_addr_t data_src1 = actx->data_src1; \- const dma_addr_t data_dst = actx->data_dst; \+ const dma_addr_t data_src0 = gctx->data_src0; \+ const dma_addr_t data_src1 = gctx->data_src1; \+ const dma_addr_t data_dst = gctx->data_dst; \\- const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \- const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \- const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \+ const size_t src0_row_size_aligned = gctx->src0_row_size_aligned; \+ const size_t src1_row_size_aligned = gctx->src1_row_size_aligned; \+ const size_t dst_row_size_aligned = gctx->dst_row_size_aligned; \\- uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \- uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \- uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \+ uint8_t * restrict src0_spad_data = gctx->vtcm_src0 + (ith * gctx->vtcm_src0_size_per_thread); \+ uint8_t * restrict src1_spad_data = gctx->vtcm_src1 + (ith * gctx->vtcm_src1_size_per_thread); \+ uint8_t * restrict dst_spad_data = gctx->vtcm_dst + (ith * gctx->vtcm_dst_size_per_thread); \\- size_t src0_spad_half_size = actx->src0_spad_half_size; \- size_t src1_spad_half_size = actx->src1_spad_half_size; \- size_t dst_spad_half_size = actx->dst_spad_half_size; \+ size_t src0_spad_half_size = gctx->src0_spad_half_size; \+ size_t src1_spad_half_size = gctx->src1_spad_half_size; \+ size_t dst_spad_half_size = gctx->dst_spad_half_size; \\- const int BLOCK = actx->block; \- if (BLOCK == 0) { \- FARF(ERROR, "%s : VTCM reservation %zu is too small, needed %zu\n", \- actx->op_str, actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \- return; \- } \+ const int BLOCK = gctx->block; \\- dma_queue * dma_q = actx->octx->ctx->dma[ith]; \- glu_compute_##SUFFIX##_fn_t compute = (glu_compute_##SUFFIX##_fn_t) actx->compute; \+ dma_queue * dma_q = gctx->octx->ctx->dma[ith]; \+ glu_compute_##SUFFIX##_fn_t compute = (glu_compute_##SUFFIX##_fn_t) gctx->compute; \\for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \@@ -662,7 +839,7 @@ static void geglu_f16(const _Float16 * restrict src0,TYPE * src1_spad = (TYPE *) dma_queue_pop(dma_q).dst; \\htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \- compute(src0_spad, src1_spad, dst_spad, block_size, actx); \+ compute(src0_spad, src1_spad, dst_spad, block_size, gctx); \htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \\dma_queue_push(dma_q, dma_make_data(data_dst + (ir * dst_row_size), dst_spad), \@@ -685,16 +862,11 @@ static void geglu_f16(const _Float16 * restrict src0,DEFINE_GLU_PER_THREAD(float, f32)DEFINE_GLU_PER_THREAD(_Float16, f16)-static int execute_op_activations_f32(struct htp_ops_context * octx) {+static int execute_op_glu_f32(struct htp_ops_context * octx) {const struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * src1 = octx->src[1];const struct htp_tensor * dst = octx->dst;- if ((dst->ne[0] * SIZEOF_FP32) != dst->nb[1]) {- FARF(ERROR, "Non-contiguous dst is not supported at this time \n");- return HTP_STATUS_NO_SUPPORT;- }-glu_compute_f32_fn_t compute_fn = NULL;const char * op_type = NULL;@@ -729,11 +901,27 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {op_type = "geglu-erf-f32";break;+ case HTP_OP_GLU_REGLU:+ compute_fn = reglu_f32;+ op_type = "reglu-f32";+ break;++ case HTP_OP_GLU_SIGMOID:+ compute_fn = sigglu_f32;+ op_type = "sigglu-f32";+ break;+default:- FARF(ERROR, "Unsupported activations Op %u\n", octx->op);+ FARF(ERROR, "Unsupported GLU Op %u\n", octx->op);return HTP_STATUS_NO_SUPPORT;}+ const struct htp_glu_kernel_params * kparams = (const struct htp_glu_kernel_params *) octx->kernel_params;++ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {+ return HTP_STATUS_INVAL_PARAMS;+ }+const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];const size_t dst_row_size = dst->ne[0] * SIZEOF_FP32;@@ -762,17 +950,16 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {const size_t src0_row_stride = src0->nb[1];const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1];- const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);- const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);- const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);+ struct htp_glu_vtcm_layout L;+ htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, kparams->vtcm_row_per_thread);- struct htp_act_vtcm_layout L;- htp_act_vtcm_layout_build(&L, src0_row_size_aligned, src1_row_size_aligned, dst_row_size_aligned, n_threads,- octx->ctx->vtcm_size);+ const size_t src0_row_size_aligned = L.row_size_aligned;+ const size_t src1_row_size_aligned = L.row_size_aligned;+ const size_t dst_row_size_aligned = L.row_size_aligned;// Make sure the reserved vtcm size is sufficient- if (L.vtcm_row_per_thread == 0) {- FARF(ERROR, "act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,+ if (kparams->vtcm_row_per_thread == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) {+ FARF(ERROR, "glu-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,(src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads);return HTP_STATUS_VTCM_TOO_SMALL;}@@ -789,58 +976,53 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {}// Prepare context- struct htp_act_context actx;- actx.octx = octx;- actx.compute = (void *) compute_fn;- actx.op_str = op_type;+ struct htp_glu_context gctx;+ gctx.octx = octx;+ gctx.compute = (void *) compute_fn;+ gctx.op_str = op_type;- actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);+ gctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);- actx.src0_row_size = src0_row_size;- actx.src1_row_size = src1_row_size;- actx.dst_row_size = dst_row_size;+ gctx.src0_row_size = src0_row_size;+ gctx.src1_row_size = src1_row_size;+ gctx.dst_row_size = dst_row_size;- actx.src0_row_size_aligned = src0_row_size_aligned;- actx.src1_row_size_aligned = src1_row_size_aligned;- actx.dst_row_size_aligned = dst_row_size_aligned;+ gctx.src0_row_size_aligned = src0_row_size_aligned;+ gctx.src1_row_size_aligned = src1_row_size_aligned;+ gctx.dst_row_size_aligned = dst_row_size_aligned;- actx.src0_row_stride = src0_row_stride;- actx.src1_row_stride = src1_row_stride;+ gctx.src0_row_stride = src0_row_stride;+ gctx.src1_row_stride = src1_row_stride;uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;- actx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);- actx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);- actx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);+ gctx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);+ gctx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);+ gctx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);- actx.vtcm_src0_size_per_thread = L.src0_bytes_per_thread;- actx.vtcm_src1_size_per_thread = L.src1_bytes_per_thread;- actx.vtcm_dst_size_per_thread = L.dst_bytes_per_thread;+ gctx.vtcm_src0_size_per_thread = (uint32_t) L.src0_bytes_per_thread;+ gctx.vtcm_src1_size_per_thread = (uint32_t) L.src1_bytes_per_thread;+ gctx.vtcm_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread;- actx.src0_spad_half_size = L.src0_bytes_per_thread / 2;- actx.src1_spad_half_size = L.src1_bytes_per_thread / 2;- actx.dst_spad_half_size = L.dst_bytes_per_thread / 2;+ gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_thread >> 1;+ gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_thread >> 1;+ gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_thread >> 1;- actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned;- actx.src0_nrows = nrows;- actx.row_start = row_start;+ gctx.block = L.block;+ gctx.src0_nrows = nrows;+ gctx.row_start = row_start;- actx.nc = dst->ne[0];+ gctx.nc = dst->ne[0];// Addresses and GLU logicdma_addr_t data_src0 = src0->data;dma_addr_t data_src1 = src1 ? src1->data : 0;- if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU ||- octx->op == HTP_OP_GLU_SWIGLU_OAI ||- octx->op == HTP_OP_GLU_SWIGLU_CLAMP ||- octx->op == HTP_OP_GLU_GEGLU ||- octx->op == HTP_OP_GLU_GEGLU_QUICK ||- octx->op == HTP_OP_GLU_GEGLU_ERF)) {+ if (!src1) {const int32_t swapped = octx->op_params[1];data_src1 = data_src0;- actx.src1_row_size = actx.src0_row_size;+ gctx.src1_row_size = gctx.src0_row_size;- size_t nc_in_bytes = actx.nc * SIZEOF_FP32;+ size_t nc_in_bytes = gctx.nc * SIZEOF_FP32;if (swapped) {data_src0 += nc_in_bytes;} else {@@ -848,24 +1030,19 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {}}- actx.data_src0 = data_src0;- actx.data_src1 = data_src1;- actx.data_dst = dst->data;+ gctx.data_src0 = data_src0;+ gctx.data_src1 = data_src1;+ gctx.data_dst = dst->data;- work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f32_per_thread, &actx, n_threads);+ work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f32_per_thread, &gctx, n_threads);return HTP_STATUS_OK;}-static int execute_op_activations_f16(struct htp_ops_context * octx) {+static int execute_op_glu_f16(struct htp_ops_context * octx) {const struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * src1 = octx->src[1];const struct htp_tensor * dst = octx->dst;- if ((dst->ne[0] * SIZEOF_FP16) != dst->nb[1]) {- FARF(ERROR, "Non-contiguous dst is not supported at this time \n");- return HTP_STATUS_NO_SUPPORT;- }-glu_compute_f16_fn_t compute_fn = NULL;const char * op_type = NULL;@@ -890,11 +1067,27 @@ static int execute_op_activations_f16(struct htp_ops_context * octx) {op_type = "geglu-f16";break;+ case HTP_OP_GLU_REGLU:+ compute_fn = reglu_f16;+ op_type = "reglu-f16";+ break;++ case HTP_OP_GLU_SIGMOID:+ compute_fn = sigglu_f16;+ op_type = "sigglu-f16";+ break;+default:- FARF(ERROR, "Unsupported activations Op %u\n", octx->op);+ FARF(ERROR, "Unsupported GLU Op %u\n", octx->op);return HTP_STATUS_NO_SUPPORT;}+ const struct htp_glu_kernel_params * kparams = (const struct htp_glu_kernel_params *) octx->kernel_params;++ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {+ return HTP_STATUS_INVAL_PARAMS;+ }+const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];const size_t dst_row_size = dst->ne[0] * SIZEOF_FP16;@@ -923,17 +1116,16 @@ static int execute_op_activations_f16(struct htp_ops_context * octx) {const size_t src0_row_stride = src0->nb[1];const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1];- const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);- const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);- const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);+ struct htp_glu_vtcm_layout L;+ htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, kparams->vtcm_row_per_thread);- struct htp_act_vtcm_layout L;- htp_act_vtcm_layout_build(&L, src0_row_size_aligned, src1_row_size_aligned, dst_row_size_aligned, n_threads,- octx->ctx->vtcm_size);+ const size_t src0_row_size_aligned = L.row_size_aligned;+ const size_t src1_row_size_aligned = L.row_size_aligned;+ const size_t dst_row_size_aligned = L.row_size_aligned;// Make sure the reserved vtcm size is sufficient- if (L.vtcm_row_per_thread == 0) {- FARF(ERROR, "act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,+ if (kparams->vtcm_row_per_thread == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) {+ FARF(ERROR, "glu-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,(src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads);return HTP_STATUS_VTCM_TOO_SMALL;}@@ -950,56 +1142,53 @@ static int execute_op_activations_f16(struct htp_ops_context * octx) {}// Prepare context- struct htp_act_context actx;- actx.octx = octx;- actx.compute = (void *) compute_fn;- actx.op_str = op_type;+ struct htp_glu_context gctx;+ gctx.octx = octx;+ gctx.compute = (void *) compute_fn;+ gctx.op_str = op_type;- actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);+ gctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);- actx.src0_row_size = src0_row_size;- actx.src1_row_size = src1_row_size;- actx.dst_row_size = dst_row_size;+ gctx.src0_row_size = src0_row_size;+ gctx.src1_row_size = src1_row_size;+ gctx.dst_row_size = dst_row_size;- actx.src0_row_size_aligned = src0_row_size_aligned;- actx.src1_row_size_aligned = src1_row_size_aligned;- actx.dst_row_size_aligned = dst_row_size_aligned;+ gctx.src0_row_size_aligned = src0_row_size_aligned;+ gctx.src1_row_size_aligned = src1_row_size_aligned;+ gctx.dst_row_size_aligned = dst_row_size_aligned;- actx.src0_row_stride = src0_row_stride;- actx.src1_row_stride = src1_row_stride;+ gctx.src0_row_stride = src0_row_stride;+ gctx.src1_row_stride = src1_row_stride;uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;- actx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);- actx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);- actx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);+ gctx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);+ gctx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);+ gctx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);- actx.vtcm_src0_size_per_thread = L.src0_bytes_per_thread;- actx.vtcm_src1_size_per_thread = L.src1_bytes_per_thread;- actx.vtcm_dst_size_per_thread = L.dst_bytes_per_thread;+ gctx.vtcm_src0_size_per_thread = (uint32_t) L.src0_bytes_per_thread;+ gctx.vtcm_src1_size_per_thread = (uint32_t) L.src1_bytes_per_thread;+ gctx.vtcm_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread;- actx.src0_spad_half_size = L.src0_bytes_per_thread / 2;- actx.src1_spad_half_size = L.src1_bytes_per_thread / 2;- actx.dst_spad_half_size = L.dst_bytes_per_thread / 2;+ gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_thread >> 1;+ gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_thread >> 1;+ gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_thread >> 1;- actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned;- actx.src0_nrows = nrows;- actx.row_start = row_start;+ gctx.block = L.block;+ gctx.src0_nrows = nrows;+ gctx.row_start = row_start;- actx.nc = dst->ne[0];+ gctx.nc = dst->ne[0];// Addresses and GLU logicdma_addr_t data_src0 = src0->data;dma_addr_t data_src1 = src1 ? src1->data : 0;- if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU ||- octx->op == HTP_OP_GLU_SWIGLU_OAI ||- octx->op == HTP_OP_GLU_SWIGLU_CLAMP ||- octx->op == HTP_OP_GLU_GEGLU)) {+ if (!src1) {const int32_t swapped = octx->op_params[1];data_src1 = data_src0;- actx.src1_row_size = actx.src0_row_size;+ gctx.src1_row_size = gctx.src0_row_size;- size_t nc_in_bytes = actx.nc * SIZEOF_FP16;+ size_t nc_in_bytes = gctx.nc * SIZEOF_FP16;if (swapped) {data_src0 += nc_in_bytes;} else {@@ -1007,22 +1196,22 @@ static int execute_op_activations_f16(struct htp_ops_context * octx) {}}- actx.data_src0 = data_src0;- actx.data_src1 = data_src1;- actx.data_dst = dst->data;+ gctx.data_src0 = data_src0;+ gctx.data_src1 = data_src1;+ gctx.data_dst = dst->data;- work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f16_per_thread, &actx, n_threads);+ work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f16_per_thread, &gctx, n_threads);return HTP_STATUS_OK;}-int op_activations(struct htp_ops_context * octx) {+int op_glu(struct htp_ops_context * octx) {switch (octx->src[0]->type) {case HTP_TYPE_F32:- return execute_op_activations_f32(octx);+ return execute_op_glu_f32(octx);case HTP_TYPE_F16:- return execute_op_activations_f16(octx);+ return execute_op_glu_f16(octx);default:return HTP_STATUS_NO_SUPPORT;diff --git a/ggml/src/ggml-hexagon/htp/glu-ops.h b/ggml/src/ggml-hexagon/htp/glu-ops.hnew file mode 100644index 0000000000..41b3857b12--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/glu-ops.h@@ -0,0 +1,76 @@+#ifndef HTP_GLU_OPS_H+#define HTP_GLU_OPS_H++#include "hex-common.h"+#include "htp-ops.h"+#include "htp-vtcm.h"+#include <stdint.h>++struct htp_glu_kernel_params {+ uint32_t n_threads;+ uint32_t vtcm_row_per_thread;+ uint32_t vtcm_size;+};++#if defined(__cplusplus)+static_assert(sizeof(struct htp_glu_kernel_params) <= 128, "htp_glu_kernel_params is too large for kernel_params blob");+#else+_Static_assert(sizeof(struct htp_glu_kernel_params) <= 128, "htp_glu_kernel_params is too large for kernel_params blob");+#endif++static inline bool htp_op_is_glu(uint32_t opcode) {+ switch (opcode) {+ case HTP_OP_GLU_SWIGLU:+ case HTP_OP_GLU_SWIGLU_OAI:+ case HTP_OP_GLU_SWIGLU_CLAMP:+ case HTP_OP_GLU_GEGLU:+ case HTP_OP_GLU_GEGLU_QUICK:+ case HTP_OP_GLU_GEGLU_ERF:+ case HTP_OP_GLU_REGLU:+ case HTP_OP_GLU_SIGMOID:+ return true;+ default:+ return false;+ }+}++struct htp_glu_vtcm_layout {+ size_t total_bytes;+ size_t off_src0;+ size_t off_src1;+ size_t off_dst;++ size_t row_size_aligned;+ size_t src0_bytes_per_thread;+ size_t src1_bytes_per_thread;+ size_t dst_bytes_per_thread;++ uint32_t vtcm_row_per_thread;+ uint32_t block;+};++static inline void htp_glu_vtcm_layout_build(+ struct htp_glu_vtcm_layout * L,+ size_t dst_row_size,+ uint32_t n_threads,+ uint32_t vtcm_row_per_thread+) {+ const size_t row_size_aligned = hex_round_up(dst_row_size, 128);++ L->row_size_aligned = row_size_aligned;+ L->vtcm_row_per_thread = vtcm_row_per_thread;+ L->block = vtcm_row_per_thread >> 1;++ L->src0_bytes_per_thread = row_size_aligned * vtcm_row_per_thread;+ L->src1_bytes_per_thread = row_size_aligned * vtcm_row_per_thread;+ L->dst_bytes_per_thread = row_size_aligned * vtcm_row_per_thread;++ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);+ VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes_per_thread * n_threads);+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);++ L->total_bytes = off;+}++#endif // HTP_GLU_OPS_Hdiff --git a/ggml/src/ggml-hexagon/htp/hex-common.h b/ggml/src/ggml-hexagon/htp/hex-common.hindex e6a52540d5..b943453a34 100644--- a/ggml/src/ggml-hexagon/htp/hex-common.h+++ b/ggml/src/ggml-hexagon/htp/hex-common.h@@ -17,6 +17,10 @@#define MIN(a, b) ((a) < (b) ? (a) : (b))#endif+#ifndef HEX_L2_LINE_SIZE+#define HEX_L2_LINE_SIZE 128+#endif+static inline uint32_t hex_ceil_pow2(uint32_t x) {if (x <= 1) { return 1; }int p = 2;diff --git a/ggml/src/ggml-hexagon/htp/hex-profile.h b/ggml/src/ggml-hexagon/htp/hex-profile.hindex a26961fc93..5c7a1a4888 100644--- a/ggml/src/ggml-hexagon/htp/hex-profile.h+++ b/ggml/src/ggml-hexagon/htp/hex-profile.h@@ -44,12 +44,14 @@ struct htp_thread_trace {};static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) {- if (tr->count < tr->max_events) {- uint32_t i = tr->count;+ if (!tr->max_events) {+ return;+ }+ uint32_t i = tr->count++;+ if (i < tr->max_events) {tr->events[i].id = id;tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0);tr->events[i].cycles = (uint32_t) hex_get_cycles();- tr->count++;}}diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.hindex 22a05b9179..5d14a85042 100644--- a/ggml/src/ggml-hexagon/htp/htp-ctx.h+++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h@@ -12,8 +12,10 @@#include <assert.h>#include <dspqueue.h>#include <stdatomic.h>+#include <stddef.h>#include <stdint.h>#include <stdbool.h>+#include <string.h>#ifndef HTP_MAX_NTHREADS#define HTP_MAX_NTHREADS 10@@ -66,14 +68,18 @@ struct htp_ops_context {// TODO convert these to an arraystruct htp_spad src0_spad;struct htp_spad src1_spad;- struct htp_spad src2_spad;- struct htp_spad src3_spad;struct htp_spad dst_spad;uint32_t flags;uint32_t n_threads;struct fastdiv_values n_threads_div;int status;++ // Must stay last: not cleared by htp_ops_context_reset(), ops init the caches they use+ union {+ dma_cache_dm dma_cache_dm[HTP_MAX_NTHREADS];+ dma_cache_fa dma_cache_fa[HTP_MAX_NTHREADS];+ } __attribute__((aligned(128)));};// Main context for htp DSP backend@@ -118,6 +124,13 @@ struct htp_context {size_t footprint;};+static inline void htp_ops_context_reset(struct htp_ops_context * octx, struct htp_context * ctx) {+ memset(octx, 0, offsetof(struct htp_ops_context, dma_cache_dm));+ octx->ctx = ctx;+ octx->n_threads = ctx->n_threads;+ octx->n_threads_div = ctx->n_threads_div;+}+static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) {if (n_threads == 0 || n_threads > octx->ctx->n_threads) {return false;@@ -148,7 +161,7 @@ int op_unary(struct htp_ops_context * octx);int op_sum(struct htp_ops_context * octx);int op_sum_rows(struct htp_ops_context * octx);int op_argmax(struct htp_ops_context * octx);-int op_activations(struct htp_ops_context * octx);+int op_glu(struct htp_ops_context * octx);int op_softmax(struct htp_ops_context * octx);int op_add_id(struct htp_ops_context * octx);int op_rope(struct htp_ops_context * octx);diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.hindex 9b4b86d3b9..63cb309ffd 100644--- a/ggml/src/ggml-hexagon/htp/htp-ops.h+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h@@ -38,11 +38,6 @@ enum htp_data_type {HTP_TYPE_INVALID};-// Constants for internal types-#define QK_Q8_0_TILED 128 // 32x32 Q8_0 tiled layout---// Do not reorder first 4 (used as an index)enum htp_op_code {HTP_OP_MUL = 0,@@ -115,6 +110,8 @@ enum htp_op_code {HTP_OP_UNARY_GELU_QUICK,HTP_OP_IM2COL_3D,HTP_OP_HADAMARD,+ HTP_OP_GLU_REGLU,+ HTP_OP_GLU_SIGMOID,HTP_OP_INVALID};@@ -125,8 +122,8 @@ enum htp_op_code {#define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS#define HTP_OP_MAX_KERN_PARAMS 32-#define HTP_OP_MAX_BUFS 16-#define HTP_OP_MAX_TENSORS 8192 // must stay under 64K (uint16)+#define HTP_OP_MAX_BUFS 64+#define HTP_OP_MAX_TENSORS 65534 // must stay under 64K (uint16)#define HTP_FENCE_TIMEOUT (1000000000ULL)#define HTP_FENCE_SLOT_SIZE 128@@ -136,9 +133,10 @@ enum htp_op_code {#define HTP_MMAP_MAX_VMEM (2147483648u)enum htp_tensor_flags {- HTP_TENSOR_WEIGHT = (1U << 0), // Tensor buffer model weight data (not compute)- HTP_TENSOR_TILED = (1U << 1), // Tensor is in tiled format- HTP_TENSOR_FENCE = (1U << 2) // Tensor is synchronization fence (explicitly managed)+ HTP_TENSOR_WEIGHT = (1U << 0), // Tensor buffer model weight data (not compute)+ HTP_TENSOR_TILED = (1U << 1), // Tensor is in tiled format+ HTP_TENSOR_FENCE = (1U << 2), // Tensor is synchronization fence (explicitly managed)+ HTP_TENSOR_FUSEABLE = (1U << 3), // Tensor can be fused (host-only)};enum htp_buf_flags {diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.hindex 9b2e13133f..01c6716796 100644--- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h+++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h@@ -233,7 +233,7 @@ static inline void quantize_block_f32_q8_0_tiled(float * restrict x, uint8_t * rstatic void quantize_row_f32_q8_0_tiled(float * restrict x, uint8_t * restrict y, uint32_t k) {assert(k % 32 == 0);- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (k + qk - 1) / qk;for (uint32_t i = 0; i < nb; i++) {@@ -244,7 +244,7 @@ static void quantize_row_f32_q8_0_tiled(float * restrict x, uint8_t * restrict ystatic void quantize_row_f32_q8_1_tiled(float * restrict x, uint8_t * restrict y, uint32_t k) {assert(k % 32 == 0);- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (k + qk - 1) / qk;for (uint32_t i = 0; i < nb; i++) {@@ -255,7 +255,7 @@ static void quantize_row_f32_q8_1_tiled(float * restrict x, uint8_t * restrict ystatic void quantize_row_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restrict y, uint32_t k) {assert(k % 32 == 0);- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (k + qk - 1) / qk;for (uint32_t i = 0; i < nb; i++) {@@ -310,7 +310,7 @@ static inline void quantize_block_f16_f16_tiled(const __fp16 * restrict x, uint8static void quantize_row_f32_f16_tiled(const float * restrict x, uint8_t * restrict y, uint32_t k) {assert(k % 32 == 0);- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (k + qk - 1) / qk;for (uint32_t i = 0; i < nb; i++) {@@ -321,7 +321,7 @@ static void quantize_row_f32_f16_tiled(const float * restrict x, uint8_t * restrstatic void quantize_row_f16_f16_tiled(const __fp16 * restrict x, uint8_t * restrict y, uint32_t k) {assert(k % 32 == 0);- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (k + qk - 1) / qk;for (uint32_t i = 0; i < nb; i++) {@@ -1963,7 +1963,7 @@ static inline void quantize_f32_q8_0_tiled_block_kernel(uint32_t c) {(void) tmp_data;- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (ne0 + qk - 1) / qk;for (uint32_t ib = ib_first; ib < ib_last; ++ib) {@@ -1993,7 +1993,7 @@ static inline void quantize_f32_q8_1_tiled_block_kernel(uint32_t c) {(void) tmp_data;- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (ne0 + qk - 1) / qk;for (uint32_t ib = ib_first; ib < ib_last; ++ib) {@@ -2023,7 +2023,7 @@ static inline void quantize_f32_q8_1_s16_tiled_block_kernel(uint32_t c) {(void) tmp_data;- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (ne0 + qk - 1) / qk;for (uint32_t ib = ib_first; ib < ib_last; ++ib) {@@ -2053,7 +2053,7 @@ static inline void quantize_f32_f16_tiled_block_kernel(uint32_t c) {(void) tmp_data;- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (ne0 + qk - 1) / qk;for (uint32_t ib = ib_first; ib < ib_last; ++ib) {@@ -2083,7 +2083,7 @@ static inline void quantize_f16_f16_tiled_block_kernel(uint32_t c) {(void) tmp_data;- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (ne0 + qk - 1) / qk;for (uint32_t ib = ib_first; ib < ib_last; ++ib) {diff --git a/ggml/src/ggml-hexagon/htp/hvx-norm.h b/ggml/src/ggml-hexagon/htp/hvx-norm.hindex 7ea945a339..145c3086e2 100644--- a/ggml/src/ggml-hexagon/htp/hvx-norm.h+++ b/ggml/src/ggml-hexagon/htp/hvx-norm.h@@ -11,7 +11,8 @@static inline void hvx_fast_rms_norm_f32(const uint8_t * restrict src,uint8_t * restrict dst,const int num_elems,- float epsilon) {+ float epsilon,+ float scale) {const HVX_Vector * restrict v_src = (HVX_Vector *) src;HVX_Vector * restrict v_dst = (HVX_Vector *) dst;@@ -48,6 +49,9 @@ static inline void hvx_fast_rms_norm_f32(const uint8_t * restrict src,// Scale full vectorsHVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v));+ if (scale != 0.0f) {+ scale_v = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(scale_v, hvx_vec_splat_f32(scale)));+ }#pragma unroll(4)for (int i = 0; i < nvec; i++) {@@ -72,7 +76,8 @@ static inline void hvx_fast_rms_norm_mul_f32(const uint8_t * restrict src,const uint8_t * restrict weight,uint8_t * restrict dst,const int num_elems,- float epsilon) {+ float epsilon,+ float scale) {const HVX_Vector * restrict v_src = (const HVX_Vector *) src;const HVX_Vector * restrict v_weight = (const HVX_Vector *) weight;HVX_Vector * restrict v_dst = (HVX_Vector *) dst;@@ -109,6 +114,9 @@ static inline void hvx_fast_rms_norm_mul_f32(const uint8_t * restrict src,// Scale and multiplyHVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v));+ if (scale != 0.0f) {+ scale_v = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(scale_v, hvx_vec_splat_f32(scale)));+ }#pragma unroll(4)for (int i = 0; i < nvec; i++) {diff --git a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.hindex b7c2d74091..d4d67ff2f0 100644--- a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h+++ b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h@@ -68,6 +68,14 @@ static inline HVX_Vector hvx_vec_tanh_f32(HVX_Vector x) {return Q6_Vsf_equals_Vqf32(res);}+static inline HVX_Vector hvx_vec_silu_f32(HVX_Vector x) {+ HVX_Vector one = hvx_vec_splat_f32(1.0f);+ HVX_Vector max_exp = hvx_vec_splat_f32(87.f);+ HVX_Vector min_exp = hvx_vec_splat_f32(-87.f);+ HVX_Vector sig = hvx_vec_fast_sigmoid_f32_guard(x, one, max_exp, min_exp);+ return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(x, sig));+}+#define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \do { \dst_type * restrict vdst = (dst_type *) dst; \diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.c b/ggml/src/ggml-hexagon/htp/im2col-ops.cindex ee3c7da111..fc49c403bd 100644--- a/ggml/src/ggml-hexagon/htp/im2col-ops.c+++ b/ggml/src/ggml-hexagon/htp/im2col-ops.c@@ -768,19 +768,13 @@ int op_im2col(struct htp_ops_context * octx) {return HTP_STATUS_NO_SUPPORT;}- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {- return HTP_STATUS_INVAL_PARAMS;- }-const struct htp_tensor * dst = octx->dst;uint32_t row_base = 0;uint32_t nrows = kparams->total_rows;if (octx->ctx->mdev.count > 1) {- const uint32_t row_size = dst->nb[2];- const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;const struct htp_tensor_mdev_range range =- htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0,+ htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0,octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);row_base = range.start;nrows = range.count;@@ -790,20 +784,27 @@ int op_im2col(struct htp_ops_context * octx) {return HTP_STATUS_OK;}+ if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nrows))) {+ return HTP_STATUS_INVAL_PARAMS;+ }++ struct htp_im2col_vtcm_layout layout;+ htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads);+struct htp_im2col_context ictx = { 0 };uint8_t * const base = octx->ctx->vtcm_base;- const uint32_t pth = MIN(octx->n_threads, nrows);+ const uint32_t pth = octx->n_threads;ictx.octx = octx;ictx.pe_row_base = row_base;ictx.pe_nrows = nrows;- ictx.pe_rows_per_thread = (nrows + pth - 1) / pth;+ ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div);ictx.pe_src_row_bytes = kparams->src_row_bytes;ictx.pe_dst_row_bytes = kparams->dst_row_bytes;- ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, kparams->off_src);- ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, kparams->off_dst);- ictx.pe_src_size_per_thread = kparams->src_bytes_per_thread;- ictx.pe_dst_size_per_thread = kparams->dst_bytes_per_thread;+ ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src);+ ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst);+ ictx.pe_src_size_per_thread = layout.src_bytes_per_thread;+ ictx.pe_dst_size_per_thread = layout.dst_bytes_per_thread;ictx.pe_owb = kparams->owb;ictx.pe_wb = kparams->wb;ictx.pe_nbpr = kparams->nbpr;@@ -821,9 +822,10 @@ int op_im2col(struct htp_ops_context * octx) {ictx.patch_stride = kparams->patch_stride;for (uint32_t ith = 0; ith < pth; ith++) {- uint32_t r = row_base + ictx.pe_rows_per_thread * ith;- ictx.thread_ioh[ith] = r % kparams->OH;- ictx.thread_in[ith] = r / kparams->OH;+ const uint32_t r = row_base + ictx.pe_rows_per_thread * ith;+ const uint32_t in = fastdiv(r, &kparams->div_OH);+ ictx.thread_ioh[ith] = r - in * kparams->OH;+ ictx.thread_in[ith] = in;}ictx.dst_elem = dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float);@@ -850,19 +852,13 @@ int op_im2col_3d(struct htp_ops_context * octx) {return HTP_STATUS_NO_SUPPORT;}- if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {- return HTP_STATUS_INVAL_PARAMS;- }-const struct htp_tensor * dst = octx->dst;uint32_t row_base = 0;uint32_t nrows = kparams->total_rows;if (octx->ctx->mdev.count > 1) {- const uint32_t row_size = dst->nb[2];- const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;const struct htp_tensor_mdev_range range =- htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0,+ htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0,octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);row_base = range.start;nrows = range.count;@@ -872,20 +868,27 @@ int op_im2col_3d(struct htp_ops_context * octx) {return HTP_STATUS_OK;}+ if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nrows))) {+ return HTP_STATUS_INVAL_PARAMS;+ }++ struct htp_im2col_vtcm_layout layout;+ htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads);+struct htp_im2col_context ictx = { 0 };uint8_t * const base = octx->ctx->vtcm_base;- const uint32_t pth = MIN(octx->n_threads, nrows);+ const uint32_t pth = octx->n_threads;ictx.octx = octx;ictx.pe_row_base = row_base;ictx.pe_nrows = nrows;- ictx.pe_rows_per_thread = (nrows + pth - 1) / pth;+ ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div);ictx.pe_src_row_bytes = kparams->src_row_bytes;ictx.pe_dst_row_bytes = kparams->dst_row_bytes;- ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, kparams->off_src);- ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, kparams->off_dst);- ictx.pe_src_size_per_thread = kparams->src_bytes_per_thread;- ictx.pe_dst_size_per_thread = kparams->dst_bytes_per_thread;+ ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src);+ ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst);+ ictx.pe_src_size_per_thread = layout.src_bytes_per_thread;+ ictx.pe_dst_size_per_thread = layout.dst_bytes_per_thread;ictx.pe_owb = kparams->owb;ictx.pe_wb = kparams->wb;ictx.pe_nbpr = kparams->nbpr;@@ -903,11 +906,12 @@ int op_im2col_3d(struct htp_ops_context * octx) {ictx.patch_stride = kparams->patch_stride;for (uint32_t ith = 0; ith < pth; ith++) {- uint32_t rem = row_base + ictx.pe_rows_per_thread * ith;- ictx.thread_ioh[ith] = rem % kparams->OH;- rem /= kparams->OH;- ictx.thread_iod[ith] = rem % kparams->OD;- ictx.thread_in[ith] = rem / kparams->OD;+ const uint32_t rem = row_base + ictx.pe_rows_per_thread * ith;+ const uint32_t q_oh = fastdiv(rem, &kparams->div_OH);+ ictx.thread_ioh[ith] = rem - q_oh * kparams->OH;+ const uint32_t in = fastdiv(q_oh, &kparams->div_OD);+ ictx.thread_iod[ith] = q_oh - in * kparams->OD;+ ictx.thread_in[ith] = in;}ictx.dst_elem = dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float);diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.h b/ggml/src/ggml-hexagon/htp/im2col-ops.hindex 9ccc21fd76..85aa3ca35f 100644--- a/ggml/src/ggml-hexagon/htp/im2col-ops.h+++ b/ggml/src/ggml-hexagon/htp/im2col-ops.h@@ -2,6 +2,8 @@#define HTP_IM2COL_OPS_H#include "hex-common.h"+#include "hex-fastdiv.h"+#include "htp-vtcm.h"#include <stdint.h>enum htp_im2col_kernel_type {@@ -11,38 +13,33 @@ enum htp_im2col_kernel_type {};struct htp_im2col_vtcm_layout {- uint32_t off_src;- uint32_t off_dst;+ size_t total_bytes;+ size_t off_src;+ size_t off_dst;uint32_t src_bytes_per_thread;uint32_t dst_bytes_per_thread;- uint32_t total_bytes;};static inline void htp_im2col_vtcm_layout_build(- struct htp_im2col_vtcm_layout * layout,+ struct htp_im2col_vtcm_layout * L,uint32_t src_row_bytes,uint32_t dst_row_bytes,uint32_t n_threads) {- layout->src_bytes_per_thread = 2 * src_row_bytes + 256;- layout->dst_bytes_per_thread = 2 * dst_row_bytes;+ L->src_bytes_per_thread = 2 * src_row_bytes + 256;+ L->dst_bytes_per_thread = 2 * dst_row_bytes;- layout->off_src = 0;- layout->off_dst = layout->off_src + layout->src_bytes_per_thread * n_threads;- layout->total_bytes = layout->off_dst + layout->dst_bytes_per_thread * n_threads;+ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_src, (size_t) L->src_bytes_per_thread * n_threads);+ VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) L->dst_bytes_per_thread * n_threads);+ L->total_bytes = off;}struct htp_im2col_kernel_params {uint8_t kernel_type;- uint8_t is_2d;- uint8_t is_3d;uint8_t n_threads;uint32_t vtcm_size;- uint32_t off_src;- uint32_t off_dst;- uint32_t src_bytes_per_thread;- uint32_t dst_bytes_per_thread;uint32_t src_row_bytes;uint32_t dst_row_bytes;@@ -57,6 +54,10 @@ struct htp_im2col_kernel_params {uint32_t OD, OH, OW;uint32_t patch_stride;uint32_t total_rows;+ uint32_t rows_per_chunk;++ struct fastdiv_values div_OH;+ struct fastdiv_values div_OD;};#if defined(__cplusplus)diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.cindex 5ac6c3c52b..01128057bf 100644--- a/ggml/src/ggml-hexagon/htp/main.c+++ b/ggml/src/ggml-hexagon/htp/main.c@@ -955,7 +955,9 @@ static int execute_op(struct htp_ops_context * octx) {case HTP_OP_GLU_GEGLU:case HTP_OP_GLU_GEGLU_QUICK:case HTP_OP_GLU_GEGLU_ERF:- return op_activations(octx);+ case HTP_OP_GLU_REGLU:+ case HTP_OP_GLU_SIGMOID:+ return op_glu(octx);case HTP_OP_SOFTMAX:return op_softmax(octx);@@ -1244,8 +1246,6 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufsoctx->src0_spad.src = NULL;octx->src1_spad.src = NULL;- octx->src2_spad.src = NULL;- octx->src3_spad.src = NULL;octx->dst_spad.src = NULL;return octx->status;@@ -1306,10 +1306,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_rprep_tensors(ctx, bufs, tens, n_tens);struct htp_ops_context *octx = &ctx->octx;- memset(octx, 0, sizeof(*octx));- octx->n_threads = ctx->n_threads;- octx->n_threads_div = ctx->n_threads_div;- octx->ctx = ctx;+ htp_ops_context_reset(octx, ctx);mdev_group_init(ctx, req);diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.cindex 9af3129254..e290835fce 100644--- a/ggml/src/ggml-hexagon/htp/matmul-ops.c+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c@@ -1816,7 +1816,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {return HTP_STATUS_NO_SUPPORT;}- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (ne10 + qk - 1) / qk;const uint32_t total_nb = act_nrows * nb;@@ -1886,7 +1886,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {mmctx->vtcm_src0_stride = src0_row_size_padded;mmctx->vtcm_act_stride = act_row_size;if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW) {- mmctx->vtcm_act_raw_stride = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));+ mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(ne10);} else if (kparams->kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM) {mmctx->vtcm_act_raw_stride = hex_round_up(ne10 * sizeof(float), 128);} else {@@ -1904,7 +1904,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {if (need_quant) {hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, m_start, cur_m_rows);- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (ne10 + qk - 1) / qk;const uint32_t total_nb = cur_m_rows * nb;uint32_t quant_tasks;@@ -3914,7 +3914,7 @@ static int hvx_mm_matmul_id(const struct htp_tensor * restrict ids = octx->src[2];const size_t src0_row_size = nb01;- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (ne10 + qk - 1) / qk;const uint32_t total_nb = act_nrows * nb;@@ -3966,7 +3966,7 @@ static int hvx_mm_matmul_id(mmctx->vtcm_src0_stride = src0_row_size_padded;mmctx->vtcm_act_stride = act_row_size;- mmctx->vtcm_act_raw_stride = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));+ mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(ne10);mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div);mmctx->vtcm_act_size_per_thread = L.act_bytes;@@ -4081,7 +4081,7 @@ static int hvx_mm_matmul_id_nx(const struct htp_tensor * restrict ids = octx->src[n_weights + 1];const size_t src0_row_size = src0->nb[1];- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (act->ne[0] + qk - 1) / qk;const uint32_t total_nb = act_nrows * nb;@@ -4124,7 +4124,7 @@ static int hvx_mm_matmul_id_nx(mmctx->vtcm_src0_stride = 0;mmctx->vtcm_act_stride = act_row_size;- mmctx->vtcm_act_raw_stride = hex_round_up(act->ne[0] * sizeof(float), QK_Q8_0_TILED * sizeof(float));+ mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(act->ne[0]);mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div);mmctx->vtcm_act_size_per_thread = L.act_bytes;@@ -4474,7 +4474,7 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) {return HTP_STATUS_NO_SUPPORT;}- const uint32_t qk = QK_Q8_0_TILED;+ const uint32_t qk = HTP_MM_ACT_BLOCK_K;const uint32_t nb = (act->ne[0] + qk - 1) / qk;const uint32_t total_nb = act_nrows * nb;@@ -4518,7 +4518,7 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) {mmctx->vtcm_src0_stride = 0;mmctx->vtcm_act_stride = act_row_size;- mmctx->vtcm_act_raw_stride = hex_round_up(act->ne[0] * sizeof(float), QK_Q8_0_TILED * sizeof(float));+ mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(act->ne[0]);mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div);mmctx->vtcm_act_size_per_thread = L.act_bytes;diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.hindex 09875a6038..474c2dbc01 100644--- a/ggml/src/ggml-hexagon/htp/matmul-ops.h+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h@@ -25,6 +25,7 @@ extern "C" {#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0 640// --- Activation Tiled Block Sizes (including padding) ---+#define HTP_MM_ACT_BLOCK_K 128#define HTP_MM_ACT_TILE_SIZE_Q8_0 1152#define HTP_MM_ACT_TILE_SIZE_Q8_1 1280#define HTP_MM_ACT_TILE_SIZE_F16 2048@@ -74,10 +75,7 @@ struct htp_mm_kernel_params {int32_t aligned_tile_size; // Aligned weight tile size (padded to 128)int32_t act_row_size; // Row size for activation scratchpadint32_t vtcm_size; // Total required scratchpad size in VTCM- int32_t vtcm_src0_size; // src0 scratchpad size in VTCM- int32_t vtcm_act_size; // activation scratchpad size in VTCMint32_t vtcm_bias_size; // bias scratchpad size in VTCM (fused only)- int32_t vtcm_dst_size; // dst scratchpad size in VTCM// Precomputed division valuesstruct fastdiv_values div_ne12_ne1;@@ -192,19 +190,19 @@ static inline bool htp_mm_weight_has_offset(int weight_type) {// --- Activation/Row Size Helpers ---static inline size_t htp_mm_q8_0_tiled_row_size(uint32_t ne) {- const uint32_t ne_padded = ((ne + 127) / 128) * 128;+ const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K);const uint32_t nb_32 = ne_padded / 32;return nb_32 * HTP_MM_ACT_TILE_SIZE_Q8_0;}static inline size_t htp_mm_q8_1_tiled_row_size(uint32_t ne) {- const uint32_t ne_padded = ((ne + 127) / 128) * 128;+ const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K);const uint32_t nb_32 = ne_padded / 32;return nb_32 * HTP_MM_ACT_TILE_SIZE_Q8_1;}static inline size_t htp_mm_f16_tiled_row_size(uint32_t ne) {- const uint32_t ne_padded = ((ne + 127) / 128) * 128;+ const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K);const uint32_t nb_32 = ne_padded / 32;return nb_32 * HTP_MM_ACT_TILE_SIZE_F16;}@@ -216,6 +214,10 @@ static inline size_t htp_mm_tiled_act_row_size(int weight_type, uint32_t ne) {return htp_mm_weight_has_offset(weight_type) ? htp_mm_q8_1_tiled_row_size(ne) : htp_mm_q8_0_tiled_row_size(ne);}+static inline size_t htp_mm_act_raw_row_size(uint32_t ne) {+ return hex_round_up(ne, HTP_MM_ACT_BLOCK_K) * sizeof(float);+}+static inline size_t htp_mm_round_up(size_t n, size_t m) {return ((n + m - 1) / m) * m;}@@ -445,7 +447,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(size_t tiled_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);size_t q_act_sz = hex_round_up(tiled_act_row_size * act_nrows, 128);- size_t raw_row_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));+ size_t raw_row_size = htp_mm_act_raw_row_size(ne10);src0_sz = weight_sz_per_thread * n_threads; // shared single-weight prefetch bufferact_sz = q_act_sz; // quantized activation buffer@@ -461,7 +463,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(act_sz = htp_mm_round_up(act_row_size_tiled * act_nrows, 256);- size_t raw_row_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));+ size_t raw_row_size = htp_mm_act_raw_row_size(ne10);src0_sz = src0_sz_per_thread * n_threads;dst_sz = 0;@@ -502,7 +504,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(size_t dst_slice_per_thread = (dst_nrows > 0 && act_nrows == 1) ? htp_mm_round_up((dst_row_size + n_threads - 1) / n_threads, 128) : 0;dst_sz = dst_slice_per_thread * n_threads;- size_t raw_row_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));+ size_t raw_row_size = htp_mm_act_raw_row_size(ne10);act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128);break;}@@ -582,7 +584,7 @@ static inline bool htp_mm_hvx_solve_vtcm_params(size_t eff_row_size = row_size;if (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {- eff_row_size += hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));+ eff_row_size += htp_mm_act_raw_row_size(ne10);}uint32_t m_chunk = (uint32_t) (avail_act / eff_row_size);diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.c b/ggml/src/ggml-hexagon/htp/pad-ops.cindex 85f25a8eb7..cda6df318c 100644--- a/ggml/src/ggml-hexagon/htp/pad-ops.c+++ b/ggml/src/ggml-hexagon/htp/pad-ops.c@@ -17,18 +17,28 @@#include "htp-ctx.h"#include "htp-ops.h"#include "htp-tensor.h"+#include "htp-vtcm.h"+#include "pad-ops.h"/* Circular wrap: maps any integer x into [0, n) */-static inline uint32_t wrap_around(int32_t x, uint32_t n) {- return (uint32_t)(((x % (int32_t)n) + (int32_t)n) % (int32_t)n);+static inline uint32_t wrap_around(int32_t x, uint32_t n, const struct fastdiv_values * div_n) {+ if (x >= 0) {+ const uint32_t ux = (uint32_t) x;+ return ux < n ? ux : fastmodulo(ux, n, div_n);+ }+ const uint32_t ux = (uint32_t) (-x);+ const uint32_t rem = ux < n ? ux : fastmodulo(ux, n, div_n);+ return rem ? (n - rem) : 0;}/* Decompose a flat dst row index into (i1, i2, i3) */static inline void pad_decompose_row(uint32_t ir, uint32_t ne1, uint32_t ne2,+ const struct htp_pad_kernel_params * kparams,uint32_t *i1, uint32_t *i2, uint32_t *i3) {- *i1 = ir % ne1;- *i2 = (ir / ne1) % ne2;- *i3 = ir / (ne1 * ne2);+ *i3 = fastdiv(ir, &kparams->div_ne1_ne2);+ const uint32_t rem = ir - (*i3) * (ne1 * ne2);+ *i2 = fastdiv(rem, &kparams->div_ne1);+ *i1 = rem - (*i2) * ne1;}/* Return non-zero if row (i1,i2,i3) falls in the non-padded interior */@@ -62,25 +72,28 @@ static inline dma_addr_t pad_src_row_data(const struct htp_tensor * src,/* Compute the DDR src row pointer for a circular row (wrap-around indexing) */static inline const uint8_t * pad_circ_src_row_ptr(const struct htp_tensor * src,+ const struct htp_pad_kernel_params * kparams,uint32_t i1, uint32_t i2, uint32_t i3,int32_t lp1, int32_t lp2, int32_t lp3) {return (const uint8_t *) src->data- + wrap_around((int32_t)i1 - lp1, src->ne[1]) * src->nb[1]- + wrap_around((int32_t)i2 - lp2, src->ne[2]) * src->nb[2]- + wrap_around((int32_t)i3 - lp3, src->ne[3]) * src->nb[3];+ + wrap_around((int32_t)i1 - lp1, src->ne[1], &kparams->div_ne01) * src->nb[1]+ + wrap_around((int32_t)i2 - lp2, src->ne[2], &kparams->div_ne02) * src->nb[2]+ + wrap_around((int32_t)i3 - lp3, src->ne[3], &kparams->div_ne03) * src->nb[3];}static inline dma_addr_t pad_circ_src_row_data(const struct htp_tensor * src,+ const struct htp_pad_kernel_params * kparams,uint32_t i1, uint32_t i2, uint32_t i3,int32_t lp1, int32_t lp2, int32_t lp3) {return src->data- + wrap_around((int32_t)i1 - lp1, src->ne[1]) * src->nb[1]- + wrap_around((int32_t)i2 - lp2, src->ne[2]) * src->nb[2]- + wrap_around((int32_t)i3 - lp3, src->ne[3]) * src->nb[3];+ + wrap_around((int32_t)i1 - lp1, src->ne[1], &kparams->div_ne01) * src->nb[1]+ + wrap_around((int32_t)i2 - lp2, src->ne[2], &kparams->div_ne02) * src->nb[2]+ + wrap_around((int32_t)i3 - lp3, src->ne[3], &kparams->div_ne03) * src->nb[3];}struct htp_pad_context {struct htp_ops_context * octx;+ const struct htp_pad_kernel_params * kparams;int32_t lp0, rp0;int32_t lp1, rp1;@@ -152,7 +165,7 @@ static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * dafor (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {uint32_t i1, i2, i3;- pad_decompose_row(dst_row, ne1, ne2, &i1, &i2, &i3);+ pad_decompose_row(dst_row, ne1, ne2, pctx->kparams, &i1, &i2, &i3);uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;@@ -219,7 +232,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, voiddst_row_size, dst_row_size_aligned, dst_row_size, 0);uint32_t i1, i2, i3;- pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3);+ pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3);const int interior = pad_is_interior(i1, i2, i3,lp1, rp1, ne1,lp2, rp2, ne2,@@ -246,7 +259,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, voiduint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma_q).dst;uint32_t i1, i2, i3;- pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3);+ pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3);const dma_addr_t dst_data = dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;@@ -278,7 +291,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, voidconst uint32_t next_row = ir + 2;if (next_row < row_end) {uint32_t ni1, ni2, ni3;- pad_decompose_row(next_row, ne1, ne2, &ni1, &ni2, &ni3);+ pad_decompose_row(next_row, ne1, ne2, pctx->kparams, &ni1, &ni2, &ni3);const int next_interior = pad_is_interior(ni1, ni2, ni3,lp1, rp1, ne1,lp2, rp2, ne2,@@ -315,10 +328,10 @@ static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith,for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {uint32_t i1, i2, i3;- pad_decompose_row(dst_row, ne1, ne2, &i1, &i2, &i3);+ pad_decompose_row(dst_row, ne1, ne2, pctx->kparams, &i1, &i2, &i3);uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;- const uint8_t * src_row = pad_circ_src_row_ptr(src, i1, i2, i3, lp1, lp2, lp3);+ const uint8_t * src_row = pad_circ_src_row_ptr(src, pctx->kparams, i1, i2, i3, lp1, lp2, lp3);if (nb00 == type_size) {@@ -393,8 +406,8 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int idst_row_size, dst_row_size_aligned, dst_row_size, 0);uint32_t pi1, pi2, pi3;- pad_decompose_row(ir, ne1, ne2, &pi1, &pi2, &pi3);- const dma_addr_t src_data = pad_circ_src_row_data(src, pi1, pi2, pi3, lp1, lp2, lp3);+ pad_decompose_row(ir, ne1, ne2, pctx->kparams, &pi1, &pi2, &pi3);+ const dma_addr_t src_data = pad_circ_src_row_data(src, pctx->kparams, pi1, pi2, pi3, lp1, lp2, lp3);dma_queue_push(dma_q,dma_make_data(src_spad_cur, src_data),src_row_size_aligned, src_row_size, src_row_size, 1);@@ -411,7 +424,7 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int iuint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma_q).dst;uint32_t i1, i2, i3;- pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3);+ pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3);const dma_addr_t dst_data = dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);@@ -455,8 +468,8 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int iconst uint32_t next_row = ir + 2;if (next_row < row_end) {uint32_t nri1, nri2, nri3;- pad_decompose_row(next_row, ne1, ne2, &nri1, &nri2, &nri3);- const dma_addr_t next_src_data = pad_circ_src_row_data(src, nri1, nri2, nri3, lp1, lp2, lp3);+ pad_decompose_row(next_row, ne1, ne2, pctx->kparams, &nri1, &nri2, &nri3);+ const dma_addr_t next_src_data = pad_circ_src_row_data(src, pctx->kparams, nri1, nri2, nri3, lp1, lp2, lp3);dma_queue_push(dma_q,dma_make_data(src_spad_cur, next_src_data),src_row_size_aligned, src_row_size, src_row_size, 1);@@ -475,15 +488,8 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int iint op_pad(struct htp_ops_context * octx) {const struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * dst = octx->dst;-- // Only F32 supported- size_t type_size;- switch (src0->type) {- case HTP_TYPE_F32: type_size = 4; break;- default:- FARF(ERROR, "pad-hvx: unsupported type %u\n", src0->type);- return HTP_STATUS_NO_SUPPORT;- }+ const struct htp_pad_kernel_params * kparams =+ (const struct htp_pad_kernel_params *) octx->kernel_params;const int32_t lp0 = octx->op_params[0];const int32_t rp0 = octx->op_params[1];@@ -495,11 +501,10 @@ int op_pad(struct htp_ops_context * octx) {const int32_t rp3 = octx->op_params[7];const int32_t circular = octx->op_params[8];- const uint32_t ne0 = dst->ne[0];- const uint32_t ne00 = src0->ne[0];-- const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3];- const size_t dst_row_size = (size_t)ne0 * type_size;+ const uint32_t total_dst_rows = kparams->total_rows;+ const size_t type_size = kparams->elem_size;+ const size_t src_row_size = src0->ne[0] * type_size;+ const size_t dst_row_size = dst->ne[0] * type_size;uint32_t row_start = 0;uint32_t nrows = total_dst_rows;@@ -516,60 +521,68 @@ int op_pad(struct htp_ops_context * octx) {return HTP_STATUS_OK;}- const uint32_t n_threads = octx->n_threads;-- const size_t src_row_size = (size_t)ne00 * type_size;- const size_t src_row_size_aligned = hex_round_up(src_row_size, VLEN);- const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);-- // Total VTCM needed: 2 buffers (ping+pong) for src and dst, per thread- const size_t vtcm_needed = (size_t)n_threads * 2 * (src_row_size_aligned + dst_row_size_aligned);+ if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {+ return HTP_STATUS_INVAL_PARAMS;+ }- const int use_dma = (src0->nb[0] == (uint32_t)type_size) && (ne00 >= 512) &&- (octx->ctx->vtcm_size >= vtcm_needed);+ const uint32_t n_threads = octx->n_threads;- if (!use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {+ if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {return HTP_STATUS_NO_SUPPORT;}- if (use_dma) {- octx->src0_spad.size_per_thread = 2 * src_row_size_aligned;- octx->dst_spad.size_per_thread = 2 * dst_row_size_aligned;- octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread;- octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread;- octx->src0_spad.data = octx->ctx->vtcm_base;- octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size;+ struct htp_pad_vtcm_layout layout = { 0 };+ if (kparams->use_dma) {+ htp_pad_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_threads);++ octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread;+ octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread;+ octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread;+ octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread;+ octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0);+ octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst);octx->src0_spad.src = NULL;octx->dst_spad.src = NULL;}struct htp_pad_context pctx = {- .octx = octx,+ .octx = octx,+ .kparams = kparams,.lp0 = lp0, .rp0 = rp0,.lp1 = lp1, .rp1 = rp1,.lp2 = lp2, .rp2 = rp2,.lp3 = lp3, .rp3 = rp3,- .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),- .total_dst_rows = nrows,- .row_start = row_start,- .type_size = type_size,+ .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),+ .total_dst_rows = nrows,+ .row_start = row_start,+ .type_size = type_size,.src_row_size = src_row_size,- .src_row_size_aligned = src_row_size_aligned,+ .src_row_size_aligned = layout.src_row_size_aligned,.dst_row_size = dst_row_size,- .dst_row_size_aligned = dst_row_size_aligned,+ .dst_row_size_aligned = layout.dst_row_size_aligned,};FARF(HIGH, "pad-hvx%s%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) pads=(%d,%d,%d,%d,%d,%d,%d,%d)\n",- circular ? "-circ" : "",- use_dma ? "-dma" : "",+ circular ? "-circ" : "",+ kparams->use_dma ? "-dma" : "",src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3);- if (circular && use_dma) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); }- else if (circular) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular, &pctx, n_threads); }- else if (use_dma) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_dma, &pctx, n_threads); }- else { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx, &pctx, n_threads); }+ switch (kparams->kernel_type) {+ case HTP_PAD_KERNEL_HVX_CIRCULAR_DMA:+ work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads);+ break;+ case HTP_PAD_KERNEL_HVX_CIRCULAR:+ work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular, &pctx, n_threads);+ break;+ case HTP_PAD_KERNEL_HVX_DMA:+ work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_dma, &pctx, n_threads);+ break;+ default:+ work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx, &pctx, n_threads);+ break;+ }return HTP_STATUS_OK;}diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.h b/ggml/src/ggml-hexagon/htp/pad-ops.hnew file mode 100644index 0000000000..5b8557538c--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/pad-ops.h@@ -0,0 +1,65 @@+#ifndef HTP_PAD_OPS_H+#define HTP_PAD_OPS_H++#include "hex-common.h"+#include "hex-fastdiv.h"+#include "htp-vtcm.h"+#include <stdint.h>++enum htp_pad_kernel_type {+ HTP_PAD_KERNEL_HVX = 0,+ HTP_PAD_KERNEL_HVX_DMA = 1,+ HTP_PAD_KERNEL_HVX_CIRCULAR = 2,+ HTP_PAD_KERNEL_HVX_CIRCULAR_DMA = 3,+};++struct htp_pad_vtcm_layout {+ size_t total_bytes;+ size_t off_src0;+ size_t off_dst;+ size_t src_row_size_aligned;+ size_t dst_row_size_aligned;+ size_t src0_bytes_per_thread;+ size_t dst_bytes_per_thread;+};++static inline void htp_pad_vtcm_layout_build(+ struct htp_pad_vtcm_layout * L,+ size_t src_row_size,+ size_t dst_row_size,+ uint32_t n_threads+) {+ L->src_row_size_aligned = hex_round_up(src_row_size, 128);+ L->dst_row_size_aligned = hex_round_up(dst_row_size, 128);+ L->src0_bytes_per_thread = 2 * L->src_row_size_aligned;+ L->dst_bytes_per_thread = 2 * L->dst_row_size_aligned;++ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);++ L->total_bytes = off;+}++struct htp_pad_kernel_params {+ uint8_t kernel_type;+ uint32_t n_threads;+ uint32_t total_rows;+ uint32_t use_dma;+ uint32_t elem_size;+ uint32_t vtcm_size;++ struct fastdiv_values div_ne1;+ struct fastdiv_values div_ne1_ne2;+ struct fastdiv_values div_ne01;+ struct fastdiv_values div_ne02;+ struct fastdiv_values div_ne03;+};++#if defined(__cplusplus)+static_assert(sizeof(struct htp_pad_kernel_params) <= 128, "htp_pad_kernel_params is too large for kernel_params blob");+#else+_Static_assert(sizeof(struct htp_pad_kernel_params) <= 128, "htp_pad_kernel_params is too large for kernel_params blob");+#endif++#endif // HTP_PAD_OPS_Hdiff --git a/ggml/src/ggml-hexagon/htp/pool-ops.c b/ggml/src/ggml-hexagon/htp/pool-ops.cindex ea1f1a5979..b41c7bed23 100644--- a/ggml/src/ggml-hexagon/htp/pool-ops.c+++ b/ggml/src/ggml-hexagon/htp/pool-ops.c@@ -247,6 +247,7 @@ typedef void (*pool_plane_fn_t)(const float * src, float * dst, const struct htpstruct pool_2d_context {struct htp_ops_context * octx;const struct htp_pool_2d_kernel_params * kparams;+ struct htp_pool_vtcm_layout layout;pool_plane_fn_t pool_plane;uint32_t n_threads;uint32_t plane_start;@@ -257,6 +258,7 @@ struct pool_2d_context {static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) {struct pool_2d_context * ctx = (struct pool_2d_context *) data;const struct htp_pool_2d_kernel_params * p = ctx->kparams;+ const struct htp_pool_vtcm_layout * L = &ctx->layout;const struct htp_tensor * src0 = ctx->octx->src[0];const struct htp_tensor * dst = ctx->octx->dst;pool_plane_fn_t pool_plane = ctx->pool_plane;@@ -271,15 +273,14 @@ static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) {struct htp_thread_trace * tr = &ctx->octx->ctx->trace[ith];dma_queue * dma_queue = ctx->octx->ctx->dma[ith];- const uint32_t src_spad_half = p->src_plane_bytes_aligned;- const uint32_t dst_spad_half = p->dst_plane_bytes_aligned;- const uint32_t src_bytes_per_thread = 2 * src_spad_half;- const uint32_t dst_bytes_per_thread = 2 * dst_spad_half;- const size_t off_dst = (size_t) ctx->n_threads * src_bytes_per_thread;+ const uint32_t src_plane_bytes = L->src_plane_bytes;+ const uint32_t dst_plane_bytes = L->dst_plane_bytes;+ const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size;+ const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size;uint8_t * vtcm_base = (uint8_t *) ctx->octx->ctx->vtcm_base;- uint8_t * src_spad = vtcm_base + ith * src_bytes_per_thread;- uint8_t * dst_spad = vtcm_base + off_dst + ith * dst_bytes_per_thread;+ uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread;+ uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread;float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) };float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) };@@ -290,14 +291,14 @@ static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) {for (uint32_t i = 0; i < total && i < 2; ++i) {dma_queue_push(dma_queue,dma_make_data(dst->data, dstb2[i]),- p->dst_plane_bytes, dst_spad_half,- p->dst_plane_bytes, 0);+ dst_plane_bytes, dst_spad_half,+ dst_plane_bytes, 0);- const dma_addr_t src_addr = src0->data + (first + i) * p->src_plane_bytes;+ const dma_addr_t src_addr = src0->data + (first + i) * src_plane_bytes;dma_queue_push(dma_queue,dma_make_data(srcb2[i], src_addr),- src_spad_half, p->src_plane_bytes,- p->src_plane_bytes, 1);+ src_spad_half, src_plane_bytes,+ src_plane_bytes, 1);}for (uint32_t i = 0; i < total; ++i) {@@ -313,18 +314,18 @@ static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) {pool_plane(srcb, dstb, p);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) plane);- const dma_addr_t dst_addr = dst->data + plane * p->dst_plane_bytes;+ const dma_addr_t dst_addr = dst->data + plane * dst_plane_bytes;dma_queue_push(dma_queue,dma_make_data(dst_addr, dstb),- p->dst_plane_bytes, dst_spad_half,- p->dst_plane_bytes, 1);+ dst_plane_bytes, dst_spad_half,+ dst_plane_bytes, 1);if (i + 2 < total) {- const dma_addr_t next_src_addr = src0->data + (plane + 2) * p->src_plane_bytes;+ const dma_addr_t next_src_addr = src0->data + (plane + 2) * src_plane_bytes;dma_queue_push(dma_queue,dma_make_data(srcb, next_src_addr),- src_spad_half, p->src_plane_bytes,- p->src_plane_bytes, 1);+ src_spad_half, src_plane_bytes,+ src_plane_bytes, 1);}}@@ -337,22 +338,15 @@ static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) {}int op_pool_2d(struct htp_ops_context * octx) {- const struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * dst = octx->dst;const struct htp_pool_2d_kernel_params * p =(const struct htp_pool_2d_kernel_params *) octx->kernel_params;- if (src0->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32 ||- (p->pool_op != HTP_POOL_AVG && p->pool_op != HTP_POOL_MAX)) {- return HTP_STATUS_NO_SUPPORT;- }-uint32_t plane_start = 0;uint32_t plane_count = p->planes;if (octx->ctx->mdev.count > 1) {- const uint32_t planes_per_chunk = (p->dst_plane_bytes > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(p->dst_plane_bytes, HEX_L2_LINE_SIZE)) : 1;const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(- plane_count, htp_tensor_mdev_data_aligned(dst) ? planes_per_chunk : 0,+ plane_count, htp_tensor_mdev_data_aligned(dst) ? p->planes_per_chunk : 0,octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);plane_start = range.start;plane_count = range.count;@@ -369,16 +363,12 @@ int op_pool_2d(struct htp_ops_context * octx) {const uint32_t planes_per_thread = fastdiv(plane_count + n_threads - 1, &octx->n_threads_div);pool_plane_fn_t pool_plane;- if (p->global_path) {- pool_plane = pool_plane_global;- } else if (p->block_path) {- pool_plane = pool_plane_block;- } else if (p->narrow_path) {- pool_plane = pool_plane_hvx_narrow;- } else if (p->fast_path) {- pool_plane = pool_plane_hvx;- } else {- pool_plane = pool_plane_general;+ switch (p->kernel_type) {+ case HTP_POOL_KERNEL_GLOBAL: pool_plane = pool_plane_global; break;+ case HTP_POOL_KERNEL_BLOCK: pool_plane = pool_plane_block; break;+ case HTP_POOL_KERNEL_HVX_NARROW: pool_plane = pool_plane_hvx_narrow; break;+ case HTP_POOL_KERNEL_HVX: pool_plane = pool_plane_hvx; break;+ default: pool_plane = pool_plane_general; break;}struct pool_2d_context ctx = {@@ -390,6 +380,7 @@ int op_pool_2d(struct htp_ops_context * octx) {.plane_count = plane_count,.planes_per_thread = planes_per_thread,};+ htp_pool_vtcm_layout_build(&ctx.layout, p->src_x, p->src_y, p->dst_x, p->dst_y, n_threads);work_queue_run(octx->ctx->work_queue, pool_2d_thread, &ctx, n_threads);return HTP_STATUS_OK;}diff --git a/ggml/src/ggml-hexagon/htp/pool-ops.h b/ggml/src/ggml-hexagon/htp/pool-ops.hindex 66b405a8b3..178e5d00fc 100644--- a/ggml/src/ggml-hexagon/htp/pool-ops.h+++ b/ggml/src/ggml-hexagon/htp/pool-ops.h@@ -7,8 +7,18 @@#include <string.h>#include "hex-common.h"+#include "htp-vtcm.h"++enum htp_pool_kernel_type {+ HTP_POOL_KERNEL_GENERAL = 0,+ HTP_POOL_KERNEL_HVX,+ HTP_POOL_KERNEL_HVX_NARROW,+ HTP_POOL_KERNEL_GLOBAL,+ HTP_POOL_KERNEL_BLOCK,+};struct htp_pool_2d_kernel_params {+ uint8_t kernel_type;uint32_t src_x;uint32_t src_y;uint32_t dst_x;@@ -19,17 +29,10 @@ struct htp_pool_2d_kernel_params {uint32_t stride_y;int32_t pad_x;int32_t pad_y;- uint32_t src_plane_bytes;- uint32_t dst_plane_bytes;- uint32_t src_plane_bytes_aligned;- uint32_t dst_plane_bytes_aligned;uint32_t n_threads;uint32_t planes;+ uint32_t planes_per_chunk;uint32_t pool_op;- uint32_t fast_path;- uint32_t narrow_path;- uint32_t global_path;- uint32_t block_path;uint32_t avg_divide_count;uint32_t ox_lo;uint32_t ox_hi;@@ -46,12 +49,36 @@ struct htp_pool_vtcm_layout {size_t total_bytes;size_t off_src;size_t off_dst;+ uint32_t src_plane_bytes;+ uint32_t dst_plane_bytes;size_t src_bytes_per_thread;size_t dst_bytes_per_thread;size_t src_spad_half_size;size_t dst_spad_half_size;};+static inline void htp_pool_vtcm_layout_build(+ struct htp_pool_vtcm_layout * L,+ uint32_t src_x,+ uint32_t src_y,+ uint32_t dst_x,+ uint32_t dst_y,+ uint32_t n_threads+) {+ // Full-plane double buffering (256 bytes guard space for vector loads)+ L->src_plane_bytes = src_x * src_y * sizeof(float);+ L->dst_plane_bytes = dst_x * dst_y * sizeof(float);+ L->src_spad_half_size = hex_round_up(L->src_plane_bytes + 256, 128);+ L->dst_spad_half_size = hex_round_up(L->dst_plane_bytes, 128);+ L->src_bytes_per_thread = 2 * L->src_spad_half_size;+ L->dst_bytes_per_thread = 2 * L->dst_spad_half_size;++ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads);+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);+ L->total_bytes = off;+}+static inline bool htp_pool_solve_layout(struct htp_pool_vtcm_layout * layout,uint32_t src_x,@@ -61,25 +88,8 @@ static inline bool htp_pool_solve_layout(uint32_t n_threads,size_t vtcm_budget) {- // Full-plane double buffering (256 bytes guard space for vector loads)- const size_t src_plane_bytes = (size_t) src_x * src_y * sizeof(float);- const size_t dst_plane_bytes = (size_t) dst_x * dst_y * sizeof(float);- const size_t src_plane_aligned = hex_round_up((uint32_t) src_plane_bytes + 256, 128);- const size_t dst_plane_aligned = hex_round_up((uint32_t) dst_plane_bytes, 128);-- const size_t spad_per_thread = 2 * (src_plane_aligned + dst_plane_aligned);- if (spad_per_thread * n_threads > vtcm_budget) {- return false;- }-- layout->src_spad_half_size = src_plane_aligned;- layout->dst_spad_half_size = dst_plane_aligned;- layout->src_bytes_per_thread = 2 * src_plane_aligned;- layout->dst_bytes_per_thread = 2 * dst_plane_aligned;- layout->off_src = 0;- layout->off_dst = layout->src_bytes_per_thread * n_threads;- layout->total_bytes = layout->off_dst + layout->dst_bytes_per_thread * n_threads;- return true;+ htp_pool_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, n_threads);+ return layout->total_bytes <= vtcm_budget;}#endif // HTP_POOL_OPS_Hdiff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.c b/ggml/src/ggml-hexagon/htp/repeat-ops.cindex 2551be225b..827ef24c57 100644--- a/ggml/src/ggml-hexagon/htp/repeat-ops.c+++ b/ggml/src/ggml-hexagon/htp/repeat-ops.c@@ -16,14 +16,11 @@#include "hex-profile.h"#include "htp-ops.h"#include "htp-tensor.h"+#include "repeat-ops.h"struct htp_repeat_context {- struct htp_ops_context * octx;-- uint32_t nr0;- uint32_t nr1;- uint32_t nr2;- uint32_t nr3;+ struct htp_ops_context * octx;+ const struct htp_repeat_kernel_params * kparams;uint32_t nrows_per_thread;uint32_t total_dst_rows; // ne1 * ne2 * ne3@@ -35,6 +32,7 @@ struct htp_repeat_context {static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * data) {const struct htp_repeat_context * rctx = (const struct htp_repeat_context *) data;struct htp_ops_context * octx = rctx->octx;+ const struct htp_repeat_kernel_params * kparams = rctx->kparams;const struct htp_tensor * src = octx->src[0];const struct htp_tensor * dst = octx->dst;@@ -58,29 +56,30 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * datconst uint32_t nb2 = dst->nb[2];const uint32_t nb3 = dst->nb[3];- const uint32_t nr0 = rctx->nr0;- const uint32_t nr1 = rctx->nr1;- const uint32_t nr2 = rctx->nr2;- const uint32_t nr3 = rctx->nr3;+ const uint32_t nr0 = kparams->nr0;const size_t row_bytes = ne00 * rctx->type_size;const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith;const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->total_dst_rows);+ if (row_start >= row_end) {+ return;+ }struct htp_thread_trace * tr = &octx->ctx->trace[ith];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start);for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {// Decompose flat dst row index into (i1, i2, i3)- const uint32_t i1 = dst_row % ne1;- const uint32_t i2 = (dst_row / ne1) % ne2;- const uint32_t i3 = dst_row / (ne1 * ne2);+ const uint32_t dst_row_div_ne1 = fastdiv(dst_row, &kparams->div_ne1);+ const uint32_t i1 = dst_row - dst_row_div_ne1 * ne1;+ const uint32_t i3 = fastdiv(dst_row, &kparams->div_ne12);+ const uint32_t i2 = dst_row_div_ne1 - i3 * ne2;// Map to source indices (tiling)- const uint32_t k1 = i1 % ne01;- const uint32_t k2 = i2 % ne02;- const uint32_t k3 = i3 % ne03;+ const uint32_t k1 = fastmodulo(i1, ne01, &kparams->div_ne01);+ const uint32_t k2 = fastmodulo(i2, ne02, &kparams->div_ne02);+ const uint32_t k3 = fastmodulo(i3, ne03, &kparams->div_ne03);const uint8_t * src_row = (const uint8_t *) src->data + k1 * nb01 + k2 * nb02 + k3 * nb03;uint8_t * dst_base = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;@@ -104,15 +103,6 @@ int op_repeat(struct htp_ops_context * octx) {const struct htp_tensor * src0 = octx->src[0];const struct htp_tensor * dst = octx->dst;- // Validate that dst dims are multiples of src dims- if (dst->ne[0] % src0->ne[0] != 0 ||- dst->ne[1] % src0->ne[1] != 0 ||- dst->ne[2] % src0->ne[2] != 0 ||- dst->ne[3] % src0->ne[3] != 0) {- FARF(ERROR, "repeat: dst dims must be multiples of src dims\n");- return HTP_STATUS_INVAL_PARAMS;- }-size_t type_size;switch (src0->type) {case HTP_TYPE_F32: type_size = 4; break;@@ -126,6 +116,13 @@ int op_repeat(struct htp_ops_context * octx) {return HTP_STATUS_NO_SUPPORT;}+ const struct htp_repeat_kernel_params * kparams =+ (const struct htp_repeat_kernel_params *) octx->kernel_params;++ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {+ return HTP_STATUS_INVAL_PARAMS;+ }+const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3];const size_t dst_row_size = dst->ne[0] * type_size;@@ -148,10 +145,7 @@ int op_repeat(struct htp_ops_context * octx) {struct htp_repeat_context rctx = {.octx = octx,- .nr0 = dst->ne[0] / src0->ne[0],- .nr1 = dst->ne[1] / src0->ne[1],- .nr2 = dst->ne[2] / src0->ne[2],- .nr3 = dst->ne[3] / src0->ne[3],+ .kparams = kparams,.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),.total_dst_rows = nrows,.row_start = row_start,@@ -161,7 +155,7 @@ int op_repeat(struct htp_ops_context * octx) {FARF(HIGH, "repeat: (%ux%ux%ux%u) -> (%ux%ux%ux%u) nr=(%u,%u,%u,%u)\n",src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],- rctx.nr0, rctx.nr1, rctx.nr2, rctx.nr3);+ kparams->nr0, kparams->nr1, kparams->nr2, kparams->nr3);work_queue_run(octx->ctx->work_queue, repeat_job_per_thread, &rctx, n_threads);diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.h b/ggml/src/ggml-hexagon/htp/repeat-ops.hnew file mode 100644index 0000000000..47759e037d--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/repeat-ops.h@@ -0,0 +1,27 @@+#ifndef HTP_REPEAT_OPS_H+#define HTP_REPEAT_OPS_H++#include "hex-fastdiv.h"+#include <stdint.h>++struct htp_repeat_kernel_params {+ uint32_t n_threads;+ uint32_t nr0;+ uint32_t nr1;+ uint32_t nr2;+ uint32_t nr3;++ struct fastdiv_values div_ne1;+ struct fastdiv_values div_ne12;+ struct fastdiv_values div_ne01;+ struct fastdiv_values div_ne02;+ struct fastdiv_values div_ne03;+};++#if defined(__cplusplus)+static_assert(sizeof(struct htp_repeat_kernel_params) <= 128, "htp_repeat_kernel_params is too large for kernel_params blob");+#else+_Static_assert(sizeof(struct htp_repeat_kernel_params) <= 128, "htp_repeat_kernel_params is too large for kernel_params blob");+#endif++#endif // HTP_REPEAT_OPS_Hdiff --git a/ggml/src/ggml-hexagon/htp/roll-ops.c b/ggml/src/ggml-hexagon/htp/roll-ops.cindex 9c373f56d3..77ded0fe90 100644--- a/ggml/src/ggml-hexagon/htp/roll-ops.c+++ b/ggml/src/ggml-hexagon/htp/roll-ops.c@@ -17,16 +17,15 @@#include "hex-profile.h"#include "htp-ops.h"#include "htp-tensor.h"+#include "roll-ops.h"struct htp_roll_context {struct htp_ops_context * octx;+ const struct htp_roll_kernel_params * kparams;uint32_t row_start;uint32_t nrows;uint32_t nrows_per_thread;-- struct fastdiv_values div_ne1;- struct fastdiv_values div_ne2_ne1;};static inline uint32_t htp_roll_wrap(int32_t i, uint32_t ne) {@@ -39,30 +38,32 @@ static inline uint32_t htp_roll_wrap(int32_t i, uint32_t ne) {return (uint32_t) i;}-#define htp_roll_preamble \- const struct htp_tensor * src0 = octx->src[0]; \- const struct htp_tensor * dst = octx->dst; \- \- const uint32_t ne0 = dst->ne[0]; \- const uint32_t ne1 = dst->ne[1]; \- const uint32_t ne2 = dst->ne[2]; \- const uint32_t ne3 = dst->ne[3]; \- \- const uint32_t nb01 = src0->nb[1]; \- const uint32_t nb02 = src0->nb[2]; \- const uint32_t nb03 = src0->nb[3]; \- \- const uint32_t nb1 = dst->nb[1]; \- const uint32_t nb2 = dst->nb[2]; \- const uint32_t nb3 = dst->nb[3]; \- \- const int32_t s0 = octx->op_params[0]; \- const int32_t s1 = octx->op_params[1]; \- const int32_t s2 = octx->op_params[2]; \- const int32_t s3 = octx->op_params[3]; \- \- const uint32_t i0_src0 = htp_roll_wrap(-s0, ne0); \- const uint32_t n0 = ne0 - i0_src0;+#define htp_roll_preamble \+ const struct htp_tensor * src0 = octx->src[0]; \+ const struct htp_tensor * dst = octx->dst; \+ const struct htp_roll_kernel_params * kparams = \+ (const struct htp_roll_kernel_params *) octx->kernel_params; \+ \+ const uint32_t ne0 = dst->ne[0]; \+ const uint32_t ne1 = dst->ne[1]; \+ const uint32_t ne2 = dst->ne[2]; \+ const uint32_t ne3 = dst->ne[3]; \+ \+ const uint32_t nb01 = src0->nb[1]; \+ const uint32_t nb02 = src0->nb[2]; \+ const uint32_t nb03 = src0->nb[3]; \+ \+ const uint32_t nb1 = dst->nb[1]; \+ const uint32_t nb2 = dst->nb[2]; \+ const uint32_t nb3 = dst->nb[3]; \+ \+ const int32_t s0 = octx->op_params[0]; \+ const int32_t s1 = octx->op_params[1]; \+ const int32_t s2 = octx->op_params[2]; \+ const int32_t s3 = octx->op_params[3]; \+ \+ const uint32_t i0_src0 = kparams->i0_src0; \+ const uint32_t n0 = kparams->n0;#define htp_roll_dma_preamble dma_queue * q = octx->ctx->dma[0];@@ -91,10 +92,10 @@ static inline void roll_dma_push_rows(dma_queue * q,uint32_t src_row,uint32_t nrows,uint32_t row_size,- uint32_t i0_src0) {+ uint32_t i0_src0,+ uint32_t n0) {const dma_addr_t dst_base = dst->data + (size_t) dst_row * row_size;const dma_addr_t src_base = src0->data + (size_t) src_row * row_size;- const uint32_t n0 = src0->ne[0] - i0_src0;roll_dma_push(q, dst_base, src_base + (size_t) i0_src0 * sizeof(float),row_size, row_size, n0 * sizeof(float), nrows);@@ -125,7 +126,7 @@ static int roll_dma_f32_contiguous(struct htp_ops_context * octx) {const uint32_t row_size = ne0 * sizeof(float);if (s1 == 0 && s2 == 0 && s3 == 0) {- roll_dma_push_rows(q, dst, src0, 0, 0, ne1 * ne2 * ne3, row_size, i0_src0);+ roll_dma_push_rows(q, dst, src0, 0, 0, ne1 * ne2 * ne3, row_size, i0_src0, n0);dma_queue_flush(q);return HTP_STATUS_OK;}@@ -139,9 +140,9 @@ static int roll_dma_f32_contiguous(struct htp_ops_context * octx) {const uint32_t n2_first = ne2 - i2_src0;roll_dma_push_rows(q, dst, src0, dst_row0, src_row0, n2_first * ne1,- row_size, i0_src0);+ row_size, i0_src0, n0);roll_dma_push_rows(q, dst, src0, dst_row0 + n2_first * ne1, i03 * ne2 * ne1,- i2_src0 * ne1, row_size, i0_src0);+ i2_src0 * ne1, row_size, i0_src0, n0);}dma_queue_flush(q);@@ -159,9 +160,9 @@ static int roll_dma_f32_contiguous(struct htp_ops_context * octx) {const uint32_t src_row0 = (i03 * ne2 + i02) * ne1;roll_dma_push_rows(q, dst, src0, dst_row0, src_row0 + i1_src0,- n1_first, row_size, i0_src0);+ n1_first, row_size, i0_src0, n0);roll_dma_push_rows(q, dst, src0, dst_row0 + n1_first, src_row0,- i1_src0, row_size, i0_src0);+ i1_src0, row_size, i0_src0, n0);}}@@ -214,9 +215,9 @@ static void roll_thread_f32(unsigned int nth, unsigned int ith, void * data) {htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start);for (uint32_t row = row_start; row < row_end; row++) {- const uint32_t i3 = fastdiv(row, &rctx->div_ne2_ne1);+ const uint32_t i3 = fastdiv(row, &kparams->div_ne2_ne1);const uint32_t rem = row - i3 * ne2 * ne1;- const uint32_t i2 = fastdiv(rem, &rctx->div_ne1);+ const uint32_t i2 = fastdiv(rem, &kparams->div_ne1);const uint32_t i1 = rem - i2 * ne1;const uint32_t i01 = htp_roll_wrap((int32_t) i1 - s1, ne1);@@ -245,23 +246,14 @@ static void roll_thread_f32(unsigned int nth, unsigned int ith, void * data) {int execute_op_roll_f32(struct htp_ops_context * octx) {htp_roll_preamble;- if (src0->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) {- FARF(ERROR, "roll: unsupported type %u -> %u\n", src0->type, dst->type);- return HTP_STATUS_NO_SUPPORT;- }-- if (src0->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) {- FARF(ERROR, "roll: unsupported nb0 %u -> %u\n", src0->nb[0], dst->nb[0]);- return HTP_STATUS_NO_SUPPORT;+ if (kparams->kernel_type == HTP_ROLL_KERNEL_DMA_CONTIGUOUS) {+ return roll_dma_f32_contiguous(octx);}-- if (src0->ne[0] != ne0 || src0->ne[1] != ne1 ||- src0->ne[2] != ne2 || src0->ne[3] != ne3) {- FARF(ERROR, "roll: shape mismatch\n");- return HTP_STATUS_INVAL_PARAMS;+ if (kparams->kernel_type == HTP_ROLL_KERNEL_DMA_STRIDED) {+ return roll_dma_f32_strided(octx);}- const uint32_t total_rows = ne1 * ne2 * ne3;+ const uint32_t total_rows = kparams->total_rows;const size_t dst_row_size = ne0 * sizeof(float);uint32_t row_start = 0;@@ -279,25 +271,21 @@ int execute_op_roll_f32(struct htp_ops_context * octx) {return HTP_STATUS_OK;}- if (octx->ctx->mdev.count <= 1) {- if (htp_tensor_is_contiguous(src0, sizeof(float)) && htp_tensor_is_contiguous(dst, sizeof(float))) {- return roll_dma_f32_contiguous(octx);- }- return roll_dma_f32_strided(octx);- }-if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {return HTP_STATUS_NO_SUPPORT;}+ if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {+ return HTP_STATUS_INVAL_PARAMS;+ }+const uint32_t n_threads = octx->n_threads;struct htp_roll_context rctx = {.octx = octx,+ .kparams = kparams,.row_start = row_start,.nrows = nrows,.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),- .div_ne1 = init_fastdiv_values(dst->ne[1]),- .div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]),};work_queue_run(octx->ctx->work_queue, roll_thread_f32, &rctx, n_threads);diff --git a/ggml/src/ggml-hexagon/htp/roll-ops.h b/ggml/src/ggml-hexagon/htp/roll-ops.hnew file mode 100644index 0000000000..2646aaa4c9--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/roll-ops.h@@ -0,0 +1,30 @@+#ifndef HTP_ROLL_OPS_H+#define HTP_ROLL_OPS_H++#include "hex-fastdiv.h"+#include <stdint.h>++enum htp_roll_kernel_type {+ HTP_ROLL_KERNEL_DMA_CONTIGUOUS = 1,+ HTP_ROLL_KERNEL_DMA_STRIDED = 2,+ HTP_ROLL_KERNEL_HVX = 3,+};++struct htp_roll_kernel_params {+ uint8_t kernel_type;+ uint32_t n_threads;+ uint32_t total_rows;+ uint32_t i0_src0;+ uint32_t n0;++ struct fastdiv_values div_ne1;+ struct fastdiv_values div_ne2_ne1;+};++#if defined(__cplusplus)+static_assert(sizeof(struct htp_roll_kernel_params) <= 128, "htp_roll_kernel_params is too large for kernel_params blob");+#else+_Static_assert(sizeof(struct htp_roll_kernel_params) <= 128, "htp_roll_kernel_params is too large for kernel_params blob");+#endif++#endif // HTP_ROLL_OPS_Hdiff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.cindex f6b4d383ce..8c55ac3af4 100644--- a/ggml/src/ggml-hexagon/htp/rope-ops.c+++ b/ggml/src/ggml-hexagon/htp/rope-ops.c@@ -700,15 +700,6 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {const struct htp_tensor * src2 = octx->src[2];const struct htp_tensor * dst = octx->dst;- switch (octx->op) {- case HTP_OP_ROPE:- break;-- default:- FARF(ERROR, "Unsupported Op %u\n", octx->op);- return HTP_STATUS_NO_SUPPORT;- }-const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params;if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {return HTP_STATUS_INVAL_PARAMS;@@ -746,20 +737,23 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {const size_t dst_row_size = dst->ne[0] * sizeof(float);const size_t dst_row_stride = dst->nb[1];+ struct htp_rope_vtcm_layout layout;+ htp_rope_vtcm_layout_build(&layout, src0->ne[0], kparams->n_threads, src2 ? src2->ne[0] : 0);+struct htp_rope_context rctx;memset(&rctx, 0, sizeof(struct htp_rope_context));rctx.octx = octx;- rctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base;- rctx.spad_per_thread = kparams->spad_per_thread;- rctx.theta_cache_offset = kparams->theta_cache_offset;+ rctx.vtcm_base = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_spad);+ rctx.spad_per_thread = layout.bytes_per_thread;+ rctx.theta_cache_offset = layout.theta_cache_size_aligned;if (src2) {dma_queue * dma_q = octx->ctx->dma[0];const size_t ff_size = src2->ne[0] * sizeof(float);- float * vtcm_freq_factors = (float *) (rctx.vtcm_base + kparams->freq_factors_offset);+ float * vtcm_freq_factors = VTCM_LAYOUT_PTR(float, octx->ctx->vtcm_base, layout.off_freq_factors);dma_queue_push(dma_q, dma_make_data(vtcm_freq_factors, src2->data),- kparams->freq_factors_size, 0, ff_size, 1);+ layout.freq_factors_size_aligned, 0, ff_size, 1);dma_queue_pop(dma_q);rctx.freq_factors = vtcm_freq_factors;}@@ -791,7 +785,7 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {rctx.src0_row_stride = src0_row_stride;rctx.dst_row_size = dst_row_size;rctx.dst_row_stride = dst_row_stride;- rctx.src0_row_size_aligned = kparams->src0_row_size_aligned;+ rctx.src0_row_size_aligned = layout.src0_row_size_aligned;rctx.src0_nrows = nrows;rctx.nrows = nrows;diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.h b/ggml/src/ggml-hexagon/htp/rope-ops.hindex ee055ccbc1..0c28ef826d 100644--- a/ggml/src/ggml-hexagon/htp/rope-ops.h+++ b/ggml/src/ggml-hexagon/htp/rope-ops.h@@ -3,6 +3,7 @@#include "hex-common.h"#include "hex-fastdiv.h"+#include "htp-vtcm.h"#define HTP_ROPE_SPAD_BLOCK 8#define HTP_ROPE_SPAD_NSLOTS 4@@ -10,14 +11,7 @@struct htp_rope_kernel_params {uint32_t n_threads;- uint32_t src0_nrows;- uint32_t src0_nrows_per_thread;uint32_t vtcm_size;- uint32_t spad_per_thread;- uint32_t theta_cache_offset;- uint32_t src0_row_size_aligned;- uint32_t freq_factors_offset;- uint32_t freq_factors_size;struct fastdiv_values div_ne2_ne1;struct fastdiv_values div_ne1;@@ -31,6 +25,8 @@ _Static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_pastruct htp_rope_vtcm_layout {size_t total_bytes;+ size_t off_spad;+ size_t off_freq_factors;size_t bytes_per_thread;size_t theta_cache_size_aligned;size_t src0_row_size_aligned;@@ -38,7 +34,7 @@ struct htp_rope_vtcm_layout {};static inline void htp_rope_vtcm_layout_build(- struct htp_rope_vtcm_layout * layout,+ struct htp_rope_vtcm_layout * L,uint32_t ne00,uint32_t n_threads,uint32_t n_freq_factors@@ -48,11 +44,15 @@ static inline void htp_rope_vtcm_layout_build(const size_t theta_cache_size_aligned = hex_round_up((uint32_t) src0_row_size, 256);const size_t freq_factors_size_aligned = hex_round_up(n_freq_factors * sizeof(float), 256);- layout->src0_row_size_aligned = src0_row_size_aligned;- layout->theta_cache_size_aligned = theta_cache_size_aligned;- layout->freq_factors_size_aligned = freq_factors_size_aligned;- layout->bytes_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned;- layout->total_bytes = layout->bytes_per_thread * n_threads + freq_factors_size_aligned;+ L->src0_row_size_aligned = src0_row_size_aligned;+ L->theta_cache_size_aligned = theta_cache_size_aligned;+ L->freq_factors_size_aligned = freq_factors_size_aligned;+ L->bytes_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned;++ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_spad, L->bytes_per_thread * n_threads);+ VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_freq_factors, freq_factors_size_aligned, n_freq_factors > 0);+ L->total_bytes = off;}static inline uint8_t * rope_spad_slot(uint8_t * base, uint32_t slot, size_t row_size_aligned) {diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.c b/ggml/src/ggml-hexagon/htp/set-rows-ops.cindex 1d72538f17..191cf0c6af 100644--- a/ggml/src/ggml-hexagon/htp/set-rows-ops.c+++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.c@@ -79,8 +79,10 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigconst uint32_t ir1 = MIN(ir0 + dr, srctx->task_start + srctx->tasks); \dma_queue * dma_q = octx->ctx->dma[ith]; \const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \- uint8_t * vtcm_src0 = srctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \- uint8_t * vtcm_dst = srctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \+ uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, srctx->vtcm_base, vtcm_layout->off_src0) + \+ ith * vtcm_layout->src0_bytes_per_thread; \+ uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, srctx->vtcm_base, vtcm_layout->off_dst) + \+ ith * vtcm_layout->dst_bytes_per_thread; \const uint32_t src0_row_size = ne00 * sizeof(float); \const uint32_t dst_row_size = htp_tensor_get_row_size(octx->dst->type, ne00); \const uint32_t nrows_per_thread = ir1 - ir0; \@@ -188,14 +190,6 @@ int op_set_rows(struct htp_ops_context * octx) {const struct htp_set_rows_kernel_params * kparams = (const struct htp_set_rows_kernel_params *)octx->kernel_params;set_rows_preamble;- if (octx->src[0]->type != HTP_TYPE_F32) {- return HTP_STATUS_NO_SUPPORT;- }-- if (octx->dst->type != HTP_TYPE_F32 && octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_Q8_0) {- return HTP_STATUS_NO_SUPPORT;- }-if (htp_tensor_is_extended(octx->src[1])) {return HTP_STATUS_NO_SUPPORT;}diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.h b/ggml/src/ggml-hexagon/htp/set-rows-ops.hindex 5e98d2cb55..b9cb6cc90a 100644--- a/ggml/src/ggml-hexagon/htp/set-rows-ops.h+++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.h@@ -2,18 +2,13 @@#define HTP_SET_ROWS_OPS_H#include "hex-fastdiv.h"+#include "htp-tensor.h"+#include "htp-vtcm.h"struct htp_set_rows_kernel_params {- int32_t n_threads;- int32_t total_tasks;- int32_t tasks_per_thread;- int32_t vtcm_size;-- // Fastdiv helpers- struct fastdiv_values div_ne11;- struct fastdiv_values div_ne12;- struct fastdiv_values div_tasks_per_thread;- struct fastdiv_values div_ne02;+ uint32_t n_threads;+ uint32_t total_tasks;+ uint32_t vtcm_size;};struct htp_set_rows_vtcm_layout {@@ -29,40 +24,27 @@ struct htp_set_rows_vtcm_layout {};static inline void htp_set_rows_vtcm_layout_build(- struct htp_set_rows_vtcm_layout * vtcm_layout,+ struct htp_set_rows_vtcm_layout * L,int dst_type,uint32_t ne00,uint32_t n_threads) {- size_t src0_row_size = ne00 * 4;- size_t dst_row_size = 0;- switch (dst_type) {- case 0: // HTP_TYPE_F32- dst_row_size = ne00 * 4;- break;- case 1: // HTP_TYPE_F16- dst_row_size = ne00 * 2;- break;- case 8: // HTP_TYPE_Q8_0- dst_row_size = (ne00 / 32) * 34;- break;- default:- dst_row_size = 0;- break;- }+ size_t src0_row_size = ne00 * sizeof(float);+ size_t dst_row_size = htp_tensor_get_row_size(dst_type, ne00);size_t src0_row_size_aligned = (src0_row_size + 255) & ~255;size_t dst_row_size_aligned = (dst_row_size + 255) & ~255;- vtcm_layout->src0_spad_half_size = src0_row_size_aligned;- vtcm_layout->dst_spad_half_size = dst_row_size_aligned;+ L->src0_spad_half_size = src0_row_size_aligned;+ L->dst_spad_half_size = dst_row_size_aligned;- vtcm_layout->src0_bytes_per_thread = src0_row_size_aligned * 2;- vtcm_layout->dst_bytes_per_thread = dst_row_size_aligned * 2;+ L->src0_bytes_per_thread = src0_row_size_aligned * 2;+ L->dst_bytes_per_thread = dst_row_size_aligned * 2;- vtcm_layout->off_src0 = 0;- vtcm_layout->off_dst = vtcm_layout->off_src0 + vtcm_layout->src0_bytes_per_thread * n_threads;- vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads;+ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);+ L->total_bytes = off;}#if defined(__cplusplus)diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.c b/ggml/src/ggml-hexagon/htp/softmax-ops.cindex 48be5d725a..d65c3fae33 100644--- a/ggml/src/ggml-hexagon/htp/softmax-ops.c+++ b/ggml/src/ggml-hexagon/htp/softmax-ops.c@@ -496,7 +496,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) {return HTP_STATUS_INVAL_PARAMS;}- if (kparams->kernel_id >= HTP_SOFTMAX_KERNEL_COUNT) {+ if (kparams->kernel_type >= HTP_SOFTMAX_KERNEL_COUNT) {return HTP_STATUS_INVAL_PARAMS;}@@ -532,34 +532,38 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) {const uint32_t n_threads = octx->n_threads;uint8_t * const vtcm_base = (uint8_t *) octx->ctx->vtcm_base;- const uint32_t off_src0 = 0;- const uint32_t off_dst = off_src0 + kparams->vtcm_src0_size_per_thread * kparams->n_threads;- const uint32_t off_src1 = off_dst + kparams->vtcm_dst_size_per_thread * kparams->n_threads;+ struct htp_softmax_vtcm_layout layout;+ htp_softmax_vtcm_layout_build(&layout,+ src0->ne[0],+ kparams->use_src1 ? octx->src[1]->ne[0] : 0,+ kparams->use_src1 != 0,+ kparams->use_f16 != 0,+ kparams->n_threads);struct htp_softmax_context smctx = {.octx = octx,.kparams = kparams,- .compute = (void *) softmax_kernels[kparams->kernel_id],+ .compute = (void *) softmax_kernels[kparams->kernel_type],.data_src0 = src0->data,.data_src1 = kparams->use_src1 ? octx->src[1]->data : 0,.data_dst = dst->data,- .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, off_src0),- .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, off_dst),- .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, vtcm_base, off_src1, kparams->use_src1),+ .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_src0),+ .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_dst),+ .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, vtcm_base, layout.off_src1, kparams->use_src1),- .vtcm_src0_size_per_thread = kparams->vtcm_src0_size_per_thread,- .vtcm_src1_size_per_thread = kparams->vtcm_src1_size_per_thread,- .vtcm_dst_size_per_thread = kparams->vtcm_dst_size_per_thread,+ .vtcm_src0_size_per_thread = (uint32_t) layout.src0_bytes_per_thread,+ .vtcm_src1_size_per_thread = (uint32_t) layout.src1_bytes_per_thread,+ .vtcm_dst_size_per_thread = (uint32_t) layout.dst_bytes_per_thread,- .src0_spad_half_size = kparams->src0_spad_half_size,- .src1_spad_half_size = kparams->src1_spad_half_size,- .dst_spad_half_size = kparams->dst_spad_half_size,+ .src0_spad_half_size = (uint32_t) layout.src0_spad_half_size,+ .src1_spad_half_size = (uint32_t) layout.src1_spad_half_size,+ .dst_spad_half_size = (uint32_t) layout.dst_spad_half_size,- .src0_row_size_aligned = kparams->src0_row_size_aligned,- .src1_row_size_aligned = kparams->src1_row_size_aligned,- .dst_row_size_aligned = kparams->dst_row_size_aligned,+ .src0_row_size_aligned = (uint32_t) layout.src0_spad_half_size,+ .src1_row_size_aligned = (uint32_t) layout.src1_spad_half_size,+ .dst_row_size_aligned = (uint32_t) layout.dst_spad_half_size,.use_f16 = kparams->use_f16 != 0,diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.h b/ggml/src/ggml-hexagon/htp/softmax-ops.hindex 8d976adb83..ef6567306d 100644--- a/ggml/src/ggml-hexagon/htp/softmax-ops.h+++ b/ggml/src/ggml-hexagon/htp/softmax-ops.h@@ -7,8 +7,9 @@#include <math.h>#include "hex-fastdiv.h"#include "hex-common.h"+#include "htp-vtcm.h"-enum htp_softmax_kernel_id {+enum htp_softmax_kernel_type {HTP_SOFTMAX_KERNEL_NOMASK = 0,HTP_SOFTMAX_KERNEL_MASK_F32,HTP_SOFTMAX_KERNEL_MASK_F16,@@ -16,28 +17,14 @@ enum htp_softmax_kernel_id {};struct htp_softmax_kernel_params {+ uint8_t kernel_type;uint32_t n_threads;- uint32_t src0_nrows;- uint32_t src0_nrows_per_thread;uint32_t vtcm_size;- uint32_t vtcm_src0_size_per_thread;- uint32_t vtcm_src1_size_per_thread;- uint32_t vtcm_dst_size_per_thread;-- uint32_t src0_row_size_aligned;- uint32_t src1_row_size_aligned;- uint32_t dst_row_size_aligned;-- uint32_t src0_spad_half_size;- uint32_t src1_spad_half_size;- uint32_t dst_spad_half_size;-uint32_t n_head;uint32_t n_head_log2;uint32_t use_src1;uint32_t use_f16;- uint32_t kernel_id;float scale;float max_bias;@@ -72,7 +59,7 @@ struct htp_softmax_vtcm_layout {};static inline void htp_softmax_vtcm_layout_build(- struct htp_softmax_vtcm_layout * layout,+ struct htp_softmax_vtcm_layout * L,uint32_t ne00,uint32_t ne10,bool use_src1,@@ -87,20 +74,21 @@ static inline void htp_softmax_vtcm_layout_build(size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128);size_t src1_row_size_aligned = use_src1 ? hex_round_up(src1_row_size, 128) : 0;- layout->src0_spad_half_size = src0_row_size_aligned;- layout->dst_spad_half_size = dst_row_size_aligned;- layout->src1_spad_half_size = src1_row_size_aligned;+ L->src0_spad_half_size = src0_row_size_aligned;+ L->dst_spad_half_size = dst_row_size_aligned;+ L->src1_spad_half_size = src1_row_size_aligned;// Double buffering: 2 half-buffers per thread- layout->src0_bytes_per_thread = src0_row_size_aligned * 2;- layout->dst_bytes_per_thread = dst_row_size_aligned * 2;- layout->src1_bytes_per_thread = src1_row_size_aligned * 2;+ L->src0_bytes_per_thread = src0_row_size_aligned * 2;+ L->dst_bytes_per_thread = dst_row_size_aligned * 2;+ L->src1_bytes_per_thread = src1_row_size_aligned * 2;- layout->off_src0 = 0;- layout->off_dst = layout->off_src0 + layout->src0_bytes_per_thread * n_threads;- layout->off_src1 = layout->off_dst + layout->dst_bytes_per_thread * n_threads;+ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);+ VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes_per_thread * n_threads, use_src1);- layout->total_bytes = layout->off_src1 + layout->src1_bytes_per_thread * n_threads;+ L->total_bytes = off;}#endif // HTP_SOFTMAX_OPS_Hdiff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c b/ggml/src/ggml-hexagon/htp/solve-tri-ops.cindex 182982fcda..628ca65e84 100644--- a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c+++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c@@ -13,14 +13,16 @@#include "htp-tensor.h"#include "hvx-types.h"#include "hvx-utils.h"+#include "solve-tri-ops.h"struct htp_solve_tri_context {- struct htp_ops_context * octx;- uint32_t jobs_per_thread;- uint32_t total_jobs;- uint32_t job_start;- uint32_t k_chunks;- uint32_t col_block;+ struct htp_ops_context * octx;+ const struct htp_solve_tri_kernel_params * kparams;+ uint32_t jobs_per_thread;+ uint32_t total_jobs;+ uint32_t job_start;+ uint32_t k_chunks;+ uint32_t col_block;};static inline void solve_tri_row_scalar(const float * A_row,@@ -80,6 +82,7 @@ static inline void solve_tri_row_hvx(const float * A_row,static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void * data) {struct htp_solve_tri_context * sctx = (struct htp_solve_tri_context *) data;struct htp_ops_context * octx = sctx->octx;+ const struct htp_solve_tri_kernel_params * kparams = sctx->kparams;const struct htp_tensor * src0 = octx->src[0]; // Aconst struct htp_tensor * src1 = octx->src[1]; // B@@ -91,7 +94,7 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, voidconst uint32_t ne02 = src0->ne[2];const uint32_t col_block = VLEN_FP32;- const uint32_t k_full = (k / col_block) * col_block;+ const uint32_t k_full = k & ~(col_block - 1);const uint32_t start_batch = sctx->job_start + sctx->jobs_per_thread * ith;const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs);@@ -100,7 +103,7 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, voidhtp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_batch);for (uint32_t batch = start_batch; batch < end_batch; ++batch) {- const uint32_t i03 = batch / ne02;+ const uint32_t i03 = fastdiv(batch, &kparams->div_ne02);const uint32_t i02 = batch - i03 * ne02;const float * A_batch =@@ -141,6 +144,7 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, voidstatic void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void * data) {struct htp_solve_tri_context * sctx = (struct htp_solve_tri_context *) data;struct htp_ops_context * octx = sctx->octx;+ const struct htp_solve_tri_kernel_params * kparams = sctx->kparams;const struct htp_tensor * src0 = octx->src[0]; // Aconst struct htp_tensor * src1 = octx->src[1]; // B@@ -158,10 +162,10 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, voidhtp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_job);for (uint32_t job = start_job; job < end_job; ++job) {- const uint32_t batch = job / sctx->k_chunks;+ const uint32_t batch = fastdiv(job, &kparams->div_k_chunks);const uint32_t chunk = job - batch * sctx->k_chunks;- const uint32_t i03 = batch / ne02;+ const uint32_t i03 = fastdiv(batch, &kparams->div_ne02);const uint32_t i02 = batch - i03 * ne02;const float * A_batch =@@ -198,36 +202,17 @@ int op_solve_tri(struct htp_ops_context * octx) {const struct htp_tensor * src0 = octx->src[0]; // Aconst struct htp_tensor * src1 = octx->src[1]; // Bconst struct htp_tensor * dst = octx->dst; // X-- if (src0->type != HTP_TYPE_F32 || src1->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) {- return HTP_STATUS_NO_SUPPORT;- }-- // left=true, lower=true, uni=false only- if (src0->ne[0] != src0->ne[1]) {- return HTP_STATUS_INVAL_PARAMS;- }- if (src0->ne[1] != src1->ne[1]) {- return HTP_STATUS_INVAL_PARAMS;- }- if (src0->ne[2] != src1->ne[2] || src0->ne[3] != src1->ne[3]) {- return HTP_STATUS_INVAL_PARAMS;- }- if (dst->ne[0] != src1->ne[0] || dst->ne[1] != src1->ne[1] || dst->ne[2] != src1->ne[2] ||- dst->ne[3] != src1->ne[3]) {- return HTP_STATUS_INVAL_PARAMS;- }+ const struct htp_solve_tri_kernel_params * kparams =+ (const struct htp_solve_tri_kernel_params *) octx->kernel_params;if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(src1) || htp_tensor_is_extended(dst)) {return HTP_STATUS_NO_SUPPORT;}- const uint32_t k = src1->ne[0];-- const uint32_t col_block = VLEN_FP32;- const uint32_t k_chunks = (k + col_block - 1) / col_block;- const uint32_t total_batches = src0->ne[2] * src0->ne[3];- const bool batched = total_batches >= (uint32_t) octx->n_threads;+ const uint32_t col_block = kparams->col_block;+ const uint32_t k_chunks = kparams->k_chunks;+ const uint32_t total_batches = kparams->total_batches;+ const bool batched = (kparams->batched != 0);FARF(HIGH, "solve-tri: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : batched %d\n",src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],@@ -239,9 +224,7 @@ int op_solve_tri(struct htp_ops_context * octx) {uint32_t njobs = total_batches;if (octx->ctx->mdev.count > 1) {- const uint32_t batch_size = dst->nb[2];- const uint32_t batches_per_chunk = (batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(batch_size, HEX_L2_LINE_SIZE)) : 1;- const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, htp_tensor_mdev_data_aligned(dst) ? batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);+ const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, htp_tensor_mdev_data_aligned(dst) ? kparams->batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);job_start = range.start;njobs = range.count;}@@ -250,11 +233,15 @@ int op_solve_tri(struct htp_ops_context * octx) {return HTP_STATUS_OK;}- // Batch-level parallelism+ if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, njobs))) {+ return HTP_STATUS_INVAL_PARAMS;+ }+const uint32_t n_threads = octx->n_threads;struct htp_solve_tri_context sctx = {.octx = octx,+ .kparams = kparams,.jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div),.total_jobs = njobs,.job_start = job_start,@@ -264,8 +251,7 @@ int op_solve_tri(struct htp_ops_context * octx) {work_queue_run(octx->ctx->work_queue, solve_tri_batch_thread_f32, &sctx, n_threads);} else {- // Chunk-level parallelism- const uint32_t total_jobs = total_batches * k_chunks;+ const uint32_t total_jobs = kparams->total_jobs;uint32_t job_start = 0;uint32_t njobs = total_jobs;@@ -281,10 +267,15 @@ int op_solve_tri(struct htp_ops_context * octx) {return HTP_STATUS_OK;}+ if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, njobs))) {+ return HTP_STATUS_INVAL_PARAMS;+ }+const uint32_t n_threads = octx->n_threads;struct htp_solve_tri_context sctx = {.octx = octx,+ .kparams = kparams,.jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div),.total_jobs = njobs,.job_start = job_start,diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.h b/ggml/src/ggml-hexagon/htp/solve-tri-ops.hnew file mode 100644index 0000000000..327415814b--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.h@@ -0,0 +1,26 @@+#ifndef HTP_SOLVE_TRI_OPS_H+#define HTP_SOLVE_TRI_OPS_H++#include "hex-fastdiv.h"+#include <stdint.h>++struct htp_solve_tri_kernel_params {+ uint32_t n_threads;+ uint32_t total_batches;+ uint32_t total_jobs;+ uint32_t k_chunks;+ uint32_t col_block;+ uint32_t batched;+ uint32_t batches_per_chunk;++ struct fastdiv_values div_ne02;+ struct fastdiv_values div_k_chunks;+};++#if defined(__cplusplus)+static_assert(sizeof(struct htp_solve_tri_kernel_params) <= 128, "htp_solve_tri_kernel_params is too large for kernel_params blob");+#else+_Static_assert(sizeof(struct htp_solve_tri_kernel_params) <= 128, "htp_solve_tri_kernel_params is too large for kernel_params blob");+#endif++#endif // HTP_SOLVE_TRI_OPS_Hdiff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.c b/ggml/src/ggml-hexagon/htp/ssm-conv.cindex 0d28c89904..dae339e4dd 100644--- a/ggml/src/ggml-hexagon/htp/ssm-conv.c+++ b/ggml/src/ggml-hexagon/htp/ssm-conv.c@@ -135,6 +135,13 @@ static inline void hvx_ssm_conv_unpack_to_T(const float * raw, float * T, uint32}}+static __attribute__((noinline)) void hvx_ssm_conv_silu_vtcm(float * restrict buf, uint32_t n_vecs) {+ HVX_Vector * restrict vbuf = (HVX_Vector *) buf;+ for (uint32_t i = 0; i < n_vecs; ++i) {+ vbuf[i] = hvx_vec_silu_f32(vbuf[i]);+ }+}+// Decode dot product specialization for d_conv == 4: multiply in the raw channel-major layout,// then deinterleave the products so each vector holds one tap of 32 channels, and sum.// Keeps both operands in DMA layout - no transpose, no scratch.@@ -189,9 +196,10 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, voiddma_queue * dma_q = octx->ctx->dma[ith];- const uint32_t d_conv = kparams->d_conv;- const uint32_t d_inner = kparams->d_inner;- const uint32_t n_s = kparams->n_s;+ const uint32_t d_conv = kparams->d_conv;+ const uint32_t d_inner = kparams->d_inner;+ const uint32_t n_s = kparams->n_s;+ const bool apply_silu = kparams->silu != 0;const uint32_t dr = scctx->nrows_per_thread;const uint32_t ir0 = scctx->row_start + dr * ith;@@ -232,7 +240,7 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void// raw_dot keeps both operands in the DMA layout, so src1 needs no prep passconst bool raw_dot = (d_conv == 4) && (d_inner_per_thread % VLEN_FP32 == 0);- const uint32_t n_tiles = (d_inner_per_thread + d_inner_tile - 1) / d_inner_tile;+ const uint32_t n_tiles = fastdiv(d_inner_per_thread + d_inner_tile - 1, &kparams->div_d_inner_tile);const uint32_t n_chunks = n_s * n_tiles;const size_t row_bytes = d_conv * sizeof(float);@@ -295,21 +303,18 @@ static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, voidhvx_ssm_conv_unpack_to_T(src0_tile_raw[c & 1], src0_T, tile_n, tile_stride, d_conv);for (uint32_t cb = 0; cb < tile_n; cb += VLEN_FP32) {- const uint32_t cb_n = MIN(VLEN_FP32, tile_n - cb);HVX_Vector acc = hvx_vec_splat_f32(0.0f);for (uint32_t j = 0; j < d_conv; ++j) {HVX_Vector x = *(const HVX_Vector *)(src0_T + j * tile_stride + cb);HVX_Vector w = *(const HVX_Vector *)(src1_T + j * d_inner_stride + tile_off + cb);acc = Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(x, w));}- HVX_Vector y = Q6_Vsf_equals_Vqf32(acc);- if (cb_n == VLEN_FP32) {- *(HVX_Vector *)(out + cb) = y;- } else {- hvx_vec_store_u(out + cb, cb_n * sizeof(float), y);- }+ *(HVX_Vector *)(out + cb) = Q6_Vsf_equals_Vqf32(acc);}}+ if (apply_silu) {+ hvx_ssm_conv_silu_vtcm(out, hex_round_up(tile_n, VLEN_FP32) / VLEN_FP32);+ }htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i3);const dma_addr_t dst_ddr = dst->data + i3 * dst_stride_seq_bytes + (ir0 + tile_off) * sizeof(float);@@ -352,11 +357,12 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, voiddma_queue * dma_q = octx->ctx->dma[ith];- const uint32_t d_conv = kparams->d_conv;- const uint32_t d_inner = kparams->d_inner;- const uint32_t n_t = kparams->n_t;- const uint32_t n_s = kparams->n_s;- const uint32_t ncs = src0->ne[0];+ const uint32_t d_conv = kparams->d_conv;+ const uint32_t d_inner = kparams->d_inner;+ const uint32_t n_t = kparams->n_t;+ const uint32_t n_s = kparams->n_s;+ const uint32_t ncs = src0->ne[0];+ const bool apply_silu = kparams->silu != 0;const uint32_t dr = scctx->nrows_per_thread;const uint32_t ir0 = scctx->row_start + dr * ith;@@ -397,7 +403,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, voidstruct htp_thread_trace * tr = &octx->ctx->trace[ith];- const uint32_t n_tiles = (d_inner_per_thread + d_inner_tile - 1) / d_inner_tile;+ const uint32_t n_tiles = fastdiv(d_inner_per_thread + d_inner_tile - 1, &kparams->div_d_inner_tile);const uint32_t n_chunks = n_s * n_tiles;const size_t row_bytes = ncs * sizeof(float);@@ -513,10 +519,14 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, voidhtp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off);}+ if (apply_silu) {+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off);+ hvx_ssm_conv_silu_vtcm(out, n_t * (dst_tile_stride / C_TILE));+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off);+ }+// Writeback dst_tile VTCM -> DDR via 2D DMA- dma_queue_push(dma_q,- dma_make_data(dst->data + i3 * dst_stride_seq_bytes + (ir0 + tile_off) * sizeof(float),- (uint8_t *) out),+ dma_queue_push(dma_q, dma_make_data(dst->data + i3 * dst_stride_seq_bytes + (ir0 + tile_off) * sizeof(float), (uint8_t *) out),dst_stride_token_bytes, dst_tile_stride * sizeof(float), tile_n * sizeof(float), n_t);if (c + 2 < n_chunks) {@@ -547,13 +557,8 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) {const struct htp_tensor * src1 = octx->src[1];const struct htp_tensor * dst = octx->dst;- if (src0->type != HTP_TYPE_F32 || src1->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) {- return HTP_STATUS_NO_SUPPORT;- }-const struct htp_ssm_conv_kernel_params * kparams = (const struct htp_ssm_conv_kernel_params *) octx->kernel_params;-if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {return HTP_STATUS_INVAL_PARAMS;}@@ -582,26 +587,30 @@ int op_ssm_conv_f32(struct htp_ops_context * octx) {return HTP_STATUS_VTCM_TOO_SMALL;}- const uint32_t n_threads = octx->n_threads;+ const uint32_t n_threads = octx->n_threads;+ const uint32_t raw_rpt = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);+ const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, VLEN_FP32);++ struct htp_ssm_conv_vtcm_layout L;+ htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_thread, kparams->d_conv, kparams->n_t, kparams->d_inner_tile, n_threads);- octx->src0_spad.size_per_thread = kparams->vtcm_src0_size_per_thread;- octx->src1_spad.size_per_thread = kparams->vtcm_src1_size_per_thread;- octx->dst_spad.size_per_thread = kparams->vtcm_dst_size_per_thread;+ uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;- octx->src0_spad.size = kparams->vtcm_src0_size;- octx->src1_spad.size = kparams->vtcm_src1_size;- octx->dst_spad.size = kparams->vtcm_dst_size;+ octx->src0_spad.size_per_thread = L.src0_bytes_per_thread;+ octx->src1_spad.size_per_thread = L.src1_bytes_per_thread;+ octx->dst_spad.size_per_thread = L.dst_bytes_per_thread;- octx->src0_spad.data = octx->ctx->vtcm_base;- octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;- octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size;+ octx->src0_spad.size = L.src0_bytes;+ octx->src1_spad.size = L.src1_bytes;+ octx->dst_spad.size = L.dst_bytes;++ octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);+ octx->src1_spad.data = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);+ octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);octx->src0_spad.src = NULL;octx->src1_spad.src = NULL;octx->dst_spad.src = NULL;- const uint32_t raw_rpt = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);- const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, VLEN_FP32);-struct htp_ssm_conv_context scctx = {.octx = octx,.kparams = kparams,diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.h b/ggml/src/ggml-hexagon/htp/ssm-conv.hindex a071d74b04..43b2575a2b 100644--- a/ggml/src/ggml-hexagon/htp/ssm-conv.h+++ b/ggml/src/ggml-hexagon/htp/ssm-conv.h@@ -3,8 +3,10 @@#include <stdint.h>+#include "hex-common.h"#include "hex-fastdiv.h"#include "htp-ops.h"+#include "htp-vtcm.h"struct htp_ssm_conv_kernel_params {uint32_t n_threads;@@ -14,16 +16,62 @@ struct htp_ssm_conv_kernel_params {uint32_t n_s;uint32_t d_inner_tile;- uint32_t vtcm_src0_size_per_thread;- uint32_t vtcm_src1_size_per_thread;- uint32_t vtcm_dst_size_per_thread;-- uint32_t vtcm_src0_size;- uint32_t vtcm_src1_size;- uint32_t vtcm_dst_size;uint32_t vtcm_size;+ uint32_t silu;++ struct fastdiv_values div_d_inner_tile;};+struct htp_ssm_conv_vtcm_layout {+ size_t total_bytes;+ size_t off_src0;+ size_t off_src1;+ size_t off_dst;+ size_t src0_bytes_per_thread;+ size_t src1_bytes_per_thread;+ size_t dst_bytes_per_thread;+ size_t src0_bytes;+ size_t src1_bytes;+ size_t dst_bytes;+};++static inline void htp_ssm_conv_vtcm_layout_build(+ struct htp_ssm_conv_vtcm_layout * L,+ uint32_t d_inner_per_thread,+ uint32_t d_conv,+ uint32_t n_t,+ uint32_t d_inner_tile,+ uint32_t n_threads+) {+ const size_t src1_raw_bytes = hex_round_up((size_t) d_inner_per_thread * d_conv * sizeof(float), 128) + 128;+ const size_t src1_T_bytes = hex_round_up((size_t) d_conv * d_inner_per_thread * sizeof(float), 128);++ L->src1_bytes_per_thread = src1_raw_bytes + src1_T_bytes;++ if (n_t == 1) {+ const size_t src0_tile_raw_bytes = hex_round_up((size_t) d_inner_tile * d_conv * sizeof(float), 128);+ const size_t src0_T_bytes = hex_round_up((size_t) d_conv * d_inner_tile * sizeof(float), 128);+ L->src0_bytes_per_thread = 2 * src0_tile_raw_bytes + src0_T_bytes;+ L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * sizeof(float), 128);+ } else {+ const uint32_t ncs = d_conv - 1 + n_t;+ const size_t src0_block_T = hex_round_up((size_t) ncs * 32 * sizeof(float), 128);+ const size_t src0_tile_raw = hex_round_up((size_t) d_inner_tile * ncs * sizeof(float), 128);+ L->src0_bytes_per_thread = 2 * src0_tile_raw + src0_block_T;+ L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * n_t * sizeof(float), 128);+ }++ L->src0_bytes = L->src0_bytes_per_thread * n_threads;+ L->src1_bytes = L->src1_bytes_per_thread * n_threads;+ L->dst_bytes = L->dst_bytes_per_thread * n_threads;++ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes);+ VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes);+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes);+ L->total_bytes = off;+}+#if defined(__cplusplus)static_assert(sizeof(struct htp_ssm_conv_kernel_params) <= 128, "htp_ssm_conv_kernel_params is too large for kernel_params blob");#elsediff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c b/ggml/src/ggml-hexagon/htp/sum-rows-ops.cindex 177c74b33a..3d931ed193 100644--- a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c+++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c@@ -18,6 +18,7 @@#include "htp-ctx.h"#include "htp-ops.h"#include "htp-tensor.h"+#include "sum-rows-ops.h"#define sum_rows_preamble \const struct htp_tensor *src0 = octx->src[0]; \@@ -52,7 +53,6 @@ struct sum_rows_context {size_t dst_stride;uint32_t rows_per_thread;uint32_t total_rows;- bool opt_path;};static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) {@@ -71,7 +71,6 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data)const size_t src_stride = smctx->src_stride;const size_t dst_stride = smctx->dst_stride;const uint32_t ne00 = smctx->ne00;- const bool opt_path = smctx->opt_path;const float * restrict src_th = (const float *) (smctx->src_data + (start_row * src_stride));float * restrict dst_th = (float *) (smctx->dst_data + (start_row * dst_stride));@@ -90,11 +89,7 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data)hex_l2fetch((const uint8_t *) src_local + src_stride, src_stride, src_stride, 1);}- if (opt_path) {- *dst_local = hvx_reduce_sum_f32_a((const uint8_t *) src_local, ne00);- } else {- *dst_local = hvx_reduce_sum_f32((const uint8_t *) src_local, ne00);- }+ *dst_local = hvx_reduce_sum_f32((const uint8_t *) src_local, ne00);}htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row);@@ -111,6 +106,13 @@ int op_sum_rows(struct htp_ops_context * octx) {return HTP_STATUS_NO_SUPPORT;}+ const struct htp_sum_rows_kernel_params * kparams =+ (const struct htp_sum_rows_kernel_params *) octx->kernel_params;++ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {+ return HTP_STATUS_INVAL_PARAMS;+ }+const uint32_t src0_nrows = ne01 * ne02 * ne03;const size_t dst_data_row_size = dst->ne[0] * sizeof(float);@@ -129,14 +131,9 @@ int op_sum_rows(struct htp_ops_context * octx) {return HTP_STATUS_OK;}- const uint32_t n_threads = octx->n_threads;+ const uint32_t n_threads = octx->n_threads;const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);- bool opt_path = false;- if ((0 == hex_is_aligned((void *) src0->data, VLEN)) && !(nb01 & (VLEN - 1))) {- opt_path = true;- }-struct sum_rows_context smctx = {.octx = octx,.src_data = (const uint8_t *) src0->data + row_start * nb01,@@ -146,7 +143,6 @@ int op_sum_rows(struct htp_ops_context * octx) {.dst_stride = nb1,.rows_per_thread = rows_per_thread,.total_rows = nrows,- .opt_path = opt_path,};work_queue_run(octx->ctx->work_queue, sum_rows_thread_f32, &smctx, n_threads);@@ -197,6 +193,13 @@ int op_sum(struct htp_ops_context * octx) {return HTP_STATUS_NO_SUPPORT;}+ const struct htp_sum_rows_kernel_params * kparams =+ (const struct htp_sum_rows_kernel_params *) octx->kernel_params;++ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {+ return HTP_STATUS_INVAL_PARAMS;+ }+if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) {return HTP_STATUS_OK;}@@ -207,9 +210,8 @@ int op_sum(struct htp_ops_context * octx) {return HTP_STATUS_OK;}- const uint32_t n_threads = (total_elems >= 1024) ? MIN(octx->n_threads, HTP_MAX_NTHREADS) : 1;- const uint32_t raw_chunk = (total_elems + n_threads - 1) / n_threads;- const uint32_t elems_per_thread = hex_round_up(raw_chunk, 32);+ const uint32_t n_threads = octx->n_threads;+ const uint32_t elems_per_thread = kparams->elems_per_thread;struct sum_context sctx = {.octx = octx,@@ -324,6 +326,13 @@ int op_argmax(struct htp_ops_context * octx) {return HTP_STATUS_NO_SUPPORT;}+ const struct htp_sum_rows_kernel_params * kparams =+ (const struct htp_sum_rows_kernel_params *) octx->kernel_params;++ if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {+ return HTP_STATUS_INVAL_PARAMS;+ }+const uint32_t ne00 = src0->ne[0];const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];@@ -341,9 +350,8 @@ int op_argmax(struct htp_ops_context * octx) {return HTP_STATUS_OK;}- const uint32_t n_threads = (ne00 >= 1024) ? MIN(octx->n_threads, HTP_MAX_NTHREADS) : 1;- const uint32_t raw_chunk = (ne00 + n_threads - 1) / n_threads;- const uint32_t elems_per_thread = hex_round_up(raw_chunk, 32);+ const uint32_t n_threads = octx->n_threads;+ const uint32_t elems_per_thread = kparams->elems_per_thread;struct argmax_context actx = {.octx = octx,@@ -389,8 +397,8 @@ int op_argmax(struct htp_ops_context * octx) {return HTP_STATUS_OK;}- const uint32_t n_threads = MIN(octx->n_threads, nrows);- const uint32_t rows_per_thread = (nrows + n_threads - 1) / n_threads;+ const uint32_t n_threads = octx->n_threads;+ const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);struct argmax_context actx = {.octx = octx,diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.h b/ggml/src/ggml-hexagon/htp/sum-rows-ops.hnew file mode 100644index 0000000000..cc9ddd4839--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.h@@ -0,0 +1,17 @@+#ifndef HTP_SUM_ROWS_OPS_H+#define HTP_SUM_ROWS_OPS_H++#include <stdint.h>++struct htp_sum_rows_kernel_params {+ uint32_t n_threads;+ uint32_t elems_per_thread;+};++#if defined(__cplusplus)+static_assert(sizeof(struct htp_sum_rows_kernel_params) <= 128, "htp_sum_rows_kernel_params is too large for kernel_params blob");+#else+_Static_assert(sizeof(struct htp_sum_rows_kernel_params) <= 128, "htp_sum_rows_kernel_params is too large for kernel_params blob");+#endif++#endif // HTP_SUM_ROWS_OPS_Hdiff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.cindex ea3d92ad04..0d7440cc1e 100644--- a/ggml/src/ggml-hexagon/htp/unary-ops.c+++ b/ggml/src/ggml-hexagon/htp/unary-ops.c@@ -109,18 +109,17 @@ static inline size_t unary_row_offset(uint32_t ir,return i1 * nb1 + i2 * nb2 + i3 * nb3;}-// Safe DMA block size from row `ir`: clamp to the tighter dim-1 slice-// boundary of src and dst so the nb1 stride stays valid for all rows.+// Safe DMA block size from row `ir`: clamp to the dim-1 slice boundary+// when non-contiguous so the nb1 stride stays valid for all rows.static inline uint32_t unary_block_size(uint32_t ir,uint32_t end_row,uint32_t block,- bool src_contig,- bool dst_contig,+ bool contig,uint32_t ne1,const struct fastdiv_values * div_ne1) {uint32_t limit = MIN(block, end_row - ir);- if (!src_contig || !dst_contig) {+ if (!contig) {const uint32_t slice_end = (fastdiv(ir, div_ne1) + 1) * ne1;limit = MIN(limit, slice_end - ir);}@@ -213,13 +212,15 @@ static void rms_norm_f32(const void * restrict src,const struct htp_unary_context * uctx) {htp_unary_op_preamble;float epsilon = 0.f;- memcpy(&epsilon, op_params, sizeof(float));+ float scale = 0.f;+ memcpy(&epsilon, &op_params[0], sizeof(float));+ memcpy(&scale, &op_params[1], sizeof(float));for (uint32_t ir = 0; ir < num_rows; ir++) {const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);- hvx_fast_rms_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon);+ hvx_fast_rms_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon, scale);}}@@ -230,14 +231,16 @@ static void rms_norm_mul_f32(const void * restrict src,const struct htp_unary_context * uctx) {htp_unary_op_preamble;float epsilon = 0.f;- memcpy(&epsilon, op_params, sizeof(float));+ float scale = 0.f;+ memcpy(&epsilon, &op_params[0], sizeof(float));+ memcpy(&scale, &op_params[1], sizeof(float));for (uint32_t ir = 0; ir < num_rows; ir++) {const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);const uint8_t * restrict w_local = (const uint8_t *)weight + (uctx->broadcast_weight ? 0 : ir * uctx->src1_row_size_aligned);uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);- hvx_fast_rms_norm_mul_f32(src_local, w_local, dst_local, ne0, epsilon);+ hvx_fast_rms_norm_mul_f32(src_local, w_local, dst_local, ne0, epsilon, scale);}}@@ -617,10 +620,11 @@ static void tri_f32(const void * restrict src,const uint32_t nloe = ne0 % VLEN_FP32;const uint32_t ne01 = uctx->octx->src[0]->ne[1];+ const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01;for (uint32_t b = 0; b < num_rows; b++) {const uint32_t abs_row = ir + b;- const uint32_t i01 = abs_row % ne01;+ const uint32_t i01 = fastmodulo(abs_row, ne01, div_ne01);const HVX_Vector * restrict v_src = (const HVX_Vector *) ((const uint8_t *) src + b * src0_row_size_aligned);HVX_Vector * restrict v_dst = (HVX_Vector *) ((uint8_t *) dst + b * dst_row_size_aligned);@@ -1163,35 +1167,23 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * daconst size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size;const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size;- const bool src0_contig = (nb02 == (size_t)ne01 * nb01) &&- (nb03 == (size_t)ne02 * nb02);- const bool dst_contig = (nb2 == (size_t)ne1 * nb1) &&- (nb3 == (size_t)ne2 * nb2);+ const bool src0_contig = uctx->kparams->src0_contig;+ const uint32_t BLOCK = uctx->block;const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01;const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02;const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;- const uint32_t src0_max_block = src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01);- const uint32_t dst_max_block = dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1);- const uint32_t BLOCK = MIN(src0_max_block, dst_max_block);- if (BLOCK == 0) {- FARF(ERROR, "unary-row-block : current VTCM reservation %zu is too small, needed at least %zu\n",- uctx->vtcm_src0_size_per_thread, src0_row_size_aligned);- return;- }-dma_queue * dma_q = octx->ctx->dma[ith];for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) {- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig,- ne01, div_ne01);+ const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01);dma_queue_push(dma_q,dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)),nb1, dst_row_size_aligned, dst_data_row_size, 0);- const size_t src0_off = src0_contig ? (ir * nb01) :+ const size_t src0_off = src0_contig ? ((size_t) ir * nb01) :unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03);dma_queue_push(dma_q,dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off),@@ -1203,8 +1195,7 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * daunary_compute_fn_t compute = (unary_compute_fn_t) uctx->compute;for (uint32_t ir = src0_start_row; ir < src0_end_row; ) {- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig,- ne01, div_ne01);+ const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01);void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src;void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst;@@ -1213,8 +1204,7 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * dacompute(src0_vtcm, dst_vtcm, block_size, uctx);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir);- const size_t dst_off = dst_contig ? (ir * nb1) :- unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3);+ const size_t dst_off = (size_t) ir * nb1;dma_queue_push(dma_q,dma_make_data(data_dst + dst_off, dst_vtcm),nb1, dst_row_size_aligned, dst_data_row_size, block_size);@@ -1222,12 +1212,12 @@ static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * daconst uint32_t next_ir = ir + block_size;if (next_ir < src0_end_row) {const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, src0_contig,- dst_contig, ne01, div_ne01);+ ne01, div_ne01);const uint32_t pref_ir = next_ir + next_block_size;if (pref_ir < src0_end_row) {const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, src0_contig,- dst_contig, ne01, div_ne01);- const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) :+ ne01, div_ne01);+ const size_t src0_pref_off = src0_contig ? ((size_t) pref_ir * nb01) :unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03);dma_queue_push(dma_q,dma_make_data(src0_vtcm, data_src + src0_pref_off),@@ -1273,10 +1263,6 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, voconst uint32_t nb11 = src1->nb[1];const uint32_t nb12 = src1->nb[2];const uint32_t nb13 = src1->nb[3];- const uint32_t nb11_bc = (src1->ne[1] > 1) ? nb11 : 0;- const uint32_t nb12_bc = (src1->ne[2] > 1) ? nb12 : 0;- const uint32_t nb13_bc = (src1->ne[3] > 1) ? nb13 : 0;- const bool src1_contig = ((nb12 == (size_t)ne01 * nb11) && (nb13 == (size_t)ne02 * nb12));uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread);uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_thread)) : NULL;@@ -1286,28 +1272,15 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, voconst size_t src1_vtcm_half_size = uctx->src1_vtcm_half_size;const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size;- const bool src0_contig = (nb02 == (size_t)ne01 * nb01) &&- (nb03 == (size_t)ne02 * nb02);- const bool dst_contig = (nb2 == (size_t)ne1 * nb1) &&- (nb3 == (size_t)ne2 * nb2);+ const bool src0_contig = uctx->kparams->src0_contig;+ const bool src1_contig = uctx->kparams->src1_contig;+ const bool block_contig = src0_contig && src1_contig;+ const uint32_t BLOCK = uctx->block;const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01;const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02;const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;- const bool src1_needs_row_clip = !uctx->broadcast_weight && !src1_contig;- const bool block_src0_contig = src0_contig && !src1_needs_row_clip;- const bool block_dst_contig = dst_contig && !src1_needs_row_clip;-- const uint32_t src0_max_block = block_src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01);- const uint32_t dst_max_block = block_dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1);- const uint32_t BLOCK = MIN(src0_max_block, dst_max_block);- if (BLOCK == 0) {- FARF(ERROR, "unary-rms-norm-mul : current VTCM reservation %zu is too small, needed at least %zu\n",- uctx->vtcm_src0_size_per_thread, src0_row_size_aligned);- return;- }-dma_queue * dma_q = octx->ctx->dma[ith];if (uctx->broadcast_weight) {@@ -1317,22 +1290,21 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vo}for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) {- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig,- ne01, div_ne01);+ const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_contig, ne01, div_ne01);dma_queue_push(dma_q,dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)),nb1, dst_row_size_aligned, dst_data_row_size, 0);- const size_t src0_off = src0_contig ? (ir * nb01) :+ const size_t src0_off = src0_contig ? ((size_t) ir * nb01) :unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03);dma_queue_push(dma_q,dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off),src0_row_size_aligned, nb01, src0_data_row_size, block_size);if (!uctx->broadcast_weight) {- const size_t src1_off = src1_contig ? (ir * nb11) :- unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc, nb13_bc);+ const size_t src1_off = src1_contig ? ((size_t) ir * nb11) :+ unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11, nb12, nb13);dma_queue_push(dma_q,dma_make_data(src1_vtcm_data + (vtcm_idx * src1_vtcm_half_size), data_src1 + src1_off),uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, block_size);@@ -1344,8 +1316,7 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vounary_rms_norm_mul_compute_fn_t compute = (unary_rms_norm_mul_compute_fn_t) uctx->compute;for (uint32_t ir = src0_start_row; ir < src0_end_row; ) {- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_src0_contig, block_dst_contig,- ne01, div_ne01);+ const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_contig, ne01, div_ne01);void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src;void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst;@@ -1359,30 +1330,28 @@ static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, vocompute(src0_vtcm, w, dst_vtcm, block_size, uctx);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir);- const size_t dst_off = dst_contig ? (ir * nb1) :- unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3);+ const size_t dst_off = (size_t) ir * nb1;dma_queue_push(dma_q,dma_make_data(data_dst + dst_off, dst_vtcm),nb1, dst_row_size_aligned, dst_data_row_size, block_size);const uint32_t next_ir = ir + block_size;if (next_ir < src0_end_row) {- const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, block_src0_contig,- block_dst_contig, ne01, div_ne01);+ const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, block_contig,+ ne01, div_ne01);const uint32_t pref_ir = next_ir + next_block_size;if (pref_ir < src0_end_row) {- const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, block_src0_contig,- block_dst_contig, ne01, div_ne01);- const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) :+ const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, block_contig,+ ne01, div_ne01);+ const size_t src0_pref_off = src0_contig ? ((size_t) pref_ir * nb01) :unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03);dma_queue_push(dma_q,dma_make_data(src0_vtcm, data_src + src0_pref_off),src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size);if (!uctx->broadcast_weight) {- const size_t src1_pref_off = src1_contig ? (pref_ir * nb11) :- unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11_bc, nb12_bc,- nb13_bc);+ const size_t src1_pref_off = src1_contig ? ((size_t) pref_ir * nb11) :+ unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11, nb12, nb13);dma_queue_push(dma_q,dma_make_data(src1_vtcm, data_src1 + src1_pref_off),uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, pref_block_size);@@ -1428,37 +1397,18 @@ static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * dataconst size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size;const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size;-- const bool src0_contig = (nb02 == (size_t)ne01 * nb01) &&- (nb03 == (size_t)ne02 * nb02);- const bool dst_contig = (nb2 == (size_t)ne1 * nb1) &&- (nb3 == (size_t)ne2 * nb2);-- const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01;- const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02;- const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;-- const uint32_t src0_max_block = src0_contig ? uctx->block : MIN((uint32_t)uctx->block, ne01);- const uint32_t dst_max_block = dst_contig ? uctx->block : MIN((uint32_t)uctx->block, ne1);- const uint32_t BLOCK = MIN(src0_max_block, dst_max_block);- if (BLOCK == 0) {- FARF(ERROR, "unary-tri : current VTCM reservation %zu is too small, needed at least %zu\n",- uctx->vtcm_src0_size_per_thread, src0_row_size_aligned);- return;- }+ const uint32_t BLOCK = uctx->block;dma_queue * dma_q = octx->ctx->dma[ith];for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) {- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig,- ne01, div_ne01);+ const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);dma_queue_push(dma_q,dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)),nb1, dst_row_size_aligned, dst_data_row_size, 0);- const size_t src0_off = src0_contig ? (ir * nb01) :- unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03);+ const size_t src0_off = (size_t) ir * nb01;dma_queue_push(dma_q,dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off),src0_row_size_aligned, nb01, src0_data_row_size, block_size);@@ -1469,8 +1419,7 @@ static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * dataunary_tri_compute_fn_t compute = (unary_tri_compute_fn_t) uctx->compute;for (uint32_t ir = src0_start_row; ir < src0_end_row; ) {- const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, dst_contig,- ne01, div_ne01);+ const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src;void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst;@@ -1479,22 +1428,18 @@ static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * datacompute(src0_vtcm, dst_vtcm, block_size, ir, uctx);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir);- const size_t dst_off = dst_contig ? (ir * nb1) :- unary_row_offset(ir, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3);+ const size_t dst_off = (size_t) ir * nb1;dma_queue_push(dma_q,dma_make_data(data_dst + dst_off, dst_vtcm),nb1, dst_row_size_aligned, dst_data_row_size, block_size);const uint32_t next_ir = ir + block_size;if (next_ir < src0_end_row) {- const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, src0_contig,- dst_contig, ne01, div_ne01);+ const uint32_t next_block_size = MIN(BLOCK, src0_end_row - next_ir);const uint32_t pref_ir = next_ir + next_block_size;if (pref_ir < src0_end_row) {- const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, src0_contig,- dst_contig, ne01, div_ne01);- const size_t src0_pref_off = src0_contig ? (pref_ir * nb01) :- unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03);+ const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_ir);+ const size_t src0_pref_off = (size_t) pref_ir * nb01;dma_queue_push(dma_q,dma_make_data(src0_vtcm, data_src + src0_pref_off),src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size);@@ -1544,21 +1489,17 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data)const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr;- const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile;-- const bool src0_contig = (nb02 == (size_t)ne01 * nb01) &&- (nb03 == (size_t)ne02 * nb02);- const bool dst_contig = (nb2 == (size_t)ne1 * nb1) &&- (nb3 == (size_t)ne2 * nb2);+ const uint32_t tiles_per_row = uctx->kparams->tiles_per_row;+ const bool src0_contig = uctx->kparams->src0_contig;const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row;for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) {- const uint32_t row = src0_start_row + t / tiles_per_row;- const uint32_t col = (t % tiles_per_row) * col_tile;+ const uint32_t row = src0_start_row + fastdiv(t, div_tpr);+ const uint32_t col = fastmodulo(t, tiles_per_row, div_tpr) * col_tile;const uint32_t tw = MIN(col_tile, ne0 - col);const size_t tb = (size_t) tw * sizeof(float);- const size_t soff = (src0_contig ? (row * nb01) :+ const size_t soff = (src0_contig ? ((size_t) row * nb01) :unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) +(size_t) col * sizeof(float);@@ -1586,9 +1527,7 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data)compute(dst_vtcm, src_vtcm, tw, uctx);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t);- const size_t doff = (dst_contig ? (row * nb1) :- unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) +- (size_t) col * sizeof(float);+ const size_t doff = (size_t) row * nb1 + (size_t) col * sizeof(float);const size_t tb = (size_t) tw * sizeof(float);dma_queue_push(dma_q, dma_make_data(data_dst + doff, dst_vtcm), tb, tb, tb, 1);@@ -1596,7 +1535,7 @@ static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data)if (pt < total_tiles) {const uint32_t ptw = MIN(col_tile, ne0 - pcol);const size_t ptb = (size_t) ptw * sizeof(float);- const size_t psoff = (src0_contig ? (prow * nb01) :+ const size_t psoff = (src0_contig ? ((size_t) prow * nb01) :unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02,nb03)) +(size_t) pcol * sizeof(float);@@ -1657,29 +1596,20 @@ static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, voiddma_queue * dma_q = octx->ctx->dma[ith];- const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01;- const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02;- const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012;- const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr;+ const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01;+ const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr;- const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile;+ const uint32_t tiles_per_row = uctx->kparams->tiles_per_row;const int32_t tri_ttype = op_params[0];- const bool src0_contig = (nb02 == (size_t)ne01 * nb01) &&- (nb03 == (size_t)ne02 * nb02);- const bool dst_contig = (nb2 == (size_t)ne1 * nb1) &&- (nb3 == (size_t)ne2 * nb2);-const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row;for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) {- const uint32_t row = src0_start_row + t / tiles_per_row;- const uint32_t col = (t % tiles_per_row) * col_tile;+ const uint32_t row = src0_start_row + fastdiv(t, div_tpr);+ const uint32_t col = fastmodulo(t, tiles_per_row, div_tpr) * col_tile;const uint32_t tw = MIN(col_tile, ne0 - col);const size_t tb = (size_t) tw * sizeof(float);- const size_t soff = (src0_contig ? (row * nb01) :- unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) +- (size_t) col * sizeof(float);+ const size_t soff = (size_t) row * nb01 + (size_t) col * sizeof(float);dma_queue_push(dma_q, dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0);dma_queue_push(dma_q, dma_make_data(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1);@@ -1706,20 +1636,15 @@ static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, voidcompute(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t);- const size_t doff = (dst_contig ? (row * nb1) :- unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) +- (size_t) col * sizeof(float);+ const size_t doff = (size_t) row * nb1 + (size_t) col * sizeof(float);const size_t tb = (size_t) tw * sizeof(float);dma_queue_push(dma_q, dma_make_data(data_dst + doff, dst_vtcm), tb, tb, tb, 1);const uint32_t pt = t + 2;if (pt < total_tiles) {- const uint32_t ptw = MIN(col_tile, ne0 - pcol);- const size_t ptb = (size_t) ptw * sizeof(float);- const size_t psoff = (src0_contig ? (prow * nb01) :- unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02,- nb03)) +- (size_t) pcol * sizeof(float);+ const uint32_t ptw = MIN(col_tile, ne0 - pcol);+ const size_t ptb = (size_t) ptw * sizeof(float);+ const size_t psoff = (size_t) prow * nb01 + (size_t) pcol * sizeof(float);dma_queue_push(dma_q, dma_make_data(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1);}@@ -1787,28 +1712,6 @@ static int execute_op_unary(struct htp_ops_context * octx) {return HTP_STATUS_NO_SUPPORT;}- // F16 only has row-block kernels for this subset of ops (see the dispatch switch- // below) - reject everything else up front, before touching kparams/VTCM.- if (is_f16) {- switch (octx->op) {- case HTP_OP_NORM:- case HTP_OP_RMS_NORM:- case HTP_OP_SCALE:- case HTP_OP_CLAMP:- case HTP_OP_SQR:- case HTP_OP_SQRT:- case HTP_OP_L2_NORM:- case HTP_OP_UNARY_ABS:- case HTP_OP_UNARY_LOG:- case HTP_OP_UNARY_STEP:- case HTP_OP_HADAMARD:- break;- default:- FARF(ERROR, "unary-%s: not supported for F16\n", op_type);- return HTP_STATUS_NO_SUPPORT;- }- }-const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params;if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {@@ -1838,15 +1741,11 @@ static int execute_op_unary(struct htp_ops_context * octx) {const uint32_t n_threads = octx->n_threads;- const size_t src0_row_size_aligned = kparams->src0_row_size_aligned;- const size_t dst_row_size_aligned = kparams->dst_row_size_aligned;-- // Always 0 for F16 - htp_unary_vtcm_layout_build() keeps F16 on the row-block path,+ // Always 0 for F16 - htp_unary_solve_layout() keeps F16 on the row-block path,// since only F32 has unary_task_f32_tiled_* kernels.const uint32_t col_tile = kparams->col_tile;size_t src1_data_row_size = 0;- size_t src1_row_size_aligned = kparams->src1_row_size_aligned;bool broadcast_weight = kparams->broadcast_weight;const struct htp_tensor * src1 = NULL;@@ -1857,8 +1756,8 @@ static int execute_op_unary(struct htp_ops_context * octx) {src1_data_row_size = src1->ne[0] * sizeof(float);}- if (octx->ctx->vtcm_size < (size_t)kparams->vtcm_size) {- FARF(ERROR, "unary-%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, (size_t)kparams->vtcm_size);+ if ((!col_tile && kparams->block == 0) || octx->ctx->vtcm_size < (size_t) kparams->vtcm_size) {+ FARF(ERROR, "unary-%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, (size_t) kparams->vtcm_size);return HTP_STATUS_VTCM_TOO_SMALL;}@@ -1866,9 +1765,13 @@ static int execute_op_unary(struct htp_ops_context * octx) {octx->src1_spad.src = NULL;octx->dst_spad.src = NULL;- FARF(HIGH, "%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %u src1-vtcm-size %u dst-vtcm-size %u\n", op_type,+ struct htp_unary_vtcm_layout L;+ htp_unary_vtcm_layout_build(&L, octx->op, src0_data_row_size, dst_data_row_size, src1_data_row_size,+ src0_elem_size, col_tile, kparams->block, broadcast_weight, kparams->n_threads);++ FARF(HIGH, "%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type,src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],- kparams->vtcm_src0_size, kparams->vtcm_src1_size, kparams->vtcm_dst_size);+ L.src0_bytes * n_threads, L.src1_bytes * n_threads, L.dst_bytes * n_threads);uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;struct htp_unary_context uctx = {@@ -1886,26 +1789,26 @@ static int execute_op_unary(struct htp_ops_context * octx) {.src1_data_row_size = src1_data_row_size,.dst_data_row_size = dst_data_row_size,- .src0_row_size_aligned = src0_row_size_aligned,- .src1_row_size_aligned = src1_row_size_aligned,- .dst_row_size_aligned = dst_row_size_aligned,+ .src0_row_size_aligned = L.src0_row_size_aligned,+ .src1_row_size_aligned = L.src1_row_size_aligned,+ .dst_row_size_aligned = L.dst_row_size_aligned,- .src0_vtcm_half_size = kparams->vtcm_src0_size_per_thread / 2,- .src1_vtcm_half_size = (octx->op == HTP_OP_RMS_NORM_MUL) ? (kparams->vtcm_src1_size_per_thread / (broadcast_weight ? 1 : 2)) : 0,- .dst_vtcm_half_size = kparams->vtcm_dst_size_per_thread / 2,+ .src0_vtcm_half_size = L.src0_bytes >> 1,+ .src1_vtcm_half_size = (octx->op == HTP_OP_RMS_NORM_MUL) ? (L.src1_bytes >> (broadcast_weight ? 0 : 1)) : 0,+ .dst_vtcm_half_size = L.dst_bytes >> 1,.block = kparams->block,.nc = src0->ne[0],.col_tile = col_tile,.broadcast_weight = broadcast_weight,- .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, 0),- .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, kparams->vtcm_src0_size, kparams->vtcm_src1_size > 0),- .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, kparams->vtcm_src0_size + kparams->vtcm_src1_size),+ .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0),+ .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_src1, L.src1_bytes > 0),+ .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst),- .vtcm_src0_size_per_thread = kparams->vtcm_src0_size_per_thread,- .vtcm_src1_size_per_thread = kparams->vtcm_src1_size_per_thread,- .vtcm_dst_size_per_thread = kparams->vtcm_dst_size_per_thread,+ .vtcm_src0_size_per_thread = L.src0_bytes,+ .vtcm_src1_size_per_thread = L.src1_bytes,+ .vtcm_dst_size_per_thread = L.dst_bytes,};FARF(HIGH, "%s: %s mode (col_tile %u)\n", op_type, col_tile ? "tiled" : "row-block", col_tile);@@ -2007,8 +1910,25 @@ static int execute_op_unary(struct htp_ops_context * octx) {int op_unary(struct htp_ops_context * octx) {switch (octx->src[0]->type) {case HTP_TYPE_F32:- case HTP_TYPE_F16:- return execute_op_unary(octx);+ case HTP_TYPE_F16: {+ if (!htp_unary_is_elementwise(octx->op)) {+ return execute_op_unary(octx);+ }++ struct htp_tensor src0_c = *octx->src[0];+ struct htp_tensor dst_c = *octx->dst;+ const size_t elem_size = (src0_c.type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float);+ HTP_UNARY_COLLAPSE_DIMS(src0_c.ne, src0_c.nb, dst_c.ne, dst_c.nb, elem_size);++ const struct htp_tensor * orig_src0 = octx->src[0];+ const struct htp_tensor * orig_dst = octx->dst;+ octx->src[0] = &src0_c;+ octx->dst = &dst_c;+ const int status = execute_op_unary(octx);+ octx->src[0] = orig_src0;+ octx->dst = orig_dst;+ return status;+ }default:return HTP_STATUS_NO_SUPPORT;diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.hindex 902028635e..ec67612b93 100644--- a/ggml/src/ggml-hexagon/htp/unary-ops.h+++ b/ggml/src/ggml-hexagon/htp/unary-ops.h@@ -2,29 +2,22 @@#define HTP_UNARY_OPS_H#include "hex-common.h"+#include "hex-fastdiv.h"#include "htp-ops.h"+#include "htp-vtcm.h"// Op-specific struct for precomputed unary paramsstruct htp_unary_kernel_params {uint32_t n_threads;uint32_t col_tile;- uint32_t vtcm_row_per_thread;uint32_t block;uint32_t broadcast_weight;- uint32_t vtcm_src0_size_per_thread;- uint32_t vtcm_src1_size_per_thread;- uint32_t vtcm_dst_size_per_thread;-- uint32_t vtcm_src0_size;- uint32_t vtcm_src1_size;- uint32_t vtcm_dst_size;-- uint32_t src0_row_size_aligned;- uint32_t src1_row_size_aligned;- uint32_t dst_row_size_aligned;+ uint32_t src0_contig;+ uint32_t src1_contig;uint32_t vtcm_size;+ uint32_t tiles_per_row;// Fastdiv helpersstruct fastdiv_values div_ne01;@@ -71,23 +64,105 @@ static inline bool htp_op_is_unary(uint32_t opcode) {}}+static inline bool htp_unary_is_elementwise(uint32_t opcode) {+ switch (opcode) {+ case HTP_OP_NORM:+ case HTP_OP_RMS_NORM:+ case HTP_OP_RMS_NORM_MUL:+ case HTP_OP_L2_NORM:+ case HTP_OP_HADAMARD:+ case HTP_OP_TRI:+ return false;+ default:+ return true;+ }+}++#define HTP_UNARY_COLLAPSE_DIMS(src_ne, src_nb, dst_ne, dst_nb, elem_size) \+ do { \+ while (((src_ne)[1] > 1 || (src_ne)[2] > 1 || (src_ne)[3] > 1) && \+ (size_t) (src_nb)[0] == (elem_size) && \+ (size_t) (dst_nb)[0] == (elem_size) && \+ (((src_ne)[1] == 1 && (dst_ne)[1] == 1) || \+ ((size_t) (src_nb)[1] == (size_t) (src_ne)[0] * (size_t) (src_nb)[0] && \+ (size_t) (dst_nb)[1] == (size_t) (dst_ne)[0] * (size_t) (dst_nb)[0] && \+ ((src_ne)[0] == 1 || \+ (uint64_t) (src_ne)[0] * (uint64_t) (src_ne)[1] * (elem_size) <= 128)))) { \+ (src_ne)[0] *= (src_ne)[1]; \+ (src_ne)[1] = (src_ne)[2]; \+ (src_ne)[2] = (src_ne)[3]; \+ (src_ne)[3] = 1; \+ (dst_ne)[0] *= (dst_ne)[1]; \+ (dst_ne)[1] = (dst_ne)[2]; \+ (dst_ne)[2] = (dst_ne)[3]; \+ (dst_ne)[3] = 1; \+ (src_nb)[1] = (src_nb)[2]; \+ (src_nb)[2] = (src_nb)[3]; \+ (dst_nb)[1] = (dst_nb)[2]; \+ (dst_nb)[2] = (dst_nb)[3]; \+ } \+ } while (0)+struct htp_unary_vtcm_layout {size_t total_bytes;size_t off_src0;size_t off_src1;size_t off_dst;+ size_t src0_row_size_aligned;+ size_t src1_row_size_aligned;+ size_t dst_row_size_aligned;+size_t src0_bytes;size_t src1_bytes;size_t dst_bytes;};static inline void htp_unary_vtcm_layout_build(+ struct htp_unary_vtcm_layout * L,+ uint32_t op,+ size_t src0_data_row_size,+ size_t dst_data_row_size,+ size_t src1_data_row_size,+ size_t elem_size,+ uint32_t col_tile,+ uint32_t block,+ bool broadcast_weight,+ uint32_t n_threads+) {+ L->src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);+ L->dst_row_size_aligned = hex_round_up(dst_data_row_size, 128);+ L->src1_row_size_aligned = (op == HTP_OP_RMS_NORM_MUL) ? hex_round_up(src1_data_row_size, 128) : 0;++ if (col_tile > 0) {+ const size_t col_tile_bytes = col_tile * elem_size;+ L->src0_bytes = col_tile_bytes * 2;+ L->dst_bytes = col_tile_bytes * 2;+ L->src1_bytes = 0;+ } else {+ L->src0_bytes = L->src0_row_size_aligned * block * 2;+ L->dst_bytes = L->dst_row_size_aligned * block * 2;+ if (op == HTP_OP_RMS_NORM_MUL) {+ L->src1_bytes = broadcast_weight ? L->src1_row_size_aligned : (L->src1_row_size_aligned * block * 2);+ } else {+ L->src1_bytes = 0;+ }+ }++ size_t off = 0;+ VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes * n_threads);+ VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes * n_threads, L->src1_bytes > 0);+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes * n_threads);+ L->total_bytes = off;+}++static inline void htp_unary_solve_layout(struct htp_unary_vtcm_layout * L,uint32_t op,uint32_t ne00,uint32_t ne10,uint32_t ne11,+ uint32_t max_block,bool broadcast_weight,uint32_t n_threads,size_t vtcm_size,@@ -98,16 +173,11 @@ static inline void htp_unary_vtcm_layout_build(const size_t dst_elem_size = (op == HTP_OP_HADAMARD) ? sizeof(float) : elem_size;const size_t src0_data_row_size = ne00 * elem_size;const size_t dst_data_row_size = ne10 * dst_elem_size;+ const size_t src1_data_row_size = (op == HTP_OP_RMS_NORM_MUL) ? (ne11 * sizeof(float)) : 0;const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128);-- size_t src1_row_size_aligned = 0;- if (op == HTP_OP_RMS_NORM_MUL) {- // RMS_NORM_MUL fusion is F32-only; its weight tensor is always F32.- const size_t src1_data_row_size = ne11 * sizeof(float);- src1_row_size_aligned = hex_round_up(src1_data_row_size, 128);- }+ const size_t src1_row_size_aligned = (op == HTP_OP_RMS_NORM_MUL) ? hex_round_up(src1_data_row_size, 128) : 0;size_t vtcm_size_per_row = 0;size_t vtcm_row_per_thread = 0;@@ -132,6 +202,10 @@ static inline void htp_unary_vtcm_layout_build(vtcm_row_per_thread = vtcm_size / (n_threads * vtcm_size_per_row);}+ if (max_block > 0 && vtcm_row_per_thread > max_block) {+ vtcm_row_per_thread = max_block;+ }+const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM ||op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM ||op == HTP_OP_HADAMARD);@@ -148,37 +222,14 @@ static inline void htp_unary_vtcm_layout_build(const size_t per_thread_budget = vtcm_size / n_threads;const size_t col_tile_bytes = hex_align_down(per_thread_budget / 4, 128);col_tile = (uint32_t) (col_tile_bytes / elem_size);-- L->src0_bytes = col_tile_bytes * 2;- L->dst_bytes = col_tile_bytes * 2;- L->src1_bytes = 0;- } else {- L->src0_bytes = src0_row_size_aligned * vtcm_row_per_thread * 2;- L->dst_bytes = dst_row_size_aligned * vtcm_row_per_thread * 2;- if (op == HTP_OP_RMS_NORM_MUL) {- if (broadcast_weight) {- L->src1_bytes = src1_row_size_aligned;- } else {- L->src1_bytes = src1_row_size_aligned * vtcm_row_per_thread * 2;- }- } else {- L->src1_bytes = 0;- }- }-- L->off_src0 = 0;- if (op == HTP_OP_RMS_NORM_MUL) {- L->off_src1 = L->off_src0 + L->src0_bytes * n_threads;- L->off_dst = L->off_src1 + L->src1_bytes * n_threads;- } else {- L->off_src1 = 0;- L->off_dst = L->off_src0 + L->src0_bytes * n_threads;}- L->total_bytes = L->off_dst + L->dst_bytes * n_threads;+ const uint32_t block = col_tile ? 0 : (uint32_t) vtcm_row_per_thread;+ htp_unary_vtcm_layout_build(L, op, src0_data_row_size, dst_data_row_size, src1_data_row_size,+ elem_size, col_tile, block, broadcast_weight, n_threads);*out_col_tile = col_tile;- *out_vtcm_row_per_thread = vtcm_row_per_thread;+ *out_vtcm_row_per_thread = (uint32_t) vtcm_row_per_thread;}#endif /* HTP_UNARY_OPS_H */diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex f490d201c7..0fc5c0b5f0 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -9971,7 +9971,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {}// fused unary + mul (gated activations that are not expressed as GGML_OP_GLU)- for (ggml_unary_op op : { GGML_UNARY_OP_GELU, GGML_UNARY_OP_SILU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) {+ for (ggml_unary_op op : { GGML_UNARY_OP_GELU, GGML_UNARY_OP_SILU, GGML_UNARY_OP_RELU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) {for (ggml_type type : { GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16 }) {for (bool swap : { false, true }) {test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, swap));