LLM inference in C/C++
C++ ggml
hexagon: L2 cache handling rework (dirty bit tracking with lazy flushing) and more MUL_MAT updates (#25762)
Max Krasnyanskyb2dd28a3b601
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex 76c71d7ee7..9388a59c1a 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -21,6 +21,11 @@#include <algorithm>#ifdef _WIN32+# define WIN32_LEAN_AND_MEAN+# ifndef NOMINMAX+# define NOMINMAX+# endif+# include <windows.h># include <sal.h>#else# include <semaphore.h>@@ -28,7 +33,9 @@#endif#pragma clang diagnostic ignored "-Wnested-anon-types"+#pragma clang diagnostic ignored "-Wlanguage-extension-token"#pragma clang diagnostic ignored "-Wgnu-anonymous-struct"+#pragma clang diagnostic ignored "-Wmicrosoft-enum-value"#include <AEEStdErr.h>#include <dspqueue.h>@@ -134,6 +141,8 @@ static const char * htp_event_name(uint16_t id) {case HTP_TRACE_EVT_HVX_FA_K_PREP: return "HVX_K_PREP";case HTP_TRACE_EVT_HVX_FA_V_PREP: return "HVX_V_PREP";case HTP_TRACE_EVT_HMX_COMP: return "HMX_COMP";+ case HTP_TRACE_EVT_L2FLUSH: return "L2FLUSH";+ case HTP_TRACE_EVT_INIT: return "INIT";default: return "UNKNOWN";}}@@ -501,6 +510,8 @@ static void repack_q4_0_tiled(ggml_tensor * t, const void * data, size_t size) {}}}++ GGML_UNUSED(size);}// repack q4_0_tiled tensor into q4_0 data@@ -554,6 +565,8 @@ static void repack_tiled_q4_0(void * data, const ggml_tensor * t, size_t size) {}}}++ GGML_UNUSED(size);}// repack q4_1 data into q4_1_tiled tensor@@ -611,6 +624,8 @@ static void repack_q4_1_tiled(ggml_tensor * t, const void * data, size_t size) {}}}++ GGML_UNUSED(size);}// repack q4_1_tiled tensor into q4_1 data@@ -665,6 +680,8 @@ static void repack_tiled_q4_1(void * data, const ggml_tensor * t, size_t size) {}}}++ GGML_UNUSED(size);}// repack q8_0 data into q8_0_tiled tensor@@ -711,6 +728,8 @@ static void repack_q8_0_tiled(ggml_tensor * t, const void * data, size_t size) {}}}++ GGML_UNUSED(size);}// repack q8_0_tiled tensor into q8_0 data@@ -761,6 +780,8 @@ static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t size) {}}}++ GGML_UNUSED(size);}// repack mxfp4 data into mxfp4_tiled tensor@@ -812,6 +833,8 @@ static void repack_mxfp4_tiled(ggml_tensor * t, const void * data, size_t size)}}}++ GGML_UNUSED(size);}// repack mxfp4_tiled tensor into mxfp4 data@@ -865,6 +888,8 @@ static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t size)}}}++ GGML_UNUSED(size);}static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer,@@ -965,11 +990,12 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,static bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t buffer,const struct ggml_tensor * src,struct ggml_tensor * dst) {+ // we might optimize this later, for now take the slow path (ie get/set_tensor)+ return false;+GGML_UNUSED(buffer);GGML_UNUSED(src);GGML_UNUSED(dst);- // we might optimize this later, for now take the slow path (ie get/set_tensor)- return false;}static void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {@@ -1025,9 +1051,9 @@ static ggml_backend_buffer_t ggml_backend_hexagon_repack_buffer_type_alloc_buffe}}-static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer_type_t buffer_type) {+static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {return 128; // HVX alignment- GGML_UNUSED(buffer_type);+ GGML_UNUSED(buft);}static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * t) {@@ -1039,20 +1065,24 @@ static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffereturn ggml_row_size(t->type, ne0) * ne1 * ne2 * ne3;}return ggml_nbytes(t);++ GGML_UNUSED(buft);}-static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buffer_type) {- auto * context = static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context);+static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {+ auto * context = static_cast<ggml_backend_hexagon_buffer_type_context *>(buft->context);return context->sess->max_bufsize;}static bool ggml_backend_hexagon_buffer_type_is_host(ggml_backend_buffer_type_t buft) {return opt_hostbuf;+GGML_UNUSED(buft);}static bool ggml_backend_hexagon_repack_buffer_type_is_host(ggml_backend_buffer_type_t buft) {return false;+GGML_UNUSED(buft);}@@ -1098,6 +1128,14 @@ struct ggml_hexagon_opbatch {std::unordered_map<const ggml_tensor*, int> t_map; // tensor ptr to indexstd::unordered_multimap<void*, int> d_map; // tensor data to index+ struct tensor_range {+ uint64_t start;+ uint64_t end;+ int bi;+ std::vector<int> tensors;+ };+ std::vector<tensor_range> ranges;+unsigned int n_bufs; // num buffers in the batchunsigned int n_tens; // num tensors ...unsigned int n_ops; // num ops ...@@ -1117,6 +1155,7 @@ struct ggml_hexagon_opbatch {b_map.clear();t_map.clear();d_map.clear();+ ranges.clear();}ggml_hexagon_opbatch(ggml_hexagon_session *sess, size_t batch_size, size_t max_vmem) {@@ -1124,7 +1163,7 @@ struct ggml_hexagon_opbatch {n_bufs_max = HTP_OP_MAX_BUFS;n_ops_max = batch_size;- n_tens_max = n_ops_max + n_ops_max * HTP_OP_MAX_INPUTS;+ n_tens_max = std::min<size_t>(n_ops_max + n_ops_max * HTP_OP_MAX_INPUTS, HTP_OP_MAX_TENSORS);b_vmem_max = max_vmem;@@ -1170,6 +1209,71 @@ struct ggml_hexagon_opbatch {return bi;}+ void add_range(const htp_tensor * h, int ti) {+ uint64_t t_start = h->data;+ uint64_t t_end = t_start + h->size;+ int bi = h->bi;++ int first_match = -1;+ int unused_idx = -1;+ for (size_t i = 0; i < ranges.size(); i++) {+ if (ranges[i].bi == -1) {+ unused_idx = i;+ continue;+ }+ if (ranges[i].bi != bi) {+ continue;+ }+ if (ranges[i].start >= t_end || ranges[i].end <= t_start) {+ continue;+ }++ if (first_match == -1) {+ first_match = i;+ HEX_VERBOSE("ggml-hex: %s range-grow #%d : bi %d [%p, %p) + #%d [%p, %p) -> [%p, %p)\n",+ sess->c_name(), (int) i, ranges[i].bi,+ (void *) (h_bufs[ranges[i].bi].base + ranges[i].start),+ (void *) (h_bufs[ranges[i].bi].base + ranges[i].end),+ ti,+ (void *) (h_bufs[bi].base + t_start),+ (void *) (h_bufs[bi].base + t_end),+ (void *) (h_bufs[ranges[i].bi].base + std::min(ranges[i].start, t_start)),+ (void *) (h_bufs[ranges[i].bi].base + std::max(ranges[i].end, t_end)));++ ranges[i].start = std::min(ranges[i].start, t_start);+ ranges[i].end = std::max(ranges[i].end, t_end);+ ranges[i].tensors.push_back(ti);+ } else {+ HEX_VERBOSE("ggml-hex: %s range-merge #%d [%p, %p) + #%d [%p, %p) -> [%p, %p)\n",+ sess->c_name(), first_match,+ (void *) (h_bufs[bi].base + ranges[first_match].start),+ (void *) (h_bufs[bi].base + ranges[first_match].end),+ (int) i,+ (void *) (h_bufs[bi].base + ranges[i].start),+ (void *) (h_bufs[bi].base + ranges[i].end),+ (void *) (h_bufs[bi].base + std::min(ranges[first_match].start, ranges[i].start)),+ (void *) (h_bufs[bi].base + std::max(ranges[first_match].end, ranges[i].end)));++ ranges[first_match].start = std::min(ranges[first_match].start, ranges[i].start);+ ranges[first_match].end = std::max(ranges[first_match].end, ranges[i].end);+ ranges[first_match].tensors.insert(+ ranges[first_match].tensors.end(),+ ranges[i].tensors.begin(),+ ranges[i].tensors.end()+ );+ ranges[i].bi = -1;+ }+ }++ if (first_match == -1) {+ if (unused_idx != -1) {+ ranges[unused_idx] = {t_start, t_end, bi, {ti}};+ } else {+ ranges.push_back({t_start, t_end, bi, {ti}});+ }+ }+ }+bool same_shape(const htp_tensor * h, const ggml_tensor * t) const {int64_t ne0 = t->ne[0];int64_t ne1 = t->ne[1];@@ -1213,6 +1317,7 @@ struct ggml_hexagon_opbatch {htp_tensor &h = h_tens[ti];h.bi = add_buffer(sbuf);+ h.ti = ti;h.data = t_offset;h.type = t->type;@@ -1235,8 +1340,11 @@ struct ggml_hexagon_opbatch {h.nb[0] = t->nb[0]; h.nb[1] = t->nb[1]; h.nb[2] = t->nb[2]; h.nb[3] = t->nb[3];}+ h.alias = ti;+ add_range(&h, ti);+h.flags = 0;- if (ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {+ if (ggml_backend_buffer_get_usage(t->buffer) != GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {h.flags |= HTP_TENSOR_COMPUTE;}@@ -1313,6 +1421,17 @@ struct ggml_hexagon_opbatch {o.dst[i] = (i < outputs.size() && outputs[i]) ? add_tensor(outputs[i]) : 0xffff;}}++ void finalize_ranges() {+ for (const auto & r : ranges) {+ if (r.bi == -1) {+ continue;+ }+ for (size_t i = 0; i < r.tensors.size(); i++) {+ h_tens[r.tensors[i]].alias = r.tensors[(i + 1) % r.tensors.size()];+ }+ }+ }};struct ggml_hexagon_opqueue {@@ -1571,6 +1690,8 @@ void ggml_hexagon_session::flush_pending(bool all) {void ggml_hexagon_session::flush_batch() {if (op_batch->empty()) { return; }+ op_batch->finalize_ranges();+htp_opbatch_req req {};dspqueue_buffer dbuf{};@@ -1647,7 +1768,7 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {GGML_LOG_DEBUG("ggml-hex: %s allocating new session\n", this->name.c_str());- domain * my_domain = get_domain(this->domain_id);+ domain * my_domain = htpdrv_get_domain(this->domain_id);if (my_domain == NULL) {GGML_LOG_ERROR("ggml-hex: unable to get domain struct for CDSP\n");throw std::runtime_error("ggml-hex: failed to get CDSP domain (see log for details)");@@ -1793,16 +1914,6 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {}}- if (opt_profile) {- htp_iface_pmu_conf pmu_conf{};- std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events);-- err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf);- if (err != 0) {- GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err);- }- }-// Allocate buffers and state for op batchingthis->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue);@@ -1821,6 +1932,16 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {throw std::runtime_error("ggml-hex: iface start failed (see log for details)");}this->valid_iface = true;++ if (opt_profile) {+ htp_iface_pmu_conf pmu_conf{};+ std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events);++ err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf);+ if (err != 0) {+ GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err);+ }+ }}void ggml_hexagon_session::release() noexcept(true) {@@ -1929,6 +2050,8 @@ static bool ggml_hexagon_flash_attn_is_hmx_eligible(}return true;++ GGML_UNUSED(sinks);}static bool ggml_hexagon_precompute_flash_attn_params(@@ -2149,8 +2272,9 @@ static bool ggml_hexagon_supported_gated_delta_net(const struct ggml_hexagon_sesreturn false;}- GGML_UNUSED(sess);return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_matmul_is_hmx_eligible(@@ -2198,6 +2322,8 @@ static bool ggml_hexagon_matmul_is_hmx_eligible(}return true;++ GGML_UNUSED(dst);}static bool ggml_hexagon_precompute_hmx_mm_params(@@ -2234,109 +2360,15 @@ static bool ggml_hexagon_precompute_hmx_mm_params(if (is_batched_val && wtype == GGML_TYPE_F16 && group_size > 1) {// Try grouped path firstconst bool use_dma_activation = (src1->nb[1]/sizeof(float) > (size_t)ne00_padded);- size_t best_mblocks = SIZE_MAX;- int best_act_threads = 0;- size_t best_m_chunk = 0;- size_t best_n_chunk = 0;- size_t best_vtcm_size = 0;-- int act_threads = n_threads;- while (act_threads >= 1) {- const size_t f32_scratch_size = use_dma_activation ? hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * ne00_padded * sizeof(float), HTP_MM_HMX_TILE_SIZE) : 0;- size_t group_overhead = 256 + f32_scratch_size;- size_t group_size_per_n, group_size_per_m, group_size_per_mn;- htp_mm_hmx_get_batched_chunk_costs(ne00_padded, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn);-- size_t m_chunk_candidate = 0;- size_t n_chunk_candidate = 0;- size_t vtcm_size_candidate = 0;-- if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded,- (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT,- &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) {- size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, ne00_padded, m_chunk_candidate, n_chunk_candidate, group_size, use_dma_activation, pipeline, act_threads);- if (exact_size <= vtcm_budget) {- size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate;- if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) {- best_mblocks = mblocks;- best_act_threads = act_threads;- best_m_chunk = m_chunk_candidate;- best_n_chunk = n_chunk_candidate;- best_vtcm_size = exact_size;- }- }- }- if (act_threads == 1) {- act_threads = 0;- } else {- act_threads /= 2;- }- }-- if (best_act_threads > 0) {- m_chunk = best_m_chunk;- n_chunk = best_n_chunk;- vtcm_size = best_vtcm_size;- act_threads_selected = best_act_threads;+ if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, ne11, group_size, use_dma_activation, n_threads, pipeline, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) {use_grouped = true;}}if (!use_grouped) {// Fallback to simple 2D path (group_size = 1)- size_t best_mblocks = SIZE_MAX;- int best_act_threads = 0;- size_t best_m_chunk = 0;- size_t best_n_chunk = 0;- size_t best_vtcm_size = 0;-- // For MUL_MAT_ID the kernel runs one 2D matmul per expert, with M equal to the number of rows routed to that expert.- // A single expert can receive up to all routed rows (dst->ne[1]*dst->ne[2] = n_expert_used*n_tokens), so size the chunk- // search for that upper bound rather than ne12 (token positions only).- // We recompute m_chunk per expert against the actual count in the NPU kernel.- const int m_id_rows = (int) ((size_t) dst->ne[1] * dst->ne[2]);- const int m_for_chunks = is_matmul_id ? hex_align_up(m_id_rows, 32) : ne11_padded;- const int m_for_cost = is_matmul_id ? m_id_rows : ne11;-- int act_threads = n_threads;- while (act_threads >= 1) {- const size_t act_f32_size = is_matmul_id ? 0 : hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * ne00_padded * sizeof(float), HTP_MM_HMX_TILE_SIZE);- size_t simple_2d_overhead = 256 + act_f32_size;- size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn;- htp_mm_hmx_get_2d_chunk_costs(wtype, ne00_padded, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn);-- size_t m_chunk_candidate = 0;- size_t n_chunk_candidate = 0;- size_t vtcm_size_candidate = 0;-- if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded,- (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT,- &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) {- size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, ne00_padded, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size);- if (exact_size <= vtcm_budget) {- size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate;- if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) {- best_mblocks = mblocks;- best_act_threads = act_threads;- best_m_chunk = m_chunk_candidate;- best_n_chunk = n_chunk_candidate;- best_vtcm_size = exact_size;- }- }- }- if (act_threads == 1) {- act_threads = 0;- } else {- act_threads /= 2;- }- }-- if (best_act_threads > 0) {- m_chunk = best_m_chunk;- n_chunk = best_n_chunk;- vtcm_size = best_vtcm_size;- act_threads_selected = best_act_threads;- } else {+ const int m_id_rows = (int) ((size_t) dst->ne[1] * dst->ne[2]);+ if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, ne11_padded, ne11, n_threads, pipeline, is_matmul_id, aligned_tile_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) {return false;}}@@ -2352,6 +2384,8 @@ static bool ggml_hexagon_precompute_hmx_mm_params(kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);kparams->vtcm_size = vtcm_size;kparams->vtcm_src0_size = 0;+ kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected);+ kparams->div_ne00_padded = init_fastdiv_values(ne00_padded);kparams->vtcm_src1_size = 0;kparams->vtcm_dst_size = 0;@@ -2361,6 +2395,8 @@ static bool ggml_hexagon_precompute_hmx_mm_params(kparams->kernel_type = HTP_MM_KERNEL_HMX_2D;}return true;++ GGML_UNUSED(src0);}static void ggml_hexagon_precompute_hvx_mm_params(@@ -2955,6 +2991,8 @@ static bool ggml_hexagon_supported_binary(const struct ggml_hexagon_session * se}return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -2981,6 +3019,8 @@ static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * se}return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -3006,6 +3046,8 @@ static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * ses}return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_sum_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -3025,10 +3067,11 @@ static bool ggml_hexagon_supported_sum_rows(const struct ggml_hexagon_session *}return true;++ GGML_UNUSED(sess);}-static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session * sess,- const struct ggml_tensor * op) {+static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {const struct ggml_tensor * src0 = op->src[0];const struct ggml_tensor * src1 = op->src[1];const struct ggml_tensor * dst = op;@@ -3057,6 +3100,8 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session}return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -3122,6 +3167,8 @@ static bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * s}return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -3142,6 +3189,8 @@ static bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session *}return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -3162,6 +3211,8 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *}return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -3182,6 +3233,8 @@ static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * s}return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -3243,6 +3296,8 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sessreturn false;}return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -3282,6 +3337,8 @@ static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session *}return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -3292,8 +3349,9 @@ static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess,return false;}- GGML_UNUSED(sess);return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_cumsum(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -3308,8 +3366,9 @@ static bool ggml_hexagon_supported_cumsum(const struct ggml_hexagon_session * sereturn false;}- GGML_UNUSED(sess);return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_diag(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -3331,8 +3390,9 @@ static bool ggml_hexagon_supported_diag(const struct ggml_hexagon_session * sessreturn false;}- GGML_UNUSED(sess);return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_solve_tri(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -3364,8 +3424,9 @@ static bool ggml_hexagon_supported_solve_tri(const struct ggml_hexagon_session *return false;}- GGML_UNUSED(sess);return true;++ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_tri(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -3812,6 +3873,8 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr}}}++ GGML_UNUSED(backend);}static struct ggml_backend_i hexagon_backend_i = {@@ -3930,6 +3993,8 @@ static bool ggml_hexagon_supported_buffers(ggml_hexagon_session *sess, const str}static bool ggml_hexagon_supported_cpy(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {+ GGML_UNUSED(sess);+const struct ggml_tensor * src0 = op->src[0];const struct ggml_tensor * dst = op;@@ -4000,6 +4065,7 @@ static bool ggml_hexagon_supported_concat(const struct ggml_hexagon_session * se}return true;+ GGML_UNUSED(sess);}static bool ggml_hexagon_supported_fill(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {@@ -4009,8 +4075,8 @@ static bool ggml_hexagon_supported_fill(const struct ggml_hexagon_session * sessreturn false;}- GGML_UNUSED(sess);return true;+ GGML_UNUSED(sess);}static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {@@ -4293,6 +4359,7 @@ static void * ggml_backend_hexagon_get_proc_address(ggml_backend_reg_t reg, cons}return NULL;+ GGML_UNUSED(reg);}template<typename T> std::vector<T> str_to_vec(const char* str) {@@ -4351,10 +4418,18 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {// Init Arch first since it affects other defaultsif (!str_arch) {- int err = get_hex_arch_ver(CDSP_DOMAIN_ID, &opt_arch);+ int err = htpdrv_get_arch(CDSP_DOMAIN_ID, &opt_arch);if (err != 0) {GGML_LOG_ERROR("ggml-hex: failed to query HTP version (err %d) defaulting to v73\n", err);opt_arch = 73;+ } else {+ if (opt_arch < 73) {+ GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is under supported range, capping at v73\n", opt_arch);+ opt_arch = 73;+ } else if (opt_arch > 81) {+ GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is over supported range, capping at v81\n", opt_arch);+ opt_arch = 81;+ }}} else {if (str_arch[0] == 'v' || str_arch[0] == 'V') {diff --git a/ggml/src/ggml-hexagon/htp-drv.cpp b/ggml/src/ggml-hexagon/htp-drv.cppindex 4c376b5fc9..40c6010540 100644--- a/ggml/src/ggml-hexagon/htp-drv.cpp+++ b/ggml/src/ggml-hexagon/htp-drv.cpp@@ -1,13 +1,8 @@-// sample drv interface--#pragma clang diagnostic ignored "-Wgnu-anonymous-struct"-#pragma clang diagnostic ignored "-Wmissing-prototypes"-#pragma clang diagnostic ignored "-Wsign-compare"-#include <filesystem>#include <set>#include <sstream>#include <string>+#ifdef _WIN32# define WIN32_LEAN_AND_MEAN# ifndef NOMINMAX@@ -16,9 +11,17 @@# include <windows.h># include <winevt.h>#else-# include <dlfcn.h>-# include <unistd.h>+# include <dlfcn.h>+# include <unistd.h>#endif++#pragma clang diagnostic ignored "-Wgnu-anonymous-struct"+#pragma clang diagnostic ignored "-Wmissing-prototypes"+#pragma clang diagnostic ignored "-Wsign-compare"+#pragma clang diagnostic ignored "-Wlanguage-extension-token"+#pragma clang diagnostic ignored "-Wmicrosoft-enum-value"+#pragma clang diagnostic ignored "-Wnested-anon-types"+#include "ggml-impl.h"#include "htp-drv.h"#include "libdl.h"@@ -359,7 +362,7 @@ int htpdrv_init() {return AEE_SUCCESS;}-domain * get_domain(int domain_id) {+domain * htpdrv_get_domain(int domain_id) {int i = 0;int size = sizeof(supported_domains) / sizeof(domain);@@ -372,7 +375,7 @@ domain * get_domain(int domain_id) {return NULL;}-int get_hex_arch_ver(int domain, int * arch) {+int htpdrv_get_arch(int domain, int * arch) {if (!remote_handle_control_pfn) {GGML_LOG_ERROR("ggml-hex: remote_handle_control is not supported on this device\n");return AEE_EUNSUPPORTEDAPI;@@ -394,25 +397,7 @@ int get_hex_arch_ver(int domain, int * arch) {return err;}- switch (arch_ver.capability & 0xff) {- case 0x68:- *arch = 68;- return 0;- case 0x69:- *arch = 69;- return 0;- case 0x73:- *arch = 73;- return 0;- case 0x75:- *arch = 75;- return 0;- case 0x79:- *arch = 79;- return 0;- case 0x81:- *arch = 81;- return 0;- }- return -1;+ uint32_t val = arch_ver.capability & 0xff;+ *arch = (int) ((val >> 4) * 10 + (val & 0x0f));+ return 0;}diff --git a/ggml/src/ggml-hexagon/htp-drv.h b/ggml/src/ggml-hexagon/htp-drv.hindex 6eba7ba17d..f3cc0da75c 100644--- a/ggml/src/ggml-hexagon/htp-drv.h+++ b/ggml/src/ggml-hexagon/htp-drv.h@@ -96,17 +96,17 @@ extern "C" {HTPDRV_API int htpdrv_init(void);/**- * get_domain API: get domain struct from domain value.+ * htpdrv_get_domain API: get domain struct from domain value.** @param[in] domain value of a domain* @return Returns domain struct of the domain if it is supported or else* returns NULL.**/-HTPDRV_API domain * get_domain(int domain_id);+HTPDRV_API domain * htpdrv_get_domain(int domain_id);/**- * get_hex_arch_ver API: query the Hexagon processor architecture version information+ * htpdrv_get_arch API: query the Hexagon processor architecture version information** @param[in] domain_id value of a domain* @param[out] Arch version (73, 75, ...)@@ -114,7 +114,7 @@ HTPDRV_API domain * get_domain(int domain_id);* non-zero if error, return value points to the error.**/-HTPDRV_API int get_hex_arch_ver(int domain, int * arch);+HTPDRV_API int htpdrv_get_arch(int domain, int * arch);#ifdef __cplusplus}diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txtindex cf9e726c00..4fb526f0c0 100644--- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt+++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt@@ -17,9 +17,12 @@ set(HTP_LIB ggml-htp-${DSP_VERSION})add_library(${HTP_LIB} SHAREDmain.chtp_iface_skel.c- worker-pool.c- hex-dma.c+ work-queue.c+ dma-queue.chmx-queue.c+ htp-tensor.c+ matmul-ops.c+ flash-attn-ops.cgated-delta-net-ops.cbinary-ops.cunary-ops.c@@ -38,8 +41,6 @@ add_library(${HTP_LIB} SHAREDdiag-ops.csolve-tri-ops.cpad-ops.c- flash-attn-ops.c- matmul-ops.cargsort-ops.c)diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.cindex 6416d2dfbc..e56f2b6a52 100644--- a/ggml/src/ggml-hexagon/htp/act-ops.c+++ b/ggml/src/ggml-hexagon/htp/act-ops.c@@ -16,6 +16,7 @@#include "htp-ctx.h"#include "htp-ops.h"#include "htp-ops.h"+#include "htp-tensor.h"#define htp_act_preamble \const struct htp_tensor * src0 = actx->octx->src[0]; \@@ -766,17 +767,11 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {}int op_activations(struct htp_ops_context * octx) {- int err = HTP_STATUS_OK;-switch (octx->src[0]->type) {case HTP_TYPE_F32:- err = execute_op_activations_f32(octx);- break;+ return execute_op_activations_f32(octx);default:- err = HTP_STATUS_NO_SUPPORT;- break;+ return HTP_STATUS_NO_SUPPORT;}-- return err;}diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.c b/ggml/src/ggml-hexagon/htp/argsort-ops.cindex 13f12a39e6..774faef5f3 100644--- a/ggml/src/ggml-hexagon/htp/argsort-ops.c+++ b/ggml/src/ggml-hexagon/htp/argsort-ops.c@@ -345,7 +345,7 @@ static void htp_argsort_f32_##ne00##_##order_name(unsigned int n, unsigned int iint32_t * indices_buf = (int32_t *) (spad + values_size); \uint32_t nb01 = src0->nb[1]; \uint32_t nb1 = dst->nb[1]; \- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[i] : NULL; \+ struct htp_thread_trace * tr = &octx->ctx->trace[i]; \htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, start_row); \for (uint32_t r = start_row; r < end_row; r++) { \uint32_t src_offset = r * nb01; \@@ -411,7 +411,7 @@ static void htp_argsort_f32_fallback(unsigned int n, unsigned int i, void * dataconst HVX_Vector ind_init_vec = *(HVX_Vector *)argosrt_ramp_lut;const HVX_Vector ind_diff_vec = Q6_V_vsplat_R(32);- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[i] : NULL;+ struct htp_thread_trace * tr = &octx->ctx->trace[i];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, start_row);for (uint32_t r = start_row; r < end_row; r++) {diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.cindex 52013ad0fe..db61779635 100644--- a/ggml/src/ggml-hexagon/htp/binary-ops.c+++ b/ggml/src/ggml-hexagon/htp/binary-ops.c@@ -16,6 +16,7 @@#include "htp-ctx.h"#include "htp-ops.h"#include "htp-ops.h"+#include "htp-tensor.h"#ifndef MIN#define MIN(a, b) ((a) < (b) ? (a) : (b))diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.c b/ggml/src/ggml-hexagon/htp/cumsum-ops.cindex 2ced197123..2d45c39f23 100644--- a/ggml/src/ggml-hexagon/htp/cumsum-ops.c+++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.c@@ -9,6 +9,7 @@#include "ggml-common.h"#include "htp-ctx.h"#include "htp-ops.h"+#include "htp-tensor.h"#include "hvx-types.h"#include "hvx-utils.h"#include "hex-dma.h"@@ -255,16 +256,10 @@ int op_cumsum_f32(struct htp_ops_context * octx) {int op_cumsum(struct htp_ops_context * octx) {const struct htp_tensor * dst = octx->dst;- int err = HTP_STATUS_OK;-switch (dst->type) {case HTP_TYPE_F32:- err = op_cumsum_f32(octx);- break;+ return op_cumsum_f32(octx);default:- err = HTP_STATUS_NO_SUPPORT;- break;+ return HTP_STATUS_NO_SUPPORT;}-- return err;}diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.c b/ggml/src/ggml-hexagon/htp/dma-queue.cnew file mode 100644index 0000000000..8f60a2c9dc--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/dma-queue.c@@ -0,0 +1,106 @@+#include "dma-queue.h"++#include <stdbool.h>+#include <stdlib.h>+#include <string.h>++#pragma clang diagnostic ignored "-Wunused-function"++static inline uint32_t pow2_ceil(uint32_t x) {+ if (x <= 1) {+ return 1;+ }+ int p = 2;+ x--;+ while (x >>= 1) {+ p <<= 1;+ }+ return p;+}++static inline uintptr_t align_up(uintptr_t addr, size_t align) {+ return (addr + align - 1) & ~(align - 1);+}++size_t dma_queue_sizeof(size_t capacity) {+ capacity = pow2_ceil(capacity);++ size_t size_q = sizeof(dma_queue);+ size_t offset_r = align_up(size_q, HEX_L2_LINE_SIZE);+ size_t size_r = sizeof(dma_ring);+ size_t offset_desc = align_up(offset_r + size_r, HEX_L2_LINE_SIZE);+ size_t size_desc = capacity * sizeof(dma_descriptor_2d);+ size_t offset_dptr = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE);+ size_t size_dptr = capacity * sizeof(dma_ptr);++ return offset_dptr + size_dptr;+}++size_t dma_queue_alignof(void) {+ return HEX_L2_LINE_SIZE;+}++dma_queue_t dma_queue_init(void * ptr, size_t capacity, uintptr_t vtcm_base, size_t vtcm_size, struct htp_thread_trace * trace) {+ capacity = pow2_ceil(capacity);++ size_t size_q = sizeof(dma_queue);+ size_t offset_r = align_up(size_q, HEX_L2_LINE_SIZE);+ size_t size_r = sizeof(dma_ring);+ size_t offset_desc = align_up(offset_r + size_r, HEX_L2_LINE_SIZE);+ size_t size_desc = capacity * sizeof(dma_descriptor_2d);+ size_t offset_dptr = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE);+ size_t size_dptr = capacity * sizeof(dma_ptr);++ size_t total_size = offset_dptr + size_dptr;+ memset(ptr, 0, total_size);++ dma_queue * q = (dma_queue *) ptr;+ dma_ring * r = (dma_ring *) ((uintptr_t) ptr + offset_r);++ q->ring = r;+ q->nocache = 0;+ q->alias = false;++ r->trace = trace;+ r->vtcm_base = vtcm_base;+ r->vtcm_end = vtcm_base + vtcm_size;+ r->capacity = capacity;+ r->idx_mask = capacity - 1;+ r->push_idx = 0;+ r->pop_idx = 0;++ r->desc = (dma_descriptor_2d *) ((uintptr_t) ptr + offset_desc);+ r->dptr = (dma_ptr *) ((uintptr_t) ptr + offset_dptr);+ r->tail = &r->desc[capacity - 1];++ FARF(HIGH, "dma-queue: capacity %u, unified memory size %zu\n", capacity, total_size);++ return q;+}++void dma_queue_free(dma_queue_t q) {+ (void) q;+}++size_t dma_queue_alias_sizeof(void) {+ return sizeof(dma_queue);+}++dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q, uint8_t nocache) {+ dma_queue * q = (dma_queue *) ptr;+ memset(q, 0, sizeof(dma_queue));++ q->ring = main_q->ring;+ q->nocache = nocache;+ q->alias = true;++ return q;+}++void dma_queue_alias_free(dma_queue_t q) {+ (void) q;+}++void dma_queue_flush(dma_queue_t q) {+ while (dma_queue_pop(q).dst != NULL) ;+}diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.hnew file mode 100644index 0000000000..dba452cfbf--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/dma-queue.h@@ -0,0 +1,396 @@+#ifndef HTP_DMA_H+#define HTP_DMA_H++#include <HAP_farf.h>+#include <hexagon_types.h>+#include <stdbool.h>+#include <stdint.h>+#include "hex-utils.h"++#include "hex-profile.h"++#ifdef __cplusplus+extern "C" {+#endif++// Define the HW descriptor structs here since the ones in HexSDK are a bit out of date+typedef struct dma_descriptor_1d_s {+ void * next;+ uint32_t size:24;+ uint32_t desc_size:2;+ uint32_t dst_comp:1;+ uint32_t src_comp:1;+ uint32_t dst_bypass:1;+ uint32_t src_bypass:1;+ uint32_t order:1;+ uint32_t done:1;+ void * src;+ void * dst;+} dma_descriptor_1d;++#if __HVX_ARCH__ < 75++typedef struct dma_descriptor_2d_s {+ void * next;+ uint32_t reserved0:24;+ uint32_t desc_size:2;+ uint32_t dst_comp:1;+ uint32_t src_comp:1;+ uint32_t dst_bypass:1;+ uint32_t src_bypass:1;+ uint32_t order:1;+ uint32_t done:1;+ void * src;+ void * dst;+ uint32_t desc_type:8;+ uint32_t reserved1:24;+ uint32_t row_size:16;+ uint32_t nrows:16;+ uint32_t src_stride:16;+ uint32_t dst_stride:16;+ uint32_t src_offset:16;+ uint32_t dst_offset:16;+} dma_descriptor_2d;++#else++typedef struct dma_descriptor_2d_s {+ void * next;+ uint32_t dst_stride:24;+ uint32_t desc_size:2;+ uint32_t dst_comp:1;+ uint32_t src_comp:1;+ uint32_t dst_bypass:1;+ uint32_t src_bypass:1;+ uint32_t order:1;+ uint32_t done:1;+ void * src;+ void * dst;+ uint32_t desc_type:8;+ uint32_t reserved0:24;+ uint32_t row_size:24;+ uint32_t nrows_lo:8;+ uint32_t nrows_hi:8;+ uint32_t src_stride:24;+ uint32_t offset:24;+ uint32_t reserved1:8;+} dma_descriptor_2d;++#endif++typedef struct {+ void *dst;+ const void *src;+} dma_ptr;++typedef struct dma_ring_s dma_ring;+struct dma_ring_s {+ dma_descriptor_2d * desc; // descriptor pointers+ dma_descriptor_2d * tail; // tail pointer+ dma_ptr * dptr; // dst/src pointers+ uint32_t push_idx;+ uint32_t pop_idx;+ uint32_t capacity;+ uint32_t idx_mask;+ struct htp_thread_trace * trace;+ uintptr_t vtcm_base;+ uintptr_t vtcm_end;+};++typedef struct dma_queue_s dma_queue;+typedef dma_queue * dma_queue_t;++struct dma_queue_s {+ dma_ring * ring; // Points to the descriptor ring state+ uint8_t nocache; // Queue-specific bypass flag+ bool alias; // When set, dma_queue_delete will not free the ring+};++void dma_queue_flush(dma_queue_t q);++size_t dma_queue_sizeof(size_t capacity);+size_t dma_queue_alignof(void);+dma_queue_t dma_queue_init(void * ptr, size_t capacity, uintptr_t vtcm_base, size_t vtcm_size, struct htp_thread_trace * trace);+void dma_queue_free(dma_queue_t q);++size_t dma_queue_alias_sizeof(void);+dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q, uint8_t nocache);+void dma_queue_alias_free(dma_queue_t q);++// TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead+// but those do not seem to always compiler properly.+static inline void dmstart(void * next) {+ asm volatile(" release(%0):at" : : "r"(next));+ asm volatile(" dmstart(%0)" : : "r"(next));+}++static inline void dmlink(void * cur, void * next) {+ asm volatile(" release(%0):at" : : "r"(next));+ asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next));+}++static inline unsigned int dmpoll(void) {+ unsigned int ret = 0;+ asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory");+ return ret;+}++static inline unsigned int dmwait(void) {+ unsigned int ret = 0;+ asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory");+ return ret;+}++static inline dma_ptr dma_make_ptr(void *dst, const void *src)+{+ dma_ptr p = { dst, src };+ return p;+}++static inline bool dma_is_vtcm(const dma_queue * q, const void * ptr) {+ return (uintptr_t) ptr >= q->ring->vtcm_base && (uintptr_t) ptr < q->ring->vtcm_end;+}++static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t size) {+ dma_ring * r = q->ring;+ if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) {+ FARF(HIGH, "dma-push: queue full\n");+ return false;+ }++ dma_descriptor_1d * desc = (dma_descriptor_1d *) &r->desc[r->push_idx];+ desc->src = (void *) dptr.src;+ desc->dst = (void *) dptr.dst;+ desc->size = size;++ r->dptr[r->push_idx] = dptr;++ if (size) {+ desc->next = NULL;+ desc->desc_size = 0; // 1D mode+ desc->src_bypass = dma_is_vtcm(q, dptr.src) ? 1 : q->nocache;+ desc->dst_bypass = dma_is_vtcm(q, dptr.dst) ? 1 : q->nocache;+ desc->order = 0;+ desc->done = 0;++ htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx);+ dmlink(r->tail, desc);+ r->tail = (dma_descriptor_2d *) desc;+ } else {+ desc->desc_size = 0;+ desc->done = 1;+ }++ r->push_idx = (r->push_idx + 1) & r->idx_mask;+ return true;+}++static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {+ dma_ring * r = q->ring;+ if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) {+ FARF(HIGH, "dma-push: queue full\n");+ return false;+ }++ dma_descriptor_2d * desc = &r->desc[r->push_idx];++ desc->next = NULL;+ desc->reserved0 = 0;+ desc->reserved1 = 0;+ desc->desc_size = 1; // 2d mode+ desc->src_bypass = dma_is_vtcm(q, dptr.src) ? 1 : q->nocache;+ desc->dst_bypass = dma_is_vtcm(q, dptr.dst) ? 1 : q->nocache;+ desc->src_comp = 0;+ desc->dst_comp = 0;+ desc->order = 0;+ desc->done = 0;+ desc->src_stride = src_stride;+ desc->dst_stride = dst_stride;+ desc->src = (void *) dptr.src;+ desc->dst = (void *) dptr.dst;+ desc->row_size = row_size;++#if __HVX_ARCH__ < 75+ desc->desc_type = 0; // 2d (16-bit) mode+ desc->nrows = nrows;+ desc->src_offset = 0;+ desc->dst_offset = 0;+#else+ desc->desc_type = 9; // 2d (24-bit) mode+ desc->nrows_lo = (nrows & 0xff);+ desc->nrows_hi = (nrows >> 8);+ desc->offset = 0;+#endif++ r->dptr[r->push_idx] = dptr;++ if (nrows) {+ htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx);+ dmlink(r->tail, desc);+ r->tail = desc;+ } else {+ desc->done = 1;+ }++ r->push_idx = (r->push_idx + 1) & r->idx_mask;+ return true;+}++static inline dma_ptr dma_queue_pop(dma_queue * q) {+ dma_ring * r = q->ring;+ dma_ptr dptr = { NULL };++ if (r->push_idx == r->pop_idx) {+ return dptr;+ }++ dma_descriptor_2d * desc = &r->desc[r->pop_idx];++ // Wait for desc to complete+ if (!desc->done) {+ while (!desc->done) {+ dmpoll();+ }+ }+ htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx);++ dptr = r->dptr[r->pop_idx];++ r->pop_idx = (r->pop_idx + 1) & r->idx_mask;+ return dptr;+}++static inline dma_ptr dma_queue_pop_nowait(dma_queue * q) {+ dma_ring * r = q->ring;+ dma_ptr dptr = { NULL };++ if (r->push_idx == r->pop_idx) {+ return dptr;+ }++ dptr = r->dptr[r->pop_idx];++ r->pop_idx = (r->pop_idx + 1) & r->idx_mask;+ return dptr;+}++static inline bool dma_queue_empty(dma_queue * q) {+ return q->ring->push_idx == q->ring->pop_idx;+}++static inline uint32_t dma_queue_depth(dma_queue * q) {+ return (q->ring->push_idx - q->ring->pop_idx) & q->ring->idx_mask;+}++static inline uint32_t dma_queue_capacity(dma_queue * q) {+ return q->ring->capacity;+}++#if __HVX_ARCH__ < 75++// Overflow-safe DMA push: all 2d descriptor fields (row_size, nrows, src_stride, dst_stride) are 16-bit, max 65535.+// This version transparently handles values that exceed the 16-bit limit and submits chained DMA transtions.++#define DMA_MAX_FIELD_VAL 65535u++static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {+ // Fast path: everything fits in 16 bits+ if (nrows == 0 || __builtin_expect(+ row_size <= DMA_MAX_FIELD_VAL &&+ nrows <= DMA_MAX_FIELD_VAL &&+ src_stride <= DMA_MAX_FIELD_VAL &&+ dst_stride <= DMA_MAX_FIELD_VAL, 1)) {+ return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows);+ }++ // Contiguous block+ // Use 1d DMA mode which supports sizes up to 24-bits (16MB)+ if (nrows == 1 || (row_size == src_stride && row_size == dst_stride)) {+ size_t total = row_size * nrows;+ return dma_queue_push_single_1d(q, dptr, total);+ }++ // Stride overflow - fall back to row-by-row.+ {+ const uint8_t *src = (const uint8_t *) dptr.src;+ uint8_t *dst = (uint8_t *) dptr.dst;+ for (size_t r = 0; r < nrows; ++r) {+ dma_ptr p = dma_make_ptr(dst + r * dst_stride, src + r * src_stride);+ if (!dma_queue_push_single_1d(q, p, row_size))+ return false;+ if (r + 1 < nrows)+ dma_queue_pop(q);+ }+ return true;+ }+}++#else // HVX_ARCH >= 75++static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {+ // On v75 and up we always use 2d 24-bit mode+ return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows);+}++#endif++static inline bool dma_queue_push_ddr_to_vtcm(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) {+ return dma_queue_push(q, dptr, dst_row_size, src_row_size, src_row_size, nrows);+}++static inline bool dma_queue_push_vtcm_to_ddr(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) {+ return dma_queue_push(q, dptr, dst_row_size, src_row_size, dst_row_size, nrows);+}++#define DMA_CACHE_MAX_SIZE 256U++typedef struct {+ uint8_t *base;+ uint32_t line_size;+ uint32_t capacity;+ uint32_t src[DMA_CACHE_MAX_SIZE];+ uint16_t age[DMA_CACHE_MAX_SIZE];+} dma_cache;++static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_size, uint32_t capacity)+{+ c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity;+ c->base = base;+ c->line_size = line_size;++ for (unsigned i=0; i < c->capacity; i++) {+ c->src[i] = 0;+ c->age[i] = 0;+ }+}++static inline bool dma_cache_push(dma_queue *q, dma_cache *c, const uint8_t * src, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows)+{+ uint32_t o_idx = 0;+ uint16_t o_age = 0;+ uint8_t * dst = 0;++ for (unsigned i=0; i < c->capacity; i++) {+ if (c->src[i] == (uint32_t) src) {+ c->age[i] = 0;+ dst = c->base + (i * c->line_size); nrows = 0; // dummy dma+ } else {+ c->age[i]++;+ if (c->age[i] > o_age) { o_age = c->age[i]; o_idx = i; }+ }+ }+ if (!dst) {+ c->age[o_idx] = 0;+ c->src[o_idx] = (uint32_t) src;+ dst = c->base + o_idx * c->line_size; // normal nrows dma+ return dma_queue_push(q, dma_make_ptr(dst, src), dst_stride, src_stride, row_size, nrows);+ }++ return dma_queue_push_single_1d(q, dma_make_ptr(dst, src), 0);+}++#ifdef __cplusplus+} // extern "C"+#endif++#endif /* HTP_DMA_H */diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.cindex 6f2a643e69..a3bd0bf8f8 100644--- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c+++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c@@ -24,7 +24,7 @@#include "hvx-reduce.h"#include "hvx-flash-attn.h"#include "htp-vtcm.h"-#include "worker-pool.h"+#include "work-queue.h"#define GGML_COMMON_DECL_C#include "ggml-common.h"@@ -204,7 +204,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *if (ir0 >= ir1) return;- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];dma_queue * dma = octx->ctx->dma[ith];@@ -486,7 +486,7 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data)return;}- struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL;+ struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start));hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles, (const __fp16 *) args->curr_k, total_rows, factx->DK,args->src_stride, start, end);@@ -494,7 +494,7 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data)}static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_rows, size_t src_stride, void * curr_k, uint32_t kv_start) {- worker_pool_context_t wp = factx->octx->ctx->worker_pool;+ work_queue_t wp = factx->octx->ctx->work_queue;uint32_t n = 1;if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) {n = factx->n_threads;@@ -502,7 +502,7 @@ static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_rowuint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2);fa_k_int_args_t args = { factx, kv_rows, src_stride, curr_k, kv_start, rows_per_t };if (n > 1) {- worker_pool_run_func(wp, fa_k_interleave_thread, &args, n);+ work_queue_run(wp, fa_k_interleave_thread, &args, n);} else {fa_k_interleave_thread(1, 0, &args);}@@ -534,7 +534,7 @@ static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data)__fp16 * v_tiles_dst = (__fp16 *) args->v_tiles_dst;- struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL;+ struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start));hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV,args->src_stride, (uint32_t) args->n_col_tiles, start, end);@@ -548,7 +548,7 @@ static void fa_phase_v_interleave(struct hmx_fa_context * factx,void * v_tiles_dst,size_t n_col_tiles,uint32_t kv_start) {- worker_pool_context_t wp = factx->octx->ctx->worker_pool;+ work_queue_t wp = factx->octx->ctx->work_queue;uint32_t n = 1;if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) {n = factx->n_threads;@@ -556,7 +556,7 @@ static void fa_phase_v_interleave(struct hmx_fa_context * factx,uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2);fa_v_int_args_t args = { factx, kv_rows, src_stride, v_src, v_tiles_dst, n_col_tiles, kv_start, rows_per_t };if (n > 1) {- worker_pool_run_func(wp, fa_v_interleave_thread, &args, n);+ work_queue_run(wp, fa_v_interleave_thread, &args, n);} else {fa_v_interleave_thread(1, 0, &args);}@@ -589,7 +589,7 @@ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) {const size_t start = (size_t) i * rows_per_t;const size_t end = hex_smin(start + rows_per_t, factx->g_br);- struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL;+ struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start));// Parallel initialization of per-block state@@ -720,7 +720,7 @@ static void fa_phase_q_load(struct hmx_fa_context * factx,uint32_t kv_head,uint32_t ib3,size_t n_rows_g) {- worker_pool_context_t wp = factx->octx->ctx->worker_pool;+ work_queue_t wp = factx->octx->ctx->work_queue;uint32_t n = 1;if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) {n = factx->n_threads;@@ -739,7 +739,7 @@ static void fa_phase_q_load(struct hmx_fa_context * factx,args.q_transposed = q->nb[1] < q->nb[2];atomic_init(&args.barrier, n);if (n > 1) {- worker_pool_run_func(wp, fa_q_load_thread, &args, n);+ work_queue_run(wp, fa_q_load_thread, &args, n);} else {fa_q_load_thread(1, 0, &args);}@@ -772,7 +772,7 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) {return;}- struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL;+ struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));const struct htp_tensor * dst = args->dst;@@ -820,7 +820,7 @@ static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) {return;}- struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL;+ struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));const struct htp_tensor * dst = args->dst;@@ -862,7 +862,7 @@ static void fa_phase_o_store(struct hmx_fa_context * factx,uint32_t kv_head,uint32_t ib3,size_t n_rows_g) {- worker_pool_context_t wp = factx->octx->ctx->worker_pool;+ work_queue_t wp = factx->octx->ctx->work_queue;uint32_t n = 1;if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) {n = factx->n_threads;@@ -871,7 +871,7 @@ static void fa_phase_o_store(struct hmx_fa_context * factx,fa_o_store_args_t args = { factx, dst, o_tile_src, q_start, kv_head, ib3, n_rows_g, rows_per_t };worker_callback_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16;if (n > 1) {- worker_pool_run_func(wp, store_fn, &args, n);+ work_queue_run(wp, store_fn, &args, n);} else {store_fn(1, 0, &args);}@@ -930,7 +930,7 @@ static inline void fa_softmax_impl(return;}- struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL;+ struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64));// Per-thread row scratch: thread i uses bufs at offset i * 2 * stride@@ -1290,7 +1290,7 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx,fa_softmax_args_t * sargs,size_t n_row_tiles,size_t n_row_tiles_g_br) {- worker_pool_context_t wp = factx->octx->ctx->worker_pool;+ work_queue_t wp = factx->octx->ctx->work_queue;const size_t n_row_vec_cnt = hmx_ceil_div(sargs->n_rows_g, 64);worker_callback_t softmax_fn = fa_softmax_thread;@@ -1307,7 +1307,7 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx,if (factx->n_threads > 1 && n_row_vec_cnt >= 2) {uint32_t n_use = (uint32_t) hex_smin((size_t) factx->n_threads, n_row_vec_cnt);sargs->thread_div = init_fastdiv_values(n_use);- worker_pool_run_func(wp, softmax_fn, sargs, n_use);+ work_queue_run(wp, softmax_fn, sargs, n_use);} else {softmax_fn(1, 0, sargs);}@@ -1519,8 +1519,8 @@ static void fa_pop_mask_dma_gqa(dma_queue * dma, uint32_t G) {// ============================================================================int hmx_flash_attn_ext(struct htp_ops_context * octx) {- struct htp_thread_trace * tr_hvx = octx->ctx ? &octx->ctx->trace[0] : NULL;- struct htp_thread_trace * tr_hmx = octx->ctx ? &octx->ctx->trace[HTP_MAX_NTHREADS] : NULL;+ struct htp_thread_trace * tr_hvx = &octx->ctx->trace[0];+ struct htp_thread_trace * tr_hmx = &octx->ctx->trace[HTP_MAX_NTHREADS];const struct htp_tensor * q = octx->src[0];const struct htp_tensor * k = octx->src[1];const struct htp_tensor * v = octx->src[2];@@ -1735,7 +1735,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {const size_t k_src_stride = size_k_row_padded / sizeof(__fp16);const size_t v_src_stride = size_v_row_padded / sizeof(__fp16);- struct hmx_queue * hmx_q = ctx->hmx_queue;+ hmx_queue_t hmx_q = ctx->hmx_queue;if (factx.pipeline) {// Pipeline path@@ -2084,7 +2084,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {}if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {- worker_pool_run_func(octx->ctx->worker_pool, flash_attn_ext_f16_thread, &factx, octx->n_threads);+ work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, octx->n_threads);}return HTP_STATUS_OK;diff --git a/ggml/src/ggml-hexagon/htp/hex-bitmap.h b/ggml/src/ggml-hexagon/htp/hex-bitmap.hnew file mode 100644index 0000000000..140898852a--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/hex-bitmap.h@@ -0,0 +1,24 @@+#ifndef HEX_BITMAP_H+#define HEX_BITMAP_H++#include <stdint.h>+#include <stdbool.h>+#include <string.h>++static inline void bitmap_set(uint32_t * bitmap, uint32_t idx) {+ bitmap[idx / 32] |= (1U << (idx % 32));+}++static inline void bitmap_clear(uint32_t * bitmap, uint32_t idx) {+ bitmap[idx / 32] &= ~(1U << (idx % 32));+}++static inline bool bitmap_test(const uint32_t * bitmap, uint32_t idx) {+ return (bitmap[idx / 32] & (1U << (idx % 32))) != 0;+}++static inline void bitmap_reset(uint32_t * bitmap, size_t size_in_bits) {+ memset(bitmap, 0, ((size_in_bits + 31) / 32) * sizeof(uint32_t));+}++#endif // HEX_BITMAP_Hdiff --git a/ggml/src/ggml-hexagon/htp/hex-dma.c b/ggml/src/ggml-hexagon/htp/hex-dma.cdeleted file mode 100644index b66e2d2603..0000000000--- a/ggml/src/ggml-hexagon/htp/hex-dma.c+++ /dev/null@@ -1,63 +0,0 @@-#include "hex-dma.h"--#include <stdbool.h>-#include <stdlib.h>-#include <string.h>--#pragma clang diagnostic ignored "-Wunused-function"--static inline uint32_t pow2_ceil(uint32_t x) {- if (x <= 1) {- return 1;- }- int p = 2;- x--;- while (x >>= 1) {- p <<= 1;- }- return p;-}--dma_queue * dma_queue_create(size_t capacity) {- dma_queue * q = (dma_queue *) memalign(32, sizeof(dma_queue));- if (q == NULL) {- FARF(ERROR, "%s: failed to allocate DMA queue\n", __FUNCTION__);- return NULL;- }-- capacity = pow2_ceil(capacity);-- memset(q, 0, sizeof(dma_queue));- q->capacity = capacity;- q->idx_mask = capacity - 1;-- q->desc = (dma_descriptor_2d *) memalign(64, capacity * sizeof(dma_descriptor_2d));- memset(q->desc, 0, capacity * sizeof(dma_descriptor_2d));-- q->dptr = (dma_ptr *) memalign(4, capacity * sizeof(dma_ptr));- memset(q->dptr, 0, capacity * sizeof(dma_ptr));-- q->tail = &q->desc[capacity - 1];-- if (!q->desc && !q->dptr) {- FARF(ERROR, "%s: failed to allocate DMA queue items\n", __FUNCTION__);- return NULL;- }-- FARF(HIGH, "dma-queue: capacity %u\n", capacity);-- return q;-}--void dma_queue_delete(dma_queue * q) {- if (!q) {- return;- }- free(q->desc);- free(q->dptr);- free(q);-}--void dma_queue_flush(dma_queue * q) {- while (dma_queue_pop(q).dst != NULL) ;-}diff --git a/ggml/src/ggml-hexagon/htp/hex-dma.h b/ggml/src/ggml-hexagon/htp/hex-dma.hindex 98fcc9fda6..9e9a5f9502 100644--- a/ggml/src/ggml-hexagon/htp/hex-dma.h+++ b/ggml/src/ggml-hexagon/htp/hex-dma.h@@ -1,375 +1,2 @@-#ifndef HTP_DMA_H-#define HTP_DMA_H--#include <HAP_farf.h>-#include <hexagon_types.h>-#include <stdbool.h>-#include <stdint.h>-#include "hex-utils.h"--#include "hex-profile.h"--#ifdef __cplusplus-extern "C" {-#endif--// Define the HW descriptor structs here since the ones in HexSDK are a bit out of date-typedef struct dma_descriptor_1d_s {- void * next;- uint32_t size:24;- uint32_t desc_size:2;- uint32_t dst_comp:1;- uint32_t src_comp:1;- uint32_t dst_bypass:1;- uint32_t src_bypass:1;- uint32_t order:1;- uint32_t done:1;- void * src;- void * dst;-} dma_descriptor_1d;--#if __HVX_ARCH__ < 75--typedef struct dma_descriptor_2d_s {- void * next;- uint32_t reserved0:24;- uint32_t desc_size:2;- uint32_t dst_comp:1;- uint32_t src_comp:1;- uint32_t dst_bypass:1;- uint32_t src_bypass:1;- uint32_t order:1;- uint32_t done:1;- void * src;- void * dst;- uint32_t desc_type:8;- uint32_t reserved1:24;- uint32_t row_size:16;- uint32_t nrows:16;- uint32_t src_stride:16;- uint32_t dst_stride:16;- uint32_t src_offset:16;- uint32_t dst_offset:16;-} dma_descriptor_2d;--#else--typedef struct dma_descriptor_2d_s {- void * next;- uint32_t dst_stride:24;- uint32_t desc_size:2;- uint32_t dst_comp:1;- uint32_t src_comp:1;- uint32_t dst_bypass:1;- uint32_t src_bypass:1;- uint32_t order:1;- uint32_t done:1;- void * src;- void * dst;- uint32_t desc_type:8;- uint32_t reserved0:24;- uint32_t row_size:24;- uint32_t nrows_lo:8;- uint32_t nrows_hi:8;- uint32_t src_stride:24;- uint32_t offset:24;- uint32_t reserved1:8;-} dma_descriptor_2d;--#endif--typedef struct {- void *dst;- const void *src;-} dma_ptr;--typedef struct {- dma_descriptor_2d * desc; // descriptor pointers- dma_descriptor_2d * tail; // tail pointer- dma_ptr * dptr; // dst/src pointers- uint32_t push_idx;- uint32_t pop_idx;- uint32_t capacity;- uint32_t idx_mask;- struct htp_thread_trace * trace;-} dma_queue;--dma_queue * dma_queue_create(size_t capacity);-void dma_queue_delete(dma_queue * q);-void dma_queue_flush(dma_queue * q);--// TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead-// but those do not seem to always compiler properly.-static inline void dmstart(void * next) {- asm volatile(" release(%0):at" : : "r"(next));- asm volatile(" dmstart(%0)" : : "r"(next));-}--static inline void dmlink(void * cur, void * next) {- asm volatile(" release(%0):at" : : "r"(next));- asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next));-}--static inline unsigned int dmpoll(void) {- unsigned int ret = 0;- asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory");- return ret;-}--static inline unsigned int dmwait(void) {- unsigned int ret = 0;- asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory");- return ret;-}--static inline dma_ptr dma_make_ptr(void *dst, const void *src)-{- dma_ptr p = { dst, src };- return p;-}--static const uint32_t dma_src_l2_bypass_on = 1;-static const uint32_t dma_dst_l2_bypass_on = 1;--static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t size) {- if (((q->push_idx + 1) & q->idx_mask) == q->pop_idx) {- FARF(HIGH, "dma-push: queue full\n");- return false;- }-- dma_descriptor_1d * desc = (dma_descriptor_1d *) &q->desc[q->push_idx];- desc->src = (void *) dptr.src;- desc->dst = (void *) dptr.dst;- desc->size = size;-- q->dptr[q->push_idx] = dptr;-- if (size) {- desc->next = NULL;- desc->desc_size = 0; // 1D mode- desc->src_bypass = dma_src_l2_bypass_on;- desc->dst_bypass = dma_dst_l2_bypass_on;- desc->order = 0;- desc->done = 0;-- htp_trace_event_start(q->trace, HTP_TRACE_EVT_DMA, q->push_idx);- dmlink(q->tail, desc);- q->tail = (dma_descriptor_2d *) desc;- } else {- desc->desc_size = 0;- desc->done = 1;- }-- q->push_idx = (q->push_idx + 1) & q->idx_mask;- return true;-}--static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {- if (((q->push_idx + 1) & q->idx_mask) == q->pop_idx) {- FARF(HIGH, "dma-push: queue full\n");- return false;- }-- dma_descriptor_2d * desc = &q->desc[q->push_idx];-- desc->next = NULL;- desc->reserved0 = 0;- desc->reserved1 = 0;- desc->desc_size = 1; // 2d mode- desc->src_bypass = dma_src_l2_bypass_on;- desc->dst_bypass = dma_dst_l2_bypass_on;- desc->src_comp = 0;- desc->dst_comp = 0;- desc->order = 0;- desc->done = 0;- desc->src_stride = src_stride;- desc->dst_stride = dst_stride;- desc->src = (void *) dptr.src;- desc->dst = (void *) dptr.dst;- desc->row_size = row_size;--#if __HVX_ARCH__ < 75- desc->desc_type = 0; // 2d (16-bit) mode- desc->nrows = nrows;- desc->src_offset = 0;- desc->dst_offset = 0;-#else- desc->desc_type = 9; // 2d (24-bit) mode- desc->nrows_lo = (nrows & 0xff);- desc->nrows_hi = (nrows >> 8);- desc->offset = 0;-#endif-- q->dptr[q->push_idx] = dptr;-- if (nrows) {- htp_trace_event_start(q->trace, HTP_TRACE_EVT_DMA, q->push_idx);- dmlink(q->tail, desc);- q->tail = desc;- } else {- desc->done = 1;- }-- // FARF(ERROR, "dma-push: i %u row-size %u nrows %d dst %p src %p\n", q->push_idx, row_size, nrows, dptr.dst, dptr.src);- q->push_idx = (q->push_idx + 1) & q->idx_mask;- return true;-}--static inline dma_ptr dma_queue_pop(dma_queue * q) {- dma_ptr dptr = { NULL };-- if (q->push_idx == q->pop_idx) {- return dptr;- }-- dma_descriptor_2d * desc = &q->desc[q->pop_idx];-- // Wait for desc to complete- if (!desc->done) {- while (!desc->done) {- dmpoll();- }- }- htp_trace_event_stop(q->trace, HTP_TRACE_EVT_DMA, q->pop_idx);-- dptr = q->dptr[q->pop_idx];-- // FARF(ERROR, "dma-pop: i %u dst %p src %p\n", q->pop_idx, dptr.dst, dptr.src);- q->pop_idx = (q->pop_idx + 1) & q->idx_mask;- return dptr;-}--static inline dma_ptr dma_queue_pop_nowait(dma_queue * q) {- dma_ptr dptr = { NULL };-- if (q->push_idx == q->pop_idx) {- return dptr;- }-- dptr = q->dptr[q->pop_idx];-- // FARF(ERROR, "dma-pop-nowait: i %u dst %p src %p\n", q->pop_idx, dptr.dst, dptr.src);- q->pop_idx = (q->pop_idx + 1) & q->idx_mask;- return dptr;-}--static inline bool dma_queue_empty(dma_queue * q) {- return q->push_idx == q->pop_idx;-}--static inline uint32_t dma_queue_depth(dma_queue * q) {- return (q->push_idx - q->pop_idx) & q->idx_mask;-}--static inline uint32_t dma_queue_capacity(dma_queue * q) {- return q->capacity;-}--#if __HVX_ARCH__ < 75--// Overflow-safe DMA push: all 2d descriptor fields (row_size, nrows, src_stride, dst_stride) are 16-bit, max 65535.-// This version transparently handles values that exceed the 16-bit limit and submits chained DMA transtions.--#define DMA_MAX_FIELD_VAL 65535u--static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {- // Fast path: everything fits in 16 bits- if (nrows == 0 || __builtin_expect(- row_size <= DMA_MAX_FIELD_VAL &&- nrows <= DMA_MAX_FIELD_VAL &&- src_stride <= DMA_MAX_FIELD_VAL &&- dst_stride <= DMA_MAX_FIELD_VAL, 1)) {- return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows);- }-- // Contiguous block- // Use 1d DMA mode which supports sizes up to 24-bits (16MB)- if (nrows == 1 || (row_size == src_stride && row_size == dst_stride)) {- size_t total = row_size * nrows;- return dma_queue_push_single_1d(q, dptr, total);- }-- // Stride overflow — fall back to row-by-row.- {- const uint8_t *src = (const uint8_t *) dptr.src;- uint8_t *dst = (uint8_t *) dptr.dst;- for (size_t r = 0; r < nrows; ++r) {- dma_ptr p = dma_make_ptr(dst + r * dst_stride, src + r * src_stride);- if (!dma_queue_push_single_1d(q, p, row_size))- return false;- if (r + 1 < nrows)- dma_queue_pop(q);- }- return true;- }-}--#else // HVX_ARCH >= 75--static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {- // On v75 and up we always use 2d 24-bit mode- return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows);-}--#endif--static inline bool dma_queue_push_ddr_to_vtcm(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) {- return dma_queue_push(q, dptr, dst_row_size, src_row_size, src_row_size, nrows);-}--static inline bool dma_queue_push_vtcm_to_ddr(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) {- return dma_queue_push(q, dptr, dst_row_size, src_row_size, dst_row_size, nrows);-}--#define DMA_CACHE_MAX_SIZE 256U--typedef struct {- uint8_t *base;- uint32_t line_size;- uint32_t capacity;- uint32_t src[DMA_CACHE_MAX_SIZE];- uint16_t age[DMA_CACHE_MAX_SIZE];-} dma_cache;--static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_size, uint32_t capacity)-{- c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity;- c->base = base;- c->line_size = line_size;-- for (unsigned i=0; i < c->capacity; i++) {- c->src[i] = 0;- c->age[i] = 0;- }-}--static inline bool dma_cache_push(dma_queue *q, dma_cache *c, const uint8_t * src, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows)-{- uint32_t o_idx = 0;- uint16_t o_age = 0;- uint8_t * dst = 0;-- for (unsigned i=0; i < c->capacity; i++) {- if (c->src[i] == (uint32_t) src) {- c->age[i] = 0;- dst = c->base + (i * c->line_size); nrows = 0; // dummy dma- } else {- c->age[i]++;- if (c->age[i] > o_age) { o_age = c->age[i]; o_idx = i; }- }- }- if (!dst) {- c->age[o_idx] = 0;- c->src[o_idx] = (uint32_t) src;- dst = c->base + o_idx * c->line_size; // normal nrows dma- return dma_queue_push(q, dma_make_ptr(dst, src), dst_stride, src_stride, row_size, nrows);- }-- return dma_queue_push_single_1d(q, dma_make_ptr(dst, src), 0);-}--#ifdef __cplusplus-} // extern "C"-#endif--#endif /* HTP_DMA_H */+#pragma once+#include "dma-queue.h"diff --git a/ggml/src/ggml-hexagon/htp/hex-profile.h b/ggml/src/ggml-hexagon/htp/hex-profile.hindex 8a37a4a066..a26961fc93 100644--- a/ggml/src/ggml-hexagon/htp/hex-profile.h+++ b/ggml/src/ggml-hexagon/htp/hex-profile.h@@ -44,11 +44,11 @@ struct htp_thread_trace {};static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) {- if (tr && tr->events && tr->count < tr->max_events) {- uint32_t idx = tr->count;- tr->events[idx].id = id;- tr->events[idx].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0);- tr->events[idx].cycles = (uint32_t) hex_get_cycles();+ if (tr->count < tr->max_events) {+ uint32_t i = tr->count;+ tr->events[i].id = id;+ tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0);+ tr->events[i].cycles = (uint32_t) hex_get_cycles();tr->count++;}}diff --git a/ggml/src/ggml-hexagon/htp/hex-utils.h b/ggml/src/ggml-hexagon/htp/hex-utils.hindex 07930bef6e..93e87efcb4 100644--- a/ggml/src/ggml-hexagon/htp/hex-utils.h+++ b/ggml/src/ggml-hexagon/htp/hex-utils.h@@ -30,21 +30,26 @@ static inline void hex_l2fetch(const void * p, uint32_t width, uint32_t stride,Q6_l2fetch_AP((void *) p, control);}-#define HEX_L2_LINE_SIZE 64-#define HEX_L2_FLUSH_SIZE (128 * 1024)+static inline void hex_l2fetch_block(const void * addr, size_t size) {+ if (size == 0) return;+ const uint32_t width = 16384; // 16KB rows+ const uint32_t height = (size + width - 1) / width;+ hex_l2fetch(addr, width, width, height);+}++#define HEX_L2_LINE_SIZE 128+#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (lines per loop iteration)+#define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024)+#define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024)static inline void hex_l2flush(void * addr, size_t size) {- if (size > HEX_L2_FLUSH_SIZE) {- qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);- } else {- const uint32_t s = (uint32_t) addr;- const uint32_t e = s + size;- for (uint32_t i = s; i < e; i += HEX_L2_LINE_SIZE * 4) {- Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 0);- Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 1);- Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 2);- Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 3);- }+ const uint32_t s = ((uint32_t) addr) & ~(HEX_L2_LINE_SIZE - 1);+ const uint32_t e = (((uint32_t) addr) + size + HEX_L2_LINE_SIZE - 1) & ~(HEX_L2_LINE_SIZE - 1);+ for (uint32_t i = s; i < e; i += HEX_L2_BLOCK_SIZE) {+ Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 0);+ Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 1);+ Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 2);+ Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 3);}}diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.hindex 740a8f87d6..a4efb1b004 100644--- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h+++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h@@ -1005,10 +1005,62 @@ static void transfer_activation_row_pair_fp32_to_fp16(}}+static void transfer_activation_row_pair_fp32_to_fp16_col_chunk(+ __fp16 *restrict vtcm_dst,+ const float *restrict row0, // offset by c_first+ const float *restrict row1, // offset by c_first+ uint32_t r,+ uint32_t k_block,+ uint32_t c_first,+ uint32_t c_len,+ uint32_t k_chunk_valid,+ bool row0_valid,+ bool row1_valid) {++ uint32_t r0 = r / HTP_MM_HMX_TILE_N_ROWS; // tile row index+ uint32_t r1 = r % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx++ uint32_t c = 0;+ for (; c + 32 <= k_chunk_valid; c += 32) {+ HVX_Vector v0 = Q6_V_vzero();+ HVX_Vector v1 = Q6_V_vzero();+ if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c);+ if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c);++ HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);++ uint32_t c0 = (c_first + c) / HTP_MM_HMX_TILE_N_COLS; // tile column index+ uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;++ HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);+ tile[r1 / 2] = v_out;+ }+ if (c < c_len) {+ HVX_Vector v0 = Q6_V_vzero();+ HVX_Vector v1 = Q6_V_vzero();+ if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c);+ if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c);++ uint32_t rem = (k_chunk_valid > c) ? (k_chunk_valid - c) : 0;+ HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0);+ v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero());+ v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero());++ HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);++ uint32_t c0 = (c_first + c) / HTP_MM_HMX_TILE_N_COLS; // tile column index+ uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;++ HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);+ tile[r1 / 2] = v_out;+ }+}+static void transfer_activation_chunk_fp32_to_fp16_gathered(__fp16 *restrict vtcm_dst,const float *restrict src,uint32_t start_row,+ uint32_t vtcm_start_row,uint32_t n_rows,uint32_t k_block,const struct mmid_row_mapping *matrix_rows,@@ -1029,8 +1081,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered(for (r = 0; r < n_rows_tiled; r += 2) {uint32_t r_idx0 = start_row + r + 0;uint32_t r_idx1 = start_row + r + 1;- uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index- uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx+ uint32_t lr = vtcm_start_row + r; // vtcm-local row+ uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index+ uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idxstruct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + r_idx0];struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + r_idx1];@@ -1073,9 +1126,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered(}for (; r < n_rows_padded; r += 2) {- uint32_t r_idx0 = start_row + r;- uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index- uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx+ uint32_t lr = vtcm_start_row + r; // vtcm-local row+ uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index+ uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idxconst bool row0_valid = (start_row + r + 0) < cne1;const bool row1_valid = (start_row + r + 1) < cne1;@@ -1135,6 +1188,7 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered_flat(__fp16 *restrict vtcm_dst,const float *restrict src,uint32_t start_row,+ uint32_t vtcm_start_row,uint32_t n_rows,uint32_t k_block,const struct mmid_row_mapping *matrix_rows,@@ -1152,8 +1206,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered_flat(for (r = 0; r < n_rows_tiled; r += 2) {uint32_t r_idx0 = start_row + r + 0;uint32_t r_idx1 = start_row + r + 1;- uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index- uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx+ uint32_t lr = vtcm_start_row + r; // vtcm-local row+ uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index+ uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idxstruct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + r_idx0];struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + r_idx1];@@ -1193,9 +1248,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered_flat(}for (; r < n_rows_padded; r += 2) {- uint32_t r_idx0 = start_row + r;- uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index- uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx+ uint32_t lr = vtcm_start_row + r; // vtcm-local row+ uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index+ uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idxconst bool row0_valid = (start_row + r + 0) < cne1;const bool row1_valid = (start_row + r + 1) < cne1;@@ -1253,6 +1308,7 @@ static void transfer_output_chunk_fp16_to_fp32_scattered(float *restrict dst,const __fp16 *restrict vtcm_src,uint32_t start_row,+ uint32_t vtcm_start_row,uint32_t n_rows,uint32_t n_cols,const struct mmid_row_mapping *matrix_rows,@@ -1269,8 +1325,9 @@ static void transfer_output_chunk_fp16_to_fp32_scattered(for (size_t r = 0; r < n_rows; r += 2) {uint32_t r_idx0 = start_row + r + 0;uint32_t r_idx1 = start_row + r + 1;- const size_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS;- const size_t r1 = (r_idx0 % HTP_MM_HMX_TILE_N_ROWS) / 2; // index of the row pair within the tile+ uint32_t lr = vtcm_start_row + r; // vtcm-local row+ const size_t r0 = (lr / HTP_MM_HMX_TILE_N_ROWS);+ const size_t r1 = (lr % HTP_MM_HMX_TILE_N_ROWS) / 2; // index of the row pair within the tileconst __fp16 *row_base = vtcm_src + r0 * tile_row_stride;if (r_idx0 >= cne1) break;diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.c b/ggml/src/ggml-hexagon/htp/hmx-queue.cindex 3add542bae..c369d3dd23 100644--- a/ggml/src/ggml-hexagon/htp/hmx-queue.c+++ b/ggml/src/ggml-hexagon/htp/hmx-queue.c@@ -14,7 +14,7 @@#define QURT_LOWEST_PRIO (254)-static inline void hmx_lock(struct hmx_queue *q)+static inline void hmx_lock(hmx_queue_t q){if (!q->hmx_locked) {HAP_compute_res_hmx_lock(q->hap_rctx);@@ -22,7 +22,7 @@ static inline void hmx_lock(struct hmx_queue *q)}}-static inline void hmx_unlock(struct hmx_queue *q)+static inline void hmx_unlock(hmx_queue_t q){if (q->hmx_locked) {HAP_compute_res_hmx_unlock(q->hap_rctx);@@ -30,7 +30,7 @@ static inline void hmx_unlock(struct hmx_queue *q)}}-static inline void hmx_queue_process(struct hmx_queue *q, bool* killed) {+static inline void hmx_queue_process(hmx_queue_t q, bool* killed) {unsigned int ir = atomic_load(&q->idx_read);while (ir != atomic_load(&q->idx_write)) {@@ -61,7 +61,7 @@ static inline void hmx_queue_process(struct hmx_queue *q, bool* killed) {}static void hmx_queue_thread(void * arg) {- struct hmx_queue * q = (struct hmx_queue *) arg;+ hmx_queue_t q = (hmx_queue_t) arg;FARF(HIGH, "hmx-queue-thread: started");@@ -93,34 +93,41 @@ static void hmx_queue_thread(void * arg) {FARF(HIGH, "hmx-queue-thread: stopped");}-struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx) {+size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size) {capacity = hex_ceil_pow2(capacity);+ size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE);+ size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE);+ size_t size_stack = stack_size;+ return size_q + size_desc + size_stack;+}++size_t hmx_queue_alignof(void) {+ return HEX_L2_LINE_SIZE;+}++hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace) {+ capacity = hex_ceil_pow2(capacity);+ size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE);+ size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE);++ uint8_t * block = (uint8_t *) ptr;++ hmx_queue_t q = (hmx_queue_t) block; block += size_q;+ memset(q, 0, sizeof(struct hmx_queue_s));- struct hmx_queue * q = (struct hmx_queue *) memalign(32, sizeof(struct hmx_queue));- if (q == NULL) {- FARF(ERROR, "%s: failed to allocate DMA queue\n", __FUNCTION__);- return NULL;- }- memset(q, 0, sizeof(struct hmx_queue));q->capacity = capacity;q->idx_mask = capacity - 1;q->hap_rctx = hap_rctx;+ q->external_mem = true;- q->desc = (struct hmx_queue_desc *) memalign(64, capacity * sizeof(struct hmx_queue_desc));- if (!q->desc) {- FARF(ERROR, "hmx-queue: failed to allocate HMX queue descriptors\n");- return NULL;- }+ q->desc = (struct hmx_queue_desc *) block; block += size_desc;memset(q->desc, 0, capacity * sizeof(struct hmx_queue_desc));- const size_t stack_size = HMX_QUEUE_THREAD_STACK_SIZE;- q->stack = (unsigned char *) memalign(64, stack_size);- if (!q->stack) {- FARF(ERROR, "hmx-queue: thread stack allocation failed (%zu bytes)", stack_size);- return NULL;- }+ q->stack = block;memset(q->stack, 0, stack_size);+ q->trace = trace;+// Match caller thread priority (same pattern as worker-pool.c).int prio = qurt_thread_get_priority(qurt_thread_get_id());if (prio < 1) {@@ -148,7 +155,7 @@ struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx) {return q;}-void hmx_queue_delete(struct hmx_queue * q) {+void hmx_queue_free(hmx_queue_t q) {if (!q) {return;}@@ -160,8 +167,4 @@ void hmx_queue_delete(struct hmx_queue * q) {int status;qurt_thread_join(q->thread, &status);-- free(q->desc);- free(q->stack);- free(q);}diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.h b/ggml/src/ggml-hexagon/htp/hmx-queue.hindex b176fa1796..c2b1859a28 100644--- a/ggml/src/ggml-hexagon/htp/hmx-queue.h+++ b/ggml/src/ggml-hexagon/htp/hmx-queue.h@@ -17,8 +17,6 @@extern "C" {#endif-#define HMX_QUEUE_THREAD_STACK_SIZE (16 * 1024)-#if __HVX_ARCH__ > 79#define HMX_QUEUE_POLL_COUNT 2000#else@@ -41,7 +39,7 @@ struct hmx_queue_desc {atomic_uint done;};-struct hmx_queue {+struct hmx_queue_s {struct hmx_queue_desc * desc;atomic_uint idx_write; // updated by producer (push)atomic_uint idx_read; // updated by consumer (process)@@ -55,19 +53,24 @@ struct hmx_queue {uint32_t hap_rctx;bool hmx_locked;struct htp_thread_trace * trace;+ bool external_mem; // memory owned externally};-struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx);-void hmx_queue_delete(struct hmx_queue * q);+typedef struct hmx_queue_s * hmx_queue_t;++size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size);+size_t hmx_queue_alignof(void);+hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace);+void hmx_queue_free(hmx_queue_t q);static inline struct hmx_queue_desc hmx_queue_make_desc(hmx_queue_func func, void * data) {struct hmx_queue_desc d = { func, data };return d;}-static inline bool hmx_queue_push(struct hmx_queue * q, struct hmx_queue_desc d) {+static inline bool hmx_queue_push(hmx_queue_t q, struct hmx_queue_desc d) {unsigned int ir = atomic_load(&q->idx_read);- unsigned int iw = q->idx_write;+ unsigned int iw = atomic_load(&q->idx_write);if (((iw + 1) & q->idx_mask) == ir) {FARF(HIGH, "hmx-queue-push: queue is full\n");@@ -87,25 +90,25 @@ static inline bool hmx_queue_push(struct hmx_queue * q, struct hmx_queue_desc d)return true;}-static inline bool hmx_queue_signal(struct hmx_queue *q, enum hmx_queue_signal sig) {+static inline bool hmx_queue_signal(hmx_queue_t q, enum hmx_queue_signal sig) {return hmx_queue_push(q, hmx_queue_make_desc((hmx_queue_func) sig, NULL));}-static inline bool hmx_queue_empty(struct hmx_queue * q) {- return q->idx_pop == q->idx_write;+static inline bool hmx_queue_empty(hmx_queue_t q) {+ return q->idx_pop == atomic_load(&q->idx_write);}-static inline uint32_t hmx_queue_depth(struct hmx_queue * q) {- return (q->idx_read - q->idx_read) & q->idx_mask;+static inline uint32_t hmx_queue_depth(hmx_queue_t q) {+ return (atomic_load(&q->idx_write) - atomic_load(&q->idx_read)) & q->idx_mask;}-static inline uint32_t hmx_queue_capacity(struct hmx_queue * q) {+static inline uint32_t hmx_queue_capacity(hmx_queue_t q) {return q->capacity;}-static inline struct hmx_queue_desc hmx_queue_pop_one(struct hmx_queue * q) {+static inline struct hmx_queue_desc hmx_queue_pop_one(hmx_queue_t q) {unsigned int ip = q->idx_pop;- unsigned int iw = q->idx_write;+ unsigned int iw = atomic_load(&q->idx_write);struct hmx_queue_desc rd = { NULL, NULL };if (ip == iw) {@@ -126,7 +129,7 @@ static inline struct hmx_queue_desc hmx_queue_pop_one(struct hmx_queue * q) {return rd;}-static inline struct hmx_queue_desc hmx_queue_pop(struct hmx_queue * q) {+static inline struct hmx_queue_desc hmx_queue_pop(hmx_queue_t q) {while (1) {struct hmx_queue_desc d = hmx_queue_pop_one(q);@@ -138,15 +141,15 @@ static inline struct hmx_queue_desc hmx_queue_pop(struct hmx_queue * q) {}}-static inline void hmx_queue_flush(struct hmx_queue * q) {+static inline void hmx_queue_flush(hmx_queue_t q) {while (hmx_queue_pop_one(q).func != NULL) ;}-static inline void hmx_queue_wakeup(struct hmx_queue * q) {+static inline void hmx_queue_wakeup(hmx_queue_t q) {hmx_queue_signal(q, HMX_QUEUE_WAKEUP);}-static inline void hmx_queue_suspend(struct hmx_queue *q) {+static inline void hmx_queue_suspend(hmx_queue_t q) {hmx_queue_signal(q, HMX_QUEUE_SUSPEND);}diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.hindex e13103fb18..c4b60eb264 100644--- a/ggml/src/ggml-hexagon/htp/htp-ctx.h+++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h@@ -5,7 +5,8 @@#include "hmx-queue.h"#include "htp-ops.h"#include "hex-profile.h"-#include "worker-pool.h"+#include "work-queue.h"+#include "hex-fastdiv.h"#include <assert.h>#include <dspqueue.h>@@ -52,6 +53,9 @@ struct htp_ops_context {const struct htp_tensor * dsts[HTP_OP_MAX_OUTPUTS];};+ dma_queue ** src_dma[HTP_OP_MAX_INPUTS];+ dma_queue ** dst_dma[HTP_OP_MAX_OUTPUTS];+// TODO convert these to an arraystruct htp_spad src0_spad;struct htp_spad src1_spad;@@ -65,11 +69,16 @@ struct htp_ops_context {// Main context for htp DSP backendstruct htp_context {- dspqueue_t queue;- dma_queue * dma[HTP_MAX_NTHREADS];+ dspqueue_t dsp_queue;+struct htp_mmap mmap[HTP_MAX_MMAPS];- worker_pool_context_t worker_pool;+ dma_queue_t dma[HTP_MAX_NTHREADS];+ dma_queue_t dma_cached[HTP_MAX_NTHREADS];+ work_queue_t work_queue;+ hmx_queue_t hmx_queue;+uint32_t n_threads;+ struct fastdiv_values n_threads_div;int thread_id;int thread_prio;@@ -86,6 +95,7 @@ struct htp_context {atomic_bool vtcm_needs_release;uint64_t max_vmem;+ uint32_t dirty_map[HTP_OP_MAX_TENSORS / 32];// Persistent DDR scratchpad for MUL_MAT_ID mappingsvoid * ddr_spad_base;@@ -93,7 +103,10 @@ struct htp_context {struct htp_ops_context octx;- struct hmx_queue * hmx_queue; // Async HMX queue for pipeline overlap+ qurt_thread_t main_thread;+ void * main_stack;+ atomic_bool killed;+ size_t footprint;};int op_matmul(struct htp_ops_context * octx);diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.hindex c9d0b3539a..1feaf2f999 100644--- a/ggml/src/ggml-hexagon/htp/htp-ops.h+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h@@ -108,8 +108,7 @@ enum htp_op_code {#define HTP_OP_MAX_KERN_PARAMS 32#define HTP_OP_MAX_BUFS 16-#define HTP_OP_MAX_REQS 256-#define HTP_OP_MAX_TENSORS (HTP_OP_MAX_REQS * HTP_OP_MAX_INPUTS + HTP_OP_MAX_REQS)+#define HTP_OP_MAX_TENSORS 8192 // must stay under 64K (uint16)#define HTP_OP_MAX_VMEM_DEFAULT (3355443200u)@@ -117,16 +116,18 @@ enum htp_op_code {enum htp_tensor_flags {HTP_TENSOR_COMPUTE = (1U << 0), // Tensor buffer temporal compute data (not weights)- HTP_TENSOR_FLUSHED = (1U << 1) // Tensor buffer has been flushed (set by the NPU)+ HTP_TENSOR_DIRTY = (1U << 1) // Tensor buffer is dirty and needs to be flushed};// Tensor descriptorstruct htp_tensor {uint32_t data; // Buffer offset in the messages, and data pointer on the NPU+ uint32_t alias; // Index of the canonical tensor for this memory bufferuint32_t size; // Data size in bytesuint32_t flags; // Buffer / tensor flags- uint16_t type; // Data type+ uint32_t type; // Data typeuint16_t bi; // Buffer index+ uint16_t ti; // Tensor indexuint32_t ne[HTP_OP_MAX_DIMS]; // Number of elementsuint32_t nb[HTP_OP_MAX_DIMS]; // Stride in bytes (see ggml.h ggml_tensor)};@@ -169,6 +170,8 @@ enum htp_profiler_mode {enum htp_trace_event_id {HTP_TRACE_EVT_DMA = 0,+ HTP_TRACE_EVT_L2FLUSH = 1,+ HTP_TRACE_EVT_INIT = 2,HTP_TRACE_EVT_HVX_COMP = 20,HTP_TRACE_EVT_HVX_A_QUANT = 21,diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.c b/ggml/src/ggml-hexagon/htp/htp-tensor.cnew file mode 100644index 0000000000..3f028a7e45--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/htp-tensor.c@@ -0,0 +1,204 @@+#include "htp-tensor.h"++#include <qurt.h>+#include <qurt_memory.h>++#include "hex-common.h"+#include "hex-utils.h"+#include "hex-fastdiv.h"+#include "hex-profile.h"+#include "htp-ctx.h"+#include "work-queue.h"++struct l2flush_task {+ struct htp_thread_trace * trace;+ uint32_t start;+ uint32_t end;+ uint32_t chunk_size;+ uint32_t ti;+};++static void l2flush_thread_worker(unsigned int n, unsigned int i, void * data) {+ struct l2flush_task * task = (struct l2flush_task *) data;+ const uint32_t start = task->start;+ const uint32_t end = task->end;+ const uint32_t ti = task->ti;+ const uint32_t chunk_size = task->chunk_size;++ const uint32_t thread_s = start + i * chunk_size;+ if (thread_s >= end) {+ return;+ }+ uint32_t thread_e = thread_s + chunk_size;+ if (thread_e > end) {+ thread_e = end;+ }++ struct htp_thread_trace * tr = &task->trace[i];+ htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, ti);+ hex_l2flush((void *) (uintptr_t) thread_s, thread_e - thread_s);+ htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, ti);+}++static void flush_all_dcache(struct htp_context * ctx) {+ struct htp_thread_trace * tr = &ctx->trace[0];+ htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0);+ qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);+ hex_l2fetch_block(ctx, ctx->footprint);+ htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0);+ bitmap_reset(ctx->dirty_map, HTP_OP_MAX_TENSORS);+}++static void flush_tensor_range(struct htp_context * ctx, const struct htp_tensor * t) {+ struct htp_thread_trace * tr = &ctx->trace[0];++ if (t->size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) {+ struct l2flush_task task;+ task.start = hex_align_down((size_t) t->data, HEX_L2_LINE_SIZE);+ task.end = hex_align_up((size_t) t->data + t->size, HEX_L2_LINE_SIZE);+ task.ti = t->ti;+ task.trace = ctx->trace;++ const uint32_t total_size = task.end - task.start;+ const uint32_t n_blocks = (total_size + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE;+ const uint32_t blocks_per_thread = fastdiv(n_blocks + ctx->n_threads - 1, &ctx->n_threads_div);+ task.chunk_size = blocks_per_thread * HEX_L2_BLOCK_SIZE;++ work_queue_run(ctx->work_queue, l2flush_thread_worker, &task, ctx->n_threads);+ } else {+ htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, t->ti);+ hex_l2flush((void *) t->data, t->size);+ htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, t->ti);+ }++ htp_tensor_make_clean(t, ctx->dirty_map);+}++void htp_tensor_flush(struct htp_context * ctx, const struct htp_tensor * t) {+ if (!bitmap_test(ctx->dirty_map, t->ti)) {+ return;+ }++ if (t->size > HEX_L2_FLUSH_ALL_THRESHOLD) {+ flush_all_dcache(ctx);+ return;+ }++ flush_tensor_range(ctx, t);+}++// One dirty tensor's line-aligned range, placed in the flattened global block space.+struct l2flush_range {+ uint32_t start; // line-aligned start address+ uint32_t end; // line-aligned end address+ uint32_t block_first; // global block index of this range's first block+ uint32_t n_blocks; // number of HEX_L2_BLOCK_SIZE chunks (last may be partial)+};++struct l2flush_multi_task {+ struct htp_thread_trace * trace;+ struct l2flush_range ranges[HTP_OP_MAX_INPUTS];+ uint32_t n_ranges;+ uint32_t total_blocks;+ uint32_t blocks_per_thread;+};++static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) {+ (void) n;+ struct l2flush_multi_task * task = (struct l2flush_multi_task *) data;++ const uint32_t gb_first = i * task->blocks_per_thread;+ uint32_t gb_last = gb_first + task->blocks_per_thread;+ if (gb_last > task->total_blocks) {+ gb_last = task->total_blocks;+ }+ if (gb_first >= gb_last) {+ return;+ }++ struct htp_thread_trace * tr = &task->trace[i];+ htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, gb_first);++ for (uint32_t r = 0; r < task->n_ranges; r++) {+ const struct l2flush_range * rg = &task->ranges[r];+ const uint32_t rb_first = rg->block_first;+ const uint32_t rb_last = rg->block_first + rg->n_blocks;++ const uint32_t lo = gb_first > rb_first ? gb_first : rb_first;+ const uint32_t hi = gb_last < rb_last ? gb_last : rb_last;+ if (lo >= hi) {+ continue;+ }++ const uint32_t s = rg->start + (lo - rb_first) * HEX_L2_BLOCK_SIZE;+ uint32_t e = rg->start + (hi - rb_first) * HEX_L2_BLOCK_SIZE;+ if (e > rg->end) {+ e = rg->end;+ }+ hex_l2flush((void *) (uintptr_t) s, e - s);+ }++ htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, gb_first);+}++void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) {+ uint64_t total_dirty = 0;+ for (uint32_t i = 0; i < n; i++) {+ const struct htp_tensor * t = tensors[i];+ if (t && bitmap_test(ctx->dirty_map, t->ti)) {+ total_dirty += t->size;+ }+ }++ if (total_dirty == 0) {+ return;+ }++ if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) {+ flush_all_dcache(ctx);+ return;+ }++ // Aggregate is small enough to walk. Thread it across all dirty ranges at once+ // when it is worth the dispatch, otherwise flush sequentially.+ if (total_dirty > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) {+ struct l2flush_multi_task task;+ task.trace = ctx->trace;+ task.n_ranges = 0;++ uint32_t block_acc = 0;+ for (uint32_t i = 0; i < n; i++) {+ const struct htp_tensor * t = tensors[i];+ if (!t || !bitmap_test(ctx->dirty_map, t->ti)) {+ continue;+ }+ // Clear as we go: dedups a tensor passed as multiple srcs (e.g. mul(x,x)).+ htp_tensor_make_clean(t, ctx->dirty_map);++ struct l2flush_range * rg = &task.ranges[task.n_ranges++];+ rg->start = hex_align_down((size_t) t->data, HEX_L2_LINE_SIZE);+ rg->end = hex_align_up((size_t) t->data + t->size, HEX_L2_LINE_SIZE);+ rg->block_first = block_acc;+ rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE;+ block_acc += rg->n_blocks;+ }++ task.total_blocks = block_acc;+ task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div);++ work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads);+ return;+ }++ struct htp_thread_trace * tr = &ctx->trace[0];+ for (uint32_t i = 0; i < n; i++) {+ const struct htp_tensor * t = tensors[i];+ if (!t || !bitmap_test(ctx->dirty_map, t->ti)) {+ continue;+ }+ htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, t->ti);+ hex_l2flush((void *) t->data, t->size);+ htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, t->ti);+ htp_tensor_make_clean(t, ctx->dirty_map);+ }+}diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.h b/ggml/src/ggml-hexagon/htp/htp-tensor.hnew file mode 100644index 0000000000..e503bb90f5--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/htp-tensor.h@@ -0,0 +1,36 @@+#ifndef HTP_TENSOR_H+#define HTP_TENSOR_H++#include <stdint.h>+#include "htp-ops.h"+#include "hex-bitmap.h"++static inline struct htp_tensor * htp_tensor_alias(const struct htp_tensor * t) {+ return (struct htp_tensor *) (uintptr_t) t->alias;+}++static inline void * htp_tensor_data(const struct htp_tensor * t) {+ return (void *) (uintptr_t) t->data;+}++static inline uint32_t * htp_tensor_flags(const struct htp_tensor * t) {+ return (uint32_t *) &t->flags;+}++static inline void htp_tensor_make_dirty(const struct htp_tensor * t, uint32_t * dirty_map) {+ struct htp_tensor * curr = (struct htp_tensor *) t;+ do {+ bitmap_set(dirty_map, curr->ti);+ curr = htp_tensor_alias(curr);+ } while (curr != t);+}++static inline void htp_tensor_make_clean(const struct htp_tensor * t, uint32_t * dirty_map) {+ bitmap_clear(dirty_map, t->ti);+}++struct htp_context;+void htp_tensor_flush(struct htp_context * ctx, const struct htp_tensor * t);+void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n);++#endif // HTP_TENSOR_Hdiff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.cindex d971b60f3a..b53bb5024e 100644--- a/ggml/src/ggml-hexagon/htp/main.c+++ b/ggml/src/ggml-hexagon/htp/main.c@@ -25,112 +25,44 @@#define GGML_COMMON_DECL_C#include "ggml-common.h"+#include "hex-bitmap.h"#include "htp-ctx.h"#include "htp-ops.h"-#include "htp-ops.h"+#include "htp-tensor.h"#include "htp_iface.h"-#include "worker-pool.h"--AEEResult htp_iface_open(const char * uri, remote_handle64 * handle) {- struct htp_context * ctx;- int err = 0;-- ctx = calloc(1, sizeof(*ctx));- if (ctx == NULL) {- return AEE_ENOMEMORY;- }-- // Use the context structure as the handle- *handle = (remote_handle64) ctx;-- // Enable FARF logs- HAP_setFARFRuntimeLoggingParams(0xffff, NULL, 0);-- // Set client class- {- HAP_power_request_t request;- memset(&request, 0, sizeof(HAP_power_request_t));- request.type = HAP_power_set_apptype;- request.apptype = HAP_POWER_COMPUTE_CLIENT_CLASS;-- if ((err = HAP_power_set((void *) ctx, &request)) != 0) {- return err;- }- }-- {- HAP_power_request_t request;- memset(&request, 0, sizeof(request));+#include "work-queue.h"+#include "hex-profile.h"- request.type = HAP_power_set_DCVS_v3;- request.dcvs_v3.set_dcvs_enable = TRUE;- request.dcvs_v3.dcvs_enable = FALSE;- request.dcvs_v3.set_bus_params = TRUE;- request.dcvs_v3.bus_params.min_corner = HAP_DCVS_VCORNER_MAX;- request.dcvs_v3.bus_params.max_corner = HAP_DCVS_VCORNER_MAX;- request.dcvs_v3.bus_params.target_corner = HAP_DCVS_VCORNER_MAX;- request.dcvs_v3.set_core_params = TRUE;- request.dcvs_v3.core_params.min_corner = HAP_DCVS_VCORNER_MAX;- request.dcvs_v3.core_params.max_corner = HAP_DCVS_VCORNER_MAX;- request.dcvs_v3.core_params.target_corner = HAP_DCVS_VCORNER_MAX;- request.dcvs_v3.set_sleep_disable = TRUE;- request.dcvs_v3.sleep_disable = TRUE;+#define HMX_QUEUE_CAPACITY 16+#define HMX_QUEUE_STACK_SIZE 16384+#define WORK_QUEUE_CAPACITY 16+#define WORK_QUEUE_STACK_SIZE 16384+#define MAIN_THREAD_STACK_SIZE 32768-#if (__HEXAGON_ARCH__ >= 79)- HAP_set_dcvs_v3_protected_bus_corners(&request, 1);-#endif- if ((err = HAP_power_set((void *) ctx, &request)) != 0) {- return err;- }+_Static_assert(WORK_QUEUE_MAX_N_THREADS >= HTP_MAX_NTHREADS,+ "work-queue thread cap must be >= HTP_MAX_NTHREADS");- memset(&request, 0, sizeof(request));- request.type = HAP_power_set_HVX;- request.hvx.power_up = TRUE;- if ((err = HAP_power_set((void *) ctx, &request)) != 0) {- return err;- }- }+struct htp_handle {+ struct htp_context * ctx;+};-#if __HVX_ARCH__ >= 75- {- // Power on HMX and set HMX clock- HAP_power_request_t request;- memset(&request, 0, sizeof(HAP_power_request_t));- request.type = HAP_power_set_HMX_v2;- request.hmx_v2.set_power = TRUE;- request.hmx_v2.power_up = TRUE;- request.hmx_v2.set_clock = TRUE;- request.hmx_v2.target_corner = HAP_DCVS_EXP_VCORNER_MAX;- request.hmx_v2.min_corner = HAP_DCVS_EXP_VCORNER_MAX;- request.hmx_v2.max_corner = HAP_DCVS_EXP_VCORNER_MAX;- request.hmx_v2.perf_mode = HAP_CLK_PERF_HIGH;- FARF(ALWAYS, "Setting HMX clock\n");- err = HAP_power_set((void *) ctx, &request);- if (err != AEE_SUCCESS) {- FARF(ERROR, "ggml-hex: error setting HMX clock.");- return err;- }- }-#else- {- // Power on HMX- HAP_power_request_t request;- memset(&request, 0, sizeof(HAP_power_request_t));- request.type = HAP_power_set_HMX;- request.hmx.power_up = TRUE;- FARF(ALWAYS, "Powering HMX on\n");- err = HAP_power_set((void *) ctx, &request);- if (err != AEE_SUCCESS) {- FARF(ERROR, "ggml-hex: error powering on HMX.");- return err;- }+AEEResult htp_iface_open(const char * uri, remote_handle64 * handle) {+ (void) uri;+ struct htp_handle * h = calloc(1, sizeof(*h));+ if (h == NULL) {+ return AEE_ENOMEMORY;}-#endif+ *handle = (remote_handle64) h;return AEE_SUCCESS;}AEEResult htp_iface_etm(remote_handle64 handle, uint32_t enable) {+ struct htp_handle * h = (struct htp_handle *) handle;+ if (!h) {+ return AEE_EBADPARM;+ }+int err = enable ? HAP_user_etm_enable() : HAP_user_etm_disable();if (err) {if (err == AEE_EVERSIONNOTSUPPORT) {@@ -143,10 +75,11 @@ AEEResult htp_iface_etm(remote_handle64 handle, uint32_t enable) {}AEEResult htp_iface_profiler(remote_handle64 handle, uint32_t mode, const htp_iface_pmu_conf* pmu_conf) {- struct htp_context * ctx = (struct htp_context *) handle;- if (!ctx) {+ struct htp_handle * h = (struct htp_handle *) handle;+ if (!h || !h->ctx) {return AEE_EBADPARM;}+ struct htp_context * ctx = h->ctx;if (mode == HTP_PROF_PMU) {const uint32_t* events = pmu_conf->events;@@ -179,48 +112,55 @@ AEEResult htp_iface_profiler(remote_handle64 handle, uint32_t mode, const htp_if}AEEResult htp_iface_close(remote_handle64 handle) {- struct htp_context * ctx = (struct htp_context *) handle;-- if (!ctx) {+ struct htp_handle * h = (struct htp_handle *) handle;+ if (!h) {return AEE_EBADPARM;}- if (ctx->queue) {- FARF(ERROR, "Closing handle with queue still open");- return AEE_EITEMBUSY;- }+ struct htp_context * ctx = h->ctx;+ if (ctx) {+ if (ctx->dsp_queue) {+ FARF(ERROR, "Closing handle with queue still open");+ return AEE_EITEMBUSY;+ }- // release the mmaps (if any)- for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {- if (ctx->mmap[i].size) {+ // release the mmaps (if any)+ for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {+ if (ctx->mmap[i].size) {#if __HVX_ARCH__ > 73- HAP_munmap2((void *) ctx->mmap[i].base, ctx->mmap[i].size);+ HAP_munmap2((void *) ctx->mmap[i].base, ctx->mmap[i].size);#else- HAP_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size);+ HAP_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size);#endif- ctx->mmap[i].size = 0;- ctx->mmap[i].base = NULL;- ctx->mmap[i].fd = -1;+ ctx->mmap[i].size = 0;+ ctx->mmap[i].base = NULL;+ ctx->mmap[i].fd = -1;+ }}- }- if (ctx->profiler) {- qurt_pmu_enable(1);- }+ if (ctx->profiler) {+ qurt_pmu_enable(1);+ }++ if (ctx->etm) {+ HAP_user_etm_disable();+ }- if (ctx->etm) {- HAP_user_etm_disable();+ // Free the unified block (ctx is the base address of the block)+ free(ctx);+ h->ctx = NULL;}- free(ctx);+ free(h);return AEE_SUCCESS;}AEEResult htp_iface_mmap(remote_handle64 handle, uint32_t fd, uint32_t size) {- struct htp_context * ctx = (struct htp_context *) handle;- if (!ctx) {+ struct htp_handle * h = (struct htp_handle *) handle;+ if (!h || !h->ctx) {return AEE_EBADPARM;}+ struct htp_context * ctx = h->ctx;// See if we already have this mappingfor (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {@@ -262,10 +202,11 @@ AEEResult htp_iface_mmap(remote_handle64 handle, uint32_t fd, uint32_t size) {}AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) {- struct htp_context * ctx = (struct htp_context *) handle;- if (!ctx) {+ struct htp_handle * h = (struct htp_handle *) handle;+ if (!h || !h->ctx) {return AEE_EBADPARM;}+ struct htp_context * ctx = h->ctx;for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {struct htp_mmap *m = &ctx->mmap[i];@@ -358,91 +299,268 @@ static void vtcm_free(struct htp_context * ctx) {}}+static void htp_main_thread(void * context);static void htp_packet_callback(dspqueue_t queue, int error, void * context);static void htp_error_callback(dspqueue_t queue, int error, void * context);AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t max_vmem) {- struct htp_context * ctx = (struct htp_context *) handle;-- if (!ctx) {+ struct htp_handle * h = (struct htp_handle *) handle;+ if (!h) {return AEE_EBADPARM;}- if (ctx->queue) {+ if (h->ctx) {FARF(ERROR, "Queue already open");return AEE_EITEMBUSY;}- // Import queue created on the CPU- int err = dspqueue_import(dsp_queue_id, // Queue ID from dspqueue_export- htp_packet_callback, // Packet callback- htp_error_callback, // Error callback; no errors expected on the DSP- (void *) ctx, // Callback context- &ctx->queue);+ // Cache the original FastRPC thread priority, then calculate compute priority+ int fastrpc_tid = qurt_thread_get_id();+ int fastrpc_prio = qurt_thread_get_priority(fastrpc_tid);+ int main_prio = fastrpc_prio - 10;+ if (main_prio < 1) main_prio = 1;++ dspqueue_t dsp_queue = NULL;+ bool use_callbacks = false;++ // Import queue with NULL callbacks to avoid starting dspueue internal threads+ int err = dspqueue_import(dsp_queue_id, NULL, NULL, (void *) h, &dsp_queue);+ if (err == AEE_EBADPARM) {+ // Fallback for devices that don't support NULL callbacks+ FARF(HIGH, "dspqueue import with NULL callbacks failed, trying with callbacks");+ use_callbacks = true;+ err = dspqueue_import(dsp_queue_id, htp_packet_callback, htp_error_callback, (void *) h, &dsp_queue);+ }+if (err) {FARF(ERROR, "Queue import failed with 0x%08x", (unsigned) err);return err;}+ qurt_sysenv_max_hthreads_t hw_threads;+ qurt_sysenv_get_max_hw_threads(&hw_threads);+ uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF;++ if (n_hvx == 0) {+ n_hvx = hw_nhvx;+ }+ if (n_hvx > hw_threads.max_hthreads) {+ n_hvx = hw_threads.max_hthreads;+ }+ if (n_hvx > HTP_MAX_NTHREADS) {+ n_hvx = HTP_MAX_NTHREADS;+ }++ // layout segments of our contiguous block++ // 1. htp_context : sits at the base (block is 4K-aligned via memalign below)+ size_t offset = sizeof(struct htp_context);++ // 2. main_stack+ size_t offset_main_stack = 0;+ size_t size_main_stack = 0;+ if (!use_callbacks) {+ offset_main_stack = hex_align_up(offset, 4096);+ size_main_stack = MAIN_THREAD_STACK_SIZE;+ offset = offset_main_stack + size_main_stack;+ }++ // 3. work_queue+ size_t wq_align = work_queue_alignof();+ size_t offset_wq = hex_align_up(offset, wq_align);+ size_t size_wq = work_queue_sizeof(n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE);+ offset = offset_wq + size_wq;++ // 4. dma_queue+ size_t dma_align = dma_queue_alignof();+ size_t offset_dma = hex_align_up(offset, dma_align);+ size_t size_dma = 0;+ for (uint32_t i = 0; i < n_hvx; i++) {+ size_dma = hex_align_up(size_dma, dma_queue_alignof());+ size_dma += dma_queue_sizeof(256);+ size_dma = hex_align_up(size_dma, dma_queue_alignof());+ size_dma += dma_queue_alias_sizeof();+ }+ offset = offset_dma + size_dma;++ // 5. hmx_queue+ size_t offset_hmx = 0;+ size_t size_hmx = 0;+ if (n_hmx) {+ size_t hmx_align = hmx_queue_alignof();+ offset_hmx = hex_align_up(offset, hmx_align);+ size_hmx = hmx_queue_sizeof(HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE);+ offset = offset_hmx + size_hmx;+ }++ size_t footprint = hex_align_up(offset, 128);++ void * block = memalign(4096, footprint);+ if (!block) {+ FARF(ERROR, "Unable to allocate unified block of size %zu\n", footprint);+ dspqueue_close(dsp_queue);+ return AEE_ENOMEMORY;+ }+ memset(block, 0, footprint);++ h->ctx = (struct htp_context *) block;+ struct htp_context * ctx = h->ctx;+ ctx->footprint = footprint;++ ctx->thread_id = fastrpc_tid;+ ctx->thread_prio = main_prio;ctx->max_vmem = max_vmem;- ctx->thread_id = qurt_thread_get_id();- ctx->thread_prio = qurt_thread_get_priority(ctx->thread_id);+ ctx->dsp_queue = dsp_queue;- // allocate VTCMerr = vtcm_alloc(ctx);if (err != AEE_SUCCESS) {FARF(ERROR, "Unable to allocate VTCM");+ htp_iface_stop(handle);return AEE_ENOMEMORY;}- ctx->hmx_enabled = n_hmx;- ctx->hmx_queue = NULL;- if (n_hmx) {- ctx->hmx_queue = hmx_queue_create(16, ctx->vtcm_rctx);- if (ctx->hmx_queue) {- ctx->hmx_queue->trace = &ctx->trace[HTP_MAX_NTHREADS];- } else {- FARF(ERROR, "hmx-queue-create failed");- ctx->hmx_enabled = false;+ HAP_setFARFRuntimeLoggingParams(0xffff, NULL, 0);++ // Set client class+ {+ HAP_power_request_t request;+ memset(&request, 0, sizeof(HAP_power_request_t));+ request.type = HAP_power_set_apptype;+ request.apptype = HAP_POWER_COMPUTE_CLIENT_CLASS;++ if ((err = HAP_power_set((void *) ctx, &request)) != 0) {+ htp_iface_stop(handle);+ return err;}}- FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx);- qurt_sysenv_max_hthreads_t hw_threads;- qurt_sysenv_get_max_hw_threads(&hw_threads);- uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF;+ // DCVS setup+ {+ HAP_power_request_t request;+ memset(&request, 0, sizeof(request));- if (n_hvx == 0) {- n_hvx = hw_nhvx;+ request.type = HAP_power_set_DCVS_v3;+ request.dcvs_v3.set_dcvs_enable = TRUE;+ request.dcvs_v3.dcvs_enable = FALSE;+ request.dcvs_v3.set_bus_params = TRUE;+ request.dcvs_v3.bus_params.min_corner = HAP_DCVS_VCORNER_MAX;+ request.dcvs_v3.bus_params.max_corner = HAP_DCVS_VCORNER_MAX;+ request.dcvs_v3.bus_params.target_corner = HAP_DCVS_VCORNER_MAX;+ request.dcvs_v3.set_core_params = TRUE;+ request.dcvs_v3.core_params.min_corner = HAP_DCVS_VCORNER_MAX;+ request.dcvs_v3.core_params.max_corner = HAP_DCVS_VCORNER_MAX;+ request.dcvs_v3.core_params.target_corner = HAP_DCVS_VCORNER_MAX;+ request.dcvs_v3.set_sleep_disable = TRUE;+ request.dcvs_v3.sleep_disable = TRUE;++#if (__HEXAGON_ARCH__ >= 79)+ HAP_set_dcvs_v3_protected_bus_corners(&request, 1);+#endif+ if ((err = HAP_power_set((void *) ctx, &request)) != 0) {+ htp_iface_stop(handle);+ return err;+ }++ memset(&request, 0, sizeof(request));+ request.type = HAP_power_set_HVX;+ request.hvx.power_up = TRUE;+ if ((err = HAP_power_set((void *) ctx, &request)) != 0) {+ htp_iface_stop(handle);+ return err;+ }}- if (n_hvx > hw_threads.max_hthreads) {- n_hvx = hw_threads.max_hthreads;++#if __HVX_ARCH__ >= 75+ {+ // Power on HMX and set HMX clock+ HAP_power_request_t request;+ memset(&request, 0, sizeof(HAP_power_request_t));+ request.type = HAP_power_set_HMX_v2;+ request.hmx_v2.set_power = TRUE;+ request.hmx_v2.power_up = TRUE;+ request.hmx_v2.set_clock = TRUE;+ request.hmx_v2.target_corner = HAP_DCVS_EXP_VCORNER_MAX;+ request.hmx_v2.min_corner = HAP_DCVS_EXP_VCORNER_MAX;+ request.hmx_v2.max_corner = HAP_DCVS_EXP_VCORNER_MAX;+ request.hmx_v2.perf_mode = HAP_CLK_PERF_HIGH;+ FARF(ALWAYS, "Setting HMX clock\n");+ err = HAP_power_set((void *) ctx, &request);+ if (err != AEE_SUCCESS) {+ FARF(ERROR, "ggml-hex: error setting HMX clock.");+ htp_iface_stop(handle);+ return err;+ }}- if (n_hvx > HTP_MAX_NTHREADS) {- n_hvx = HTP_MAX_NTHREADS;+#else+ {+ // Power on HMX+ HAP_power_request_t request;+ memset(&request, 0, sizeof(HAP_power_request_t));+ request.type = HAP_power_set_HMX;+ request.hmx.power_up = TRUE;+ FARF(ALWAYS, "Powering HMX on\n");+ err = HAP_power_set((void *) ctx, &request);+ if (err != AEE_SUCCESS) {+ FARF(ERROR, "ggml-hex: error powering on HMX.");+ htp_iface_stop(handle);+ return err;+ }}+#endif++ ctx->hmx_enabled = n_hmx;+ ctx->hmx_queue = NULL;+ if (n_hmx) {+ void * hmx_ptr = (void *) ((uintptr_t) block + offset_hmx);+ ctx->hmx_queue = hmx_queue_init(hmx_ptr, HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE, ctx->vtcm_rctx, &ctx->trace[HTP_MAX_NTHREADS]);+ }+ FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx);ctx->n_threads = n_hvx;+ ctx->n_threads_div = init_fastdiv_values(ctx->n_threads);++ // Initialize DMA queues+ uint8_t * dma_ptr_curr = (uint8_t *) ((uintptr_t) block + offset_dma);+ size_t size_dma_q = dma_queue_sizeof(256);+ size_t size_dma_alias = dma_queue_alias_sizeof();+for (int i = 0; i < ctx->n_threads; i++) {- ctx->dma[i] = dma_queue_create(256); // queue depth- if (ctx->dma[i]) {- ctx->dma[i]->trace = &ctx->trace[i];- }+ dma_ptr_curr = (uint8_t *) hex_align_up((uintptr_t) dma_ptr_curr, dma_queue_alignof());+ ctx->dma_cached[i] = dma_queue_init(dma_ptr_curr, 256, (uintptr_t) ctx->vtcm_base, ctx->vtcm_size, &ctx->trace[i]);+ dma_ptr_curr += size_dma_q;++ dma_ptr_curr = (uint8_t *) hex_align_up((uintptr_t) dma_ptr_curr, dma_queue_alignof());+ ctx->dma[i] = dma_queue_alias_init(dma_ptr_curr, ctx->dma_cached[i], 1);+ dma_ptr_curr += size_dma_alias;}ctx->ddr_spad_size = 512 * 1024; // 512 KBctx->ddr_spad_base = memalign(128, ctx->ddr_spad_size);- // init worker pool- err = worker_pool_init(&ctx->worker_pool, n_hvx);- if (err != AEE_SUCCESS) {- FARF(ERROR, "Unable to create worker pool");- if (ctx->ddr_spad_base) {- free(ctx->ddr_spad_base);- ctx->ddr_spad_base = NULL;- ctx->ddr_spad_size = 0;+ void * wq_ptr = (void *) ((uintptr_t) block + offset_wq);+ ctx->work_queue = work_queue_init(wq_ptr, n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE);++ ctx->main_stack = NULL;+ ctx->main_thread = 0;+ atomic_store(&ctx->killed, false);++ if (!use_callbacks) {+ // Start main compute thread+ ctx->main_stack = (void *) ((uintptr_t) block + offset_main_stack);++ qurt_thread_attr_t attr;+ qurt_thread_attr_init(&attr);+ qurt_thread_attr_set_stack_addr(&attr, ctx->main_stack);+ qurt_thread_attr_set_stack_size(&attr, size_main_stack);+ qurt_thread_attr_set_priority(&attr, main_prio);+ qurt_thread_attr_set_name(&attr, "htp-main");++ int err_thread = qurt_thread_create(&ctx->main_thread, &attr, htp_main_thread, ctx);+ if (err_thread) {+ FARF(ERROR, "Unable to create htp main thread: %d", err_thread);+ htp_iface_stop(handle);+ return AEE_ENOMEMORY;}- return err;}FARF(HIGH, "session %u started: n-hvx %u vtcm-size %zu vtcm-rctx %u n-threads %u thread-id %d thread-prio %d \n",@@ -452,35 +570,34 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp}AEEResult htp_iface_stop(remote_handle64 handle) {- struct htp_context * ctx = (struct htp_context *) handle;- if (!ctx) {+ struct htp_handle * h = (struct htp_handle *) handle;+ if (!h || !h->ctx) {return AEE_EBADPARM;}+ struct htp_context * ctx = h->ctx;- if (!ctx->queue) {- FARF(ERROR, "Queue not open");- return AEE_EBADSTATE;+ if (ctx->main_thread) {+ atomic_store(&ctx->killed, true);+ int status;+ (void) qurt_thread_join(ctx->main_thread, &status);+ ctx->main_thread = 0;}- // Close queue. dspqueue_close() will also wait for callbacks to finish.- int err = dspqueue_close(ctx->queue);- ctx->queue = NULL;+ int err = dspqueue_close(ctx->dsp_queue); ctx->dsp_queue = NULL;if (err != 0) {FARF(ERROR, "Queue close failed with 0x%08x", (unsigned) err);return err;}- if (ctx->worker_pool) {- // Release worker pool- worker_pool_release(&ctx->worker_pool);- }+ work_queue_free(ctx->work_queue);for (int i = 0; i < ctx->n_threads; i++) {- dma_queue_delete(ctx->dma[i]);+ dma_queue_alias_free(ctx->dma[i]);+ dma_queue_free(ctx->dma_cached[i]);}if (ctx->hmx_queue) {- hmx_queue_delete(ctx->hmx_queue);+ hmx_queue_free(ctx->hmx_queue);ctx->hmx_queue = NULL;}ctx->hmx_enabled = false;@@ -493,6 +610,9 @@ AEEResult htp_iface_stop(remote_handle64 handle) {ctx->ddr_spad_size = 0;}+ free(ctx);+ h->ctx = NULL;+return AEE_SUCCESS;}@@ -671,8 +791,6 @@ static int execute_op(struct htp_ops_context * octx) {case HTP_OP_INVALID:break;-- // No default to catch missing cases}FARF(ERROR, "Unknown Op %u", octx->op);@@ -778,12 +896,14 @@ static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uin}}-static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, uint32_t idx, struct htp_tensor *t) {+static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, struct htp_tensor *tens, uint32_t idx, struct htp_tensor *t) {uint32_t offset = t->data;uint32_t size = t->size;uint32_t bi = t->bi;+ uint32_t alias = t->alias;- t->data = bufs[bi].base + offset; // update data to the actual pointer+ t->data = (uint32_t) (bufs[bi].base + offset); // update data to the actual pointer+ t->alias = (uint32_t) (tens + alias); // update alias to the actual pointerFARF(HIGH, "prep-tensor #%u: bi %u offset %u size %u data %p : %u:%u:%u:%u", idx, t->bi, offset, t->size, (void*) t->data,t->ne[0], t->ne[1], t->ne[3], t->ne[3]);@@ -791,7 +911,7 @@ static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, uintstatic void prep_tensors(struct htp_context *ctx, struct htp_buf_desc *bufs, struct htp_tensor *tens, uint32_t n_tens) {for (uint32_t i=0; i < n_tens; i++) {- prep_tensor(ctx, bufs, i, tens + i);+ prep_tensor(ctx, bufs, tens, i, tens + i);}}@@ -805,29 +925,36 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u// Prep input tensorsfor (uint32_t i=0; i<HTP_OP_MAX_INPUTS; i++) {- struct htp_tensor *src = op->src[i] == 0xffff ? NULL : tens + op->src[i];-- octx->src[i] = src;- if (!src) continue;-- if (!(src->flags & HTP_TENSOR_FLUSHED) && (src->flags & HTP_TENSOR_COMPUTE)) {- // flush compute buffers on input- hex_l2flush((void *) src->data, src->size);+ uint16_t src_idx = op->src[i];+ if (src_idx == 0xffff) {+ octx->src[i] = NULL;+ octx->src_dma[i] = NULL;+ continue;}+ struct htp_tensor *src = tens + src_idx;+ octx->src[i] = src;+ octx->src_dma[i] = octx->ctx->dma; // FIXME: ? octx->ctx->dma_cached : octx->ctx->dma;+FARF(HIGH, "prep-src #%u: data %p size %u : %u:%u:%u:%u", op->src[i], (void*) src->data, src->size,src->ne[0], src->ne[1], src->ne[3], src->ne[3]);}+ htp_tensor_flush_all(octx->ctx, octx->src, HTP_OP_MAX_INPUTS);+// Prep output tensorsfor (uint32_t i = 0; i < HTP_OP_MAX_OUTPUTS; i++) {uint16_t dst_idx = op->dst[i];if (dst_idx == 0xffff) {- octx->dsts[i] = NULL;+ octx->dsts[i] = NULL;+ octx->dst_dma[i] = NULL;continue;}struct htp_tensor *dst = tens + dst_idx;- octx->dsts[i] = dst;+ octx->dsts[i] = dst;+ octx->dst_dma[i] = octx->ctx->dma; // FIXME: ? octx->ctx->dma_cached : octx->ctx->dma;++ htp_tensor_make_dirty(dst, octx->ctx->dirty_map);FARF(HIGH, "prep-dst[%u] #%u: data %p size %u : %u:%u:%u:%u", i, dst_idx, (void*) dst->data, dst->size,dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);@@ -841,34 +968,134 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, uoctx->src3_spad.src = NULL;octx->dst_spad.src = NULL;- // flush buffers on output- for (uint32_t i = 0; i < HTP_OP_MAX_OUTPUTS; i++) {- if (octx->dsts[i]) {- struct htp_tensor *dst = (struct htp_tensor *)octx->dsts[i];- hex_l2flush((void *) dst->data, dst->size);- dst->flags |= HTP_TENSOR_FLUSHED;+ return status;+}++static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_req * req, const struct dspqueue_buffer * dbuf) {+ dspqueue_t queue = ctx->dsp_queue;+ int err;- FARF(HIGH, "post-dst[%u] #%u: data %p size %u : %u:%u:%u:%u", i, op->dst[i], (void*) dst->data, dst->size,- dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);+ const uint32_t n_bufs = req->n_bufs;+ const uint32_t n_tens = req->n_tensors;+ const uint32_t n_ops = req->n_ops;++ const uint32_t b_size = sizeof(struct htp_buf_desc) * n_bufs;+ const uint32_t t_size = sizeof(struct htp_tensor) * n_tens;+ const uint32_t o_size = sizeof(struct htp_op_desc) * n_ops;+ const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops;+ const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req->n_traces * sizeof(struct htp_trace_desc);++ if (dbuf->size < b_size + t_size + o_size + p_size + tr_size) {+ FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf->size, b_size + t_size + o_size + p_size + tr_size);+ return;+ }++ FARF(HIGH, "processing opbatch #%u: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", req->id,+ n_bufs, n_tens, n_ops, req->n_traces, dbuf->size, b_size, t_size, o_size);++ // Clean cache at the start of the batch+ // We cant trace this part because the trace buffer is setup later+ qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);+ hex_l2fetch_block(ctx, ctx->footprint);+ bitmap_reset(ctx->dirty_map, HTP_OP_MAX_TENSORS);++ // Setup descriptor pointers+ uint8_t * m_ptr = dbuf->ptr;+ struct htp_buf_desc* bufs = (struct htp_buf_desc*) m_ptr; m_ptr += b_size;+ struct htp_tensor* tens = (struct htp_tensor*) m_ptr; m_ptr += t_size;+ struct htp_op_desc* ops = (struct htp_op_desc*) m_ptr; m_ptr += o_size;+ struct htp_prof_desc* pds = (struct htp_prof_desc*) m_ptr;++ prep_op_bufs(ctx, bufs, n_bufs);+ prep_tensors(ctx, bufs, tens, n_tens);++ struct htp_ops_context *octx = &ctx->octx;+ memset(octx, 0, sizeof(*octx));+ octx->n_threads = ctx->n_threads;+ octx->ctx = ctx;++ memset(ctx->trace, 0, sizeof(ctx->trace));+ if (ctx->profiler == HTP_PROF_TRACE) {+ struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size);+ for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {+ ctx->trace[t].events = &trace_events[t * req->n_traces];+ ctx->trace[t].max_events = req->n_traces;}}- return status;+ work_queue_wakeup(ctx->work_queue);+ if (ctx->hmx_queue) {+ hmx_queue_wakeup(ctx->hmx_queue);+ }++ int op_status = HTP_STATUS_OK;+ for (uint32_t i = 0; i < n_ops && op_status == HTP_STATUS_OK; i++) {+ struct profile_data prof;++ profile_start(ctx->profiler, &prof);++ op_status = proc_op_req(octx, tens, i, &ops[i]);++ profile_stop(ctx->profiler, &prof);++ if (ctx->profiler) {+ pds[i].opcode = ops[i].opcode;+ pds[i].usecs = prof.usecs;+ pds[i].cycles_start = prof.cycles_start;+ pds[i].cycles_stop = prof.cycles_stop;+ for (int j = 0; j < HEX_NUM_PMU_COUNTERS; j++) {+ pds[i].pmu[j] = prof.pmu_counters[j];+ }+ }+ }++ if (ctx->hmx_queue) {+ hmx_queue_suspend(ctx->hmx_queue);+ hmx_queue_flush(ctx->hmx_queue);+ }+ work_queue_suspend(ctx->work_queue);++ struct htp_opbatch_rsp rsp;+ memset(&rsp, 0, sizeof(rsp));+ rsp.id = req->id;+ rsp.status = op_status;+ rsp.n_bufs = n_bufs;+ rsp.n_tensors = n_tens;+ rsp.n_ops = n_ops;++ if (ctx->profiler == HTP_PROF_TRACE) {+ for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {+ rsp.n_traces[t] = ctx->trace[t].count;+ }+ }++ struct dspqueue_buffer write_dbuf = *dbuf;+ write_dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;++ // Flush remaining dirty tensors at the end of the batch+ htp_trace_event_start(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0);+ qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);+ htp_trace_event_stop(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0);++ err = dspqueue_write(queue, 0, 1, &write_dbuf, sizeof(rsp), (const uint8_t *) &rsp, DSPQUEUE_TIMEOUT_NONE);+ if (err != 0) {+ FARF(ERROR, "dspqueue_write failed: 0x%08x", (unsigned) err);+ }}+#define DSPQUEUE_READ_TIMEOUT_USEC 5000#define DSPQUEUE_POLL_TIMEOUT_USEC 100#define DSPQUEUE_POLL_COUNT 100-static void htp_packet_callback(dspqueue_t queue, int error, void * context) {- struct htp_context * ctx = (struct htp_context *) context;-+static void process_ops(struct htp_context * ctx) {+ dspqueue_t queue = ctx->dsp_queue;int err;uint32_t poll_count = DSPQUEUE_POLL_COUNT;vtcm_acquire(ctx);- while (!ctx->vtcm_needs_release) {+ while (!ctx->vtcm_needs_release && !atomic_load(&ctx->killed)) {struct htp_opbatch_req req;uint32_t r_size = sizeof(req);@@ -898,111 +1125,41 @@ static void htp_packet_callback(dspqueue_t queue, int error, void * context) {// Reset poll count for valid requestspoll_count = DSPQUEUE_POLL_COUNT;- const uint32_t n_bufs = req.n_bufs;- const uint32_t n_tens = req.n_tensors;- const uint32_t n_ops = req.n_ops;-- const uint32_t b_size = sizeof(struct htp_buf_desc) * n_bufs;- const uint32_t t_size = sizeof(struct htp_tensor) * n_tens;- const uint32_t o_size = sizeof(struct htp_op_desc) * n_ops;- const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops;- const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req.n_traces * sizeof(struct htp_trace_desc);-- if (dbuf.size < b_size + t_size + o_size + p_size + tr_size) {- FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf.size, b_size + t_size + o_size + p_size + tr_size);- break;- }-- FARF(HIGH, "processing opbatch #%u: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", req.id,- n_bufs, n_tens, n_ops, req.n_traces, dbuf.size, b_size, t_size, o_size);-- // Setup descriptor pointers- uint8_t * m_ptr = dbuf.ptr;- struct htp_buf_desc* bufs = (struct htp_buf_desc*) m_ptr; m_ptr += b_size;- struct htp_tensor* tens = (struct htp_tensor*) m_ptr; m_ptr += t_size;- struct htp_op_desc* ops = (struct htp_op_desc*) m_ptr; m_ptr += o_size;- struct htp_prof_desc* pds = (struct htp_prof_desc*) m_ptr;-- prep_op_bufs(ctx, bufs, n_bufs);- prep_tensors(ctx, bufs, tens, n_tens);-- struct htp_ops_context *octx = &ctx->octx;- memset(octx, 0, sizeof(*octx));- octx->n_threads = ctx->n_threads;- octx->ctx = ctx;-- if (ctx->profiler == HTP_PROF_TRACE) {- memset(ctx->trace, 0, sizeof(ctx->trace));- struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size);- for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {- ctx->trace[t].events = &trace_events[t * req.n_traces];- ctx->trace[t].max_events = req.n_traces;- }- } else {- for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {- ctx->trace[t].events = NULL;- ctx->trace[t].max_events = 0;- }- }-- int op_status = HTP_STATUS_OK;- uint32_t op_wakeup = n_ops / 2; // half-way throgh the batch-- hmx_queue_wakeup(ctx->hmx_queue);-- for (uint32_t i=0; i < n_ops; i++) {- struct profile_data prof;-- if (i == op_wakeup) {- dspqueue_write_early_wakeup_noblock(queue, 0, 0);- }+ process_opbatch(ctx, &req, &dbuf);+ }- profile_start(ctx->profiler, &prof);+ vtcm_release(ctx);+}- op_status = proc_op_req(octx, tens, i, &ops[i]);+static void htp_packet_callback(dspqueue_t queue, int error, void * context) {+ (void) queue;+ (void) error;+ struct htp_handle * h = (struct htp_handle *) context;+ if (h && h->ctx) {+ process_ops(h->ctx);+ }+}- profile_stop(ctx->profiler, &prof);+static void htp_main_thread(void * context) {+ struct htp_context * ctx = (struct htp_context *) context;- if (op_status != HTP_STATUS_OK) {- break;- }+ FARF(HIGH, "htp-main-thread: started");- if (ctx->profiler) {- pds[i].opcode = ops[i].opcode;- pds[i].usecs = prof.usecs;- pds[i].cycles_start = prof.cycles_start;- pds[i].cycles_stop = prof.cycles_stop;- for (int j = 0; j < HEX_NUM_PMU_COUNTERS; j++) {- pds[i].pmu[j] = prof.pmu_counters[j];- }- }- }+ while (!atomic_load(&ctx->killed)) {+ uint32_t flags = 0;+ uint32_t num_buffers = 0;+ uint32_t message_length = 0;- hmx_queue_suspend(ctx->hmx_queue);-- struct htp_opbatch_rsp rsp;- rsp.id = req.id;- rsp.status = op_status;- rsp.n_bufs = n_bufs;- rsp.n_tensors = n_tens;- rsp.n_ops = n_ops;- memset(rsp.pad, 0, sizeof(rsp.pad));- if (ctx->profiler == HTP_PROF_TRACE) {- for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {- rsp.n_traces[t] = ctx->trace[t].count;- }+ int err = dspqueue_peek(ctx->dsp_queue, &flags, &num_buffers, &message_length, 50000);+ if (err == 0) {+ process_ops(ctx);+ } else if (err == AEE_EWOULDBLOCK || err == AEE_EEXPIRED) {+ continue;} else {- memset(rsp.n_traces, 0, sizeof(rsp.n_traces));- }-- dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;-- err = dspqueue_write(queue, 0, 1, &dbuf, sizeof(rsp), (const uint8_t *) &rsp, DSPQUEUE_TIMEOUT_NONE);- if (err != 0) {- FARF(ERROR, "dspqueue_write failed: 0x%08x", (unsigned) err);+ FARF(ERROR, "dspqueue_peek failed: 0x%08x", (unsigned) err);break;}}- vtcm_release(ctx);+ FARF(HIGH, "htp-main-thread: stopped");}diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.cindex 1683131a81..0611aa9826 100644--- a/ggml/src/ggml-hexagon/htp/matmul-ops.c+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c@@ -92,10 +92,10 @@ struct htp_mm_context {// Per thread quant tasks// Precomputed block-parallel quantization valuesworker_callback_t quant_task_func;- uint32_t quant_ib_first[MAX_NUM_WORKERS];- uint32_t quant_ib_last[MAX_NUM_WORKERS];- uint32_t quant_r[MAX_NUM_WORKERS];- uint32_t quant_c[MAX_NUM_WORKERS];+ uint32_t quant_ib_first[WORK_QUEUE_MAX_N_THREADS];+ uint32_t quant_ib_last[WORK_QUEUE_MAX_N_THREADS];+ uint32_t quant_r[WORK_QUEUE_MAX_N_THREADS];+ uint32_t quant_c[WORK_QUEUE_MAX_N_THREADS];uint32_t n_quant_tasks;uint32_t n_quant_rows_per_thread;atomic_uint quant_barrier;@@ -254,7 +254,7 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) {return;}- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0_start);const uint32_t blck_0 = 64;@@ -309,7 +309,7 @@ static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, voidconst uint32_t src0_start_row = src0_nrows_per_thread * ith; \const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \\- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \+ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \\const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \const uint32_t n_prefetch = kparams->n_prefetch; \@@ -410,7 +410,7 @@ static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, voidconst uint32_t src0_start_row = src0_nrows_per_thread * ith; \const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \\- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \+ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \\const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \const uint32_t n_prefetch = kparams->n_prefetch; \@@ -523,7 +523,7 @@ static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith,uint8_t * restrict vtcm_src3_ptr = mmctx->vtcm_src3 + mmctx->vtcm_src3_size_per_thread * ith; \uint8_t * restrict src1_data = mmctx->vtcm_src1; \\- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \+ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \\const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \const uint32_t n_prefetch = kparams->n_prefetch; \@@ -699,7 +699,7 @@ static void hvx_mm_ffn_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith,uint8_t * restrict vtcm_src2_ptr = mmctx->vtcm_src2 + mmctx->vtcm_src2_size_per_thread * ith; \uint8_t * restrict src1_data = mmctx->vtcm_src1; \\- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \+ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \\const uint8_t * restrict src0_row = (const uint8_t *) src0->data; \const uint8_t * restrict src2_row = (const uint8_t *) src2->data; \@@ -820,7 +820,7 @@ static void name(unsigned int nth, unsigned int ith, void * data) {return; \} \\- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \+ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \\uint8_t * restrict dst = mmctx->vtcm_src1; \@@ -846,7 +846,7 @@ QUANTIZE_IMPL(quantize_f16_f16_flat, "quantize-f16-f16", quantize_f16_fstatic void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, void * data) {struct htp_mm_context * mmctx = data;struct htp_ops_context * octx = mmctx->octx;- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);const struct htp_tensor * src = octx->src[1];@@ -870,7 +870,7 @@ static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, vostatic void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, void * data) {struct htp_mm_context * mmctx = data;struct htp_ops_context * octx = mmctx->octx;- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);const struct htp_tensor * src = octx->src[1];@@ -944,7 +944,7 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) {const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U);- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];const size_t dst_row_size = nb1;const size_t src0_row_size = nb01;@@ -1040,7 +1040,7 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) {const uint32_t src0_start_row = src0_nrows_per_thread * ith;const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];const size_t dst_row_size = nb1;const size_t src0_row_size = nb01;@@ -1155,7 +1155,7 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) {return;}- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;const uint32_t n_prefetch = kparams->n_prefetch;@@ -1244,7 +1244,7 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) {return;}- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;const uint32_t n_prefetch = kparams->n_prefetch;@@ -1338,6 +1338,9 @@ static int hvx_mm_init_vec_dot(struct htp_mm_context * mmctx, enum htp_data_typestatic int hvx_mm_matmul(struct htp_ops_context * octx) {htp_matmul_tensors_preamble;+ struct htp_thread_trace * tr = &octx->ctx->trace[0];+ htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);+struct htp_mm_context mmctx_struct = {0};struct htp_mm_context * mmctx = &mmctx_struct;mmctx->octx = octx;@@ -1557,9 +1560,6 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {mmctx->vtcm_src0_stride = src0_row_size_padded;mmctx->vtcm_src1_stride = src1_row_size;- if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)- return HTP_STATUS_OK;-if (need_quant) {mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks;mmctx->quant_task_func = quant_task_func;@@ -1570,8 +1570,9 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {mmctx->n_quant_tasks = 0;}- const uint32_t n_matmul_jobs = octx->n_threads;- worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs);+ htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);++ worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, octx->n_threads);return HTP_STATUS_OK;}@@ -1874,7 +1875,7 @@ static void hvx_mm_ffn_2d(unsigned int nth, unsigned int ith, void * data) {#define DEQUANTIZE_WORKER_LOOP_IMPL(SUFFIX) \static void dequantize_tiled_worker_loop_##SUFFIX(unsigned int n, unsigned int i, void *data) { \tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data; \- struct htp_thread_trace * tr = state->traces ? &state->traces[i] : NULL; \+ struct htp_thread_trace * tr = &state->traces[i]; \htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); \for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) { \int start = task_id * state->n_tiles_per_task; \@@ -1892,7 +1893,7 @@ DEQUANTIZE_WORKER_LOOP_IMPL(q8_0)static void convert_f16_worker_loop(unsigned int n, unsigned int i, void *data) {tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data;- struct htp_thread_trace * tr = state->traces ? &state->traces[i] : NULL;+ struct htp_thread_trace * tr = &state->traces[i];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i);for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) {int start = task_id * state->n_tiles_per_task;@@ -1905,7 +1906,7 @@ static void convert_f16_worker_loop(unsigned int n, unsigned int i, void *data)static void quantize_f32_worker_loop(unsigned int n, unsigned int i, void *data) {tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data;- struct htp_thread_trace * tr = state->traces ? &state->traces[i] : NULL;+ struct htp_thread_trace * tr = &state->traces[i];htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, i);for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) {@@ -1920,7 +1921,7 @@ static void quantize_f32_worker_loop(unsigned int n, unsigned int i, void *data)static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void *data) {output_transfer_task_state_t *st = (output_transfer_task_state_t *) data;- struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL;+ struct htp_thread_trace * tr = &st->traces[i];int start_chunk_idx = i * st->n_chunks_per_task;htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, start_chunk_idx);@@ -1955,6 +1956,170 @@ typedef struct {uint32_t dma_step_rows_shift;} activation_transfer_task_state_t;+typedef struct {+ __fp16 *dst;+ const float *src;+ uint32_t n_rows;+ uint32_t k_block;+ uint32_t k_stride;+ uint32_t k_valid;+ uint32_t n_col_chunks;+ struct fastdiv_values n_threads_div;+ float *vtcm_f32_act;+ size_t vtcm_f32_act_bytes;+ struct htp_thread_trace *traces;+ struct htp_context *ctx;+ uint32_t dma_step_rows;+ uint32_t dma_step_rows_shift;+} activation_transfer_col_chunk_state_t;++static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk(+ dma_queue *dma_q,+ __fp16 *restrict vtcm_dst,+ const float *restrict src,+ uint32_t n_rows,+ uint32_t k_block,+ uint32_t k_stride,+ uint32_t k_chunk_valid,+ uint32_t c_first,+ uint32_t c_len,+ float *thread_f32_act,+ struct htp_thread_trace *tr,+ uint32_t dma_step_rows,+ uint32_t dma_step_rows_shift) {++ const uint32_t R = dma_step_rows;+ const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS);++ const uint32_t n_steps = n_rows_padded >> dma_step_rows_shift;++ // Push step 0+ if (n_steps > 0 && n_rows > 0) {+ uint32_t nrows_to_fetch = hex_smin(n_rows, R);+ dma_queue_push(dma_q, dma_make_ptr(thread_f32_act, src + c_first),+ c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch);+ }+ // Push step 1+ if (n_steps > 1) {+ uint32_t next_r = R * 1;+ if (next_r < n_rows) {+ uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R);+ const float *next_src = src + next_r * k_stride + c_first;+ float *next_buf = thread_f32_act + 1 * R * c_len;+ dma_queue_push(dma_q, dma_make_ptr(next_buf, next_src),+ c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch);+ }+ }+ for (uint32_t s = 0; s < n_steps; ++s) {+ uint32_t r = s << dma_step_rows_shift;+ float *curr_buf = thread_f32_act;++ if (r < n_rows) {+ curr_buf = (float *) dma_queue_pop(dma_q).dst;+ }++ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, r);+ for (uint32_t p = 0; p < (R >> 1); ++p) {+ uint32_t row_idx = r + (p << 1);+ float *pair_buf = curr_buf + (p << 1) * c_len;+ bool r0_valid = ((row_idx + 0) < n_rows);+ bool r1_valid = ((row_idx + 1) < n_rows);++ transfer_activation_row_pair_fp32_to_fp16_col_chunk(+ vtcm_dst, pair_buf, pair_buf + c_len, row_idx, k_block, c_first, c_len, k_chunk_valid, r0_valid, r1_valid+ );+ }+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, r);++ // Push step s + 2+ uint32_t next_s = s + 2;+ uint32_t next_r = next_s << dma_step_rows_shift;+ if (next_r < n_rows) {+ uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R);+ const float *next_src = src + next_r * k_stride + c_first;+ dma_queue_push(dma_q, dma_make_ptr(curr_buf, next_src),+ c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch);+ }+ }+}++static void transfer_activation_chunk_fp32_to_fp16_col_chunk(+ __fp16 *restrict vtcm_dst,+ const float *restrict src,+ uint32_t n_rows,+ uint32_t k_block,+ uint32_t k_stride,+ uint32_t c_first,+ uint32_t c_len,+ uint32_t k_chunk_valid) {+ const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS);+ const uint32_t n_rows_tiled = (n_rows / HTP_MM_HMX_TILE_N_ROWS) * HTP_MM_HMX_TILE_N_ROWS;++ uint32_t r = 0;++ #pragma unroll(2)+ for (r = 0; r < n_rows_tiled; r += 2) {+ const float *ptr_in0 = src + (r + 0) * k_stride + c_first;+ const float *ptr_in1 = src + (r + 1) * k_stride + c_first;++ transfer_activation_row_pair_fp32_to_fp16_col_chunk(+ vtcm_dst, ptr_in0, ptr_in1, r, k_block, c_first, c_len, k_chunk_valid, true, true+ );+ }++ for (; r < n_rows_padded; r += 2) {+ const bool row0_valid = r < n_rows;+ const bool row1_valid = (r + 1) < n_rows;++ const float *ptr_in0 = row0_valid ? (src + (r + 0) * k_stride + c_first) : NULL;+ const float *ptr_in1 = row1_valid ? (src + (r + 1) * k_stride + c_first) : NULL;++ transfer_activation_row_pair_fp32_to_fp16_col_chunk(+ vtcm_dst, ptr_in0, ptr_in1, r, k_block, c_first, c_len, k_chunk_valid, row0_valid, row1_valid+ );+ }+}++static void transfer_activation_chunk_col_chunk_worker_fn(unsigned int n, unsigned int i, void *data) {+ activation_transfer_col_chunk_state_t *st = (activation_transfer_col_chunk_state_t *) data;+ struct htp_thread_trace * tr = &st->traces[i];++ uint32_t n_blocks = st->k_block / 32;+ uint32_t b_first = fastdiv(n_blocks * i, &st->n_threads_div);+ uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_threads_div);+ uint32_t c_first = b_first * 32;+ uint32_t c_last = b_last * 32;+ uint32_t c_len = c_last - c_first;++ if (c_len == 0) {+ return;+ }++ uint32_t k_chunk_valid = 0;+ if (st->k_valid > c_first) {+ k_chunk_valid = hex_smin(st->k_valid, c_last) - c_first;+ }++ __fp16 *dst = st->dst;+ const float *src = st->src;++ if (st->vtcm_f32_act) {+ size_t thread_scratch_bytes = hex_align_down(fastdiv(st->vtcm_f32_act_bytes, &st->n_threads_div), 128);+ float *thread_f32_act = (float *)((char *)st->vtcm_f32_act + i * thread_scratch_bytes);++ transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk(+ st->ctx->dma[i], dst, src, st->n_rows, st->k_block, st->k_stride, k_chunk_valid,+ c_first, c_len, thread_f32_act, tr, st->dma_step_rows, st->dma_step_rows_shift+ );+ } else {+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, c_first);+ transfer_activation_chunk_fp32_to_fp16_col_chunk(+ dst, src, st->n_rows, st->k_block, st->k_stride, c_first, c_len, k_chunk_valid+ );+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, c_first);+ }+}+static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined(dma_queue *dma_q,__fp16 *restrict vtcm_dst,@@ -2024,7 +2189,7 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined(static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i, void *data) {activation_transfer_task_state_t *st = (activation_transfer_task_state_t *) data;- struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL;+ struct htp_thread_trace * tr = &st->traces[i];for (unsigned int task_id = i; task_id < (unsigned int)st->n_tasks; task_id += n) {int chunk_idx = task_id * st->n_chunks_per_task;@@ -2085,15 +2250,16 @@ typedef struct {static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigned int i, void *data) {activation_transfer_gathered_task_state_t *st = data;- struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL;+ struct htp_thread_trace * tr = &st->traces[i];int chunk_idx = i;int chunk_size = st->n_chunks_per_task;- int start_row = st->start_row + chunk_idx * chunk_size;+ int vtcm_start_row = chunk_idx * chunk_size;+ int start_row = st->start_row + vtcm_start_row;int n_rows = hex_smin(st->cne1 - start_row, chunk_size);if (n_rows > 0) {htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx);transfer_activation_chunk_fp32_to_fp16_gathered(- st->dst, st->src, start_row, n_rows, st->k_block,+ st->dst, st->src, start_row, vtcm_start_row, n_rows, st->k_block,st->matrix_rows, st->cur_a, st->mapping_stride,st->ne11, &st->ne11_div, st->nb11, st->nb12, st->cne1, st->k_valid);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx);@@ -2102,15 +2268,16 @@ static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsignestatic void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, unsigned int i, void *data) {activation_transfer_gathered_task_state_t *st = data;- struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL;+ struct htp_thread_trace * tr = &st->traces[i];int chunk_idx = i;int chunk_size = st->n_chunks_per_task;- int start_row = st->start_row + chunk_idx * chunk_size;+ int vtcm_start_row = chunk_idx * chunk_size;+ int start_row = st->start_row + vtcm_start_row;int n_rows = hex_smin(st->cne1 - start_row, chunk_size);if (n_rows > 0) {htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx);transfer_activation_chunk_fp32_to_fp16_gathered_flat(- st->dst, st->src, start_row, n_rows, st->k_block,+ st->dst, st->src, start_row, vtcm_start_row, n_rows, st->k_block,st->matrix_rows, st->cur_a, st->mapping_stride,st->nb12, st->cne1, st->k_valid);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx);@@ -2119,15 +2286,16 @@ static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, unstatic void transfer_output_chunk_scattered_worker_fn(unsigned int n, unsigned int i, void *data) {output_transfer_scattered_task_state_t *st = data;- struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL;+ struct htp_thread_trace * tr = &st->traces[i];int chunk_idx = i;int chunk_size = st->n_chunks_per_task;- int start_row = st->start_row + chunk_idx * chunk_size;+ int vtcm_start_row = chunk_idx * chunk_size;+ int start_row = st->start_row + vtcm_start_row;int n_rows = hex_smin(st->cne1 - start_row, chunk_size);if (n_rows > 0) {htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, chunk_idx);transfer_output_chunk_fp16_to_fp32_scattered(- st->dst, st->vtcm_src, start_row, n_rows, st->n_cols,+ st->dst, st->vtcm_src, start_row, vtcm_start_row, n_rows, st->n_cols,st->matrix_rows, st->cur_a, st->mapping_stride,st->dst_nb1, st->dst_nb2, st->cne1);htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, chunk_idx);@@ -2210,42 +2378,81 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst,}}-static void transfer_activation_chunk_threaded(- struct htp_context *ctx,- __fp16 *dst,- const float *src,- int n_rows,- int k_block,- int k_stride,- int n_threads,- int k_valid,- float *vtcm_f32_act,- size_t vtcm_f32_act_bytes) {+struct activation_transfer_params {+ struct htp_context * ctx;+ __fp16 * dst;+ const float * src;+ int n_rows;+ int k_block;+ int k_stride;+ int n_threads;+ const struct fastdiv_values * act_threads_div;+ const struct fastdiv_values * k_div;+ int k_valid;+ float * vtcm_f32_act;+ size_t vtcm_f32_act_bytes;+};++static void transfer_activation_chunk_threaded(const struct activation_transfer_params * params) {+ struct htp_context * ctx = params->ctx;+ __fp16 * dst = params->dst;+ const float * src = params->src;+ int n_rows = params->n_rows;+ int k_block = params->k_block;+ int k_stride = params->k_stride;+ int n_threads = params->n_threads;+ const struct fastdiv_values * act_threads_div = params->act_threads_div;+ const struct fastdiv_values * k_div = params->k_div;+ int k_valid = params->k_valid;+ float * vtcm_f32_act = params->vtcm_f32_act;+ size_t vtcm_f32_act_bytes = params->vtcm_f32_act_bytes;+if (n_rows <= 0) {return;}+ const size_t n_tasks = (n_rows + 31) >> 5;+ if (n_threads > 1 && k_block > 32 && n_tasks < (size_t) n_threads) {+ // Calculate step rows parameters for column-chunked dma pipelining+ uint32_t dma_step_rows = 2;+ uint32_t dma_step_rows_shift = 1;+ if (vtcm_f32_act && vtcm_f32_act_bytes > 0 && k_block > 0) {+ size_t thread_scratch_bytes = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_threads_div), 128);+ size_t thread_scratch_elements = thread_scratch_bytes / sizeof(float);+ size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div);+ if (dma_step_rows_max >= 4) {+ dma_step_rows = 4;+ dma_step_rows_shift = 2;+ }+ }++ activation_transfer_col_chunk_state_t col_state;+ col_state.dst = dst;+ col_state.src = src;+ col_state.n_rows = n_rows;+ col_state.k_block = k_block;+ col_state.k_stride = k_stride;+ col_state.k_valid = k_valid;+ col_state.n_col_chunks = n_threads;+ col_state.n_threads_div = *act_threads_div;+ col_state.vtcm_f32_act = vtcm_f32_act;+ col_state.vtcm_f32_act_bytes = vtcm_f32_act_bytes;+ col_state.traces = ctx->trace;+ col_state.ctx = ctx;+ col_state.dma_step_rows = dma_step_rows;+ col_state.dma_step_rows_shift = dma_step_rows_shift;++ worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_col_chunk_worker_fn, &col_state, n_threads);+ return;+ }+assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0 && k_stride % HTP_MM_HMX_TILE_N_COLS == 0);size_t n_tot_chunks = n_rows;size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : 32; // must be multiple of 32 to ensure correct destination address- uint32_t dma_step_rows = 2;- uint32_t dma_step_rows_shift = 1;- if (vtcm_f32_act && vtcm_f32_act_bytes > 0 && k_block > 0) {- size_t thread_scratch_elements = vtcm_f32_act_bytes / (n_threads * sizeof(float));- size_t dma_step_rows_max = (thread_scratch_elements / 2) / k_block;- if (dma_step_rows_max >= 4) {- dma_step_rows = 4;- dma_step_rows_shift = 2;- } else {- dma_step_rows = 2;- dma_step_rows_shift = 1;- }- }-activation_transfer_task_state_t state;- state.n_tasks = (n_tot_chunks + n_chunks_per_task - 1) / n_chunks_per_task;+ state.n_tasks = (n_threads == 1) ? 1 : hmx_ceil_div(n_tot_chunks, 32);state.n_tot_chunks = n_tot_chunks;state.n_chunks_per_task = n_chunks_per_task;state.dst = dst;@@ -2258,7 +2465,18 @@ static void transfer_activation_chunk_threaded(state.vtcm_f32_act = vtcm_f32_act;int active_threads = hex_smin(n_threads, (int)state.n_tasks);- state.vtcm_f32_act_bytes_per_thread = (vtcm_f32_act_bytes / active_threads) & ~127u;+ state.vtcm_f32_act_bytes_per_thread = hex_align_down(vtcm_f32_act_bytes / active_threads, 128);++ uint32_t dma_step_rows = 2;+ uint32_t dma_step_rows_shift = 1;+ if (vtcm_f32_act && state.vtcm_f32_act_bytes_per_thread > 0 && k_block > 0) {+ size_t thread_scratch_elements = state.vtcm_f32_act_bytes_per_thread / sizeof(float);+ size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div);+ if (dma_step_rows_max >= 4) {+ dma_step_rows = 4;+ dma_step_rows_shift = 2;+ }+ }state.dma_step_rows = dma_step_rows;state.dma_step_rows_shift = dma_step_rows_shift;@@ -2321,9 +2539,14 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,int pipeline,int n_threads,int act_threads,+ const struct fastdiv_values * act_threads_div,+ const struct fastdiv_values * k_div,int tile_size,int aligned_tile_size,int vtcm_size) {+ struct htp_thread_trace * tr = &ctx->trace[0];+ htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);+if (k % 32 != 0 || n % 32 != 0) { return -1; }if (!hex_is_aligned(dst, VLEN) || !hex_is_aligned(activation, VLEN)) { return -1; }@@ -2393,6 +2616,8 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols);+ htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);+if (pipeline) {// --- Asynchronous Pipelined Loop ---hmx_matmul_job_t job_slots[2]; // persistent double-buffered job descriptors@@ -2403,7 +2628,21 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,void *vtcm_weight_bufs[2] = { vtcm_scratch0, vtcm_scratch1 };void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 };- transfer_activation_chunk_threaded(ctx, vtcm_f16_act, activation + mr * act_stride, n_rows, k, act_stride, act_threads, k_valid, vtcm_f32_act, L.act_f32_bytes);+ struct activation_transfer_params act_params = {+ .ctx = ctx,+ .dst = vtcm_f16_act,+ .src = activation + mr * act_stride,+ .n_rows = (int) n_rows,+ .k_block = k,+ .k_stride = act_stride,+ .n_threads = act_threads,+ .act_threads_div = act_threads_div,+ .k_div = k_div,+ .k_valid = k_valid,+ .vtcm_f32_act = vtcm_f32_act,+ .vtcm_f32_act_bytes = L.act_f32_bytes,+ };+ transfer_activation_chunk_threaded(&act_params);// Prologue: push A0 and optionally A1 (if n_chunk_cnt > 1)const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols);@@ -2480,7 +2719,21 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) {const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows);- transfer_activation_chunk_threaded(ctx, vtcm_f16_act, activation + mr * act_stride, n_rows, k, act_stride, act_threads, k_valid, vtcm_f32_act, L.act_f32_bytes);+ struct activation_transfer_params act_params = {+ .ctx = ctx,+ .dst = vtcm_f16_act,+ .src = activation + mr * act_stride,+ .n_rows = (int) n_rows,+ .k_block = k,+ .k_stride = act_stride,+ .n_threads = act_threads,+ .act_threads_div = act_threads_div,+ .k_div = k_div,+ .k_valid = k_valid,+ .vtcm_f32_act = vtcm_f32_act,+ .vtcm_f32_act_bytes = L.act_f32_bytes,+ };+ transfer_activation_chunk_threaded(&act_params);// A0: Pre-fetch the first weight chunk (nc = 0)if (n > 0) {@@ -2570,7 +2823,8 @@ static inline const float *hmx_mm_src2_batch_ptr(const hmx_mm_f16_f32_batched_pastatic int hmx_mm_f16_f32_batched_simple(struct htp_context *ctx,const hmx_mm_f16_f32_batched_params_t *params,- int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size) {+ int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size,+ const struct fastdiv_values * act_threads_div, const struct fastdiv_values * k_div) {int ret = 0;for (int b3 = 0; b3 < params->ne13 && ret == 0; ++b3) {for (int b2 = 0; b2 < params->ne12 && ret == 0; ++b2) {@@ -2582,14 +2836,17 @@ static int hmx_mm_f16_f32_batched_simple(struct htp_context *ctx,params->act_stride, params->weight_stride * (int)sizeof(__fp16),HTP_TYPE_F16, params->k, params->dst_stride, params->src2_stride, params->n,m_chunk, n_chunk, pipeline, n_threads, act_threads,- 0, 0, vtcm_size);+ act_threads_div, k_div, 0, 0, vtcm_size);}}return ret;}static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_batched_params_t *params,- int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size) {+ int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads,+ const struct fastdiv_values * act_threads_div,+ const struct fastdiv_values * k_div,+ int vtcm_size) {if (params->act_stride < params->k || params->weight_stride < params->k || params->dst_stride < params->n) { return -1; }if (params->ne02 <= 0 || params->ne03 <= 0 || params->ne12 <= 0 || params->ne13 <= 0) { return -1; }if (params->ne12 % params->ne02 != 0 || params->ne13 % params->ne03 != 0) { return -1; }@@ -2604,9 +2861,12 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_// Grouped path is only valid if group_size > 1 and it fits within VTCM budget.bool run_grouped = (group_size > 1 && (size_t)vtcm_size <= vtcm_budget);if (!run_grouped) {- return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size);+ return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div);}+ struct htp_thread_trace * tr = &ctx->trace[0];+ htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);+const size_t vec_dot_size = params->k * sizeof(__fp16);const bool use_dma_activation = (params->act_stride > params->k);@@ -2622,7 +2882,8 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_if (L.total_bytes > vtcm_budget) {FARF(HIGH, "%s: grouped layout overflowed VTCM, falling back to simple batched loop", __func__);- return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size);+ htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);+ return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div);}uint8_t * const base = (uint8_t *) ctx->vtcm_base;@@ -2644,6 +2905,8 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_const size_t fp16_row_bytes = (size_t) params->k * sizeof(__fp16);const size_t weight_row_bytes = (size_t) params->weight_stride * sizeof(__fp16);+ htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);+hmx_matmul_job_t job;for (int b3 = 0; b3 < params->ne13; ++b3) {@@ -2662,9 +2925,21 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_for (int g = 0; g < group_size; ++g) {const float *activation_chunk = hmx_mm_activation_batch_ptr(params, b2_base + g, b3) + mr * params->act_stride;__fp16 *vtcm_act_g = vtcm_f16_act + (size_t) g * L.act_head_stride;- transfer_activation_chunk_threaded(ctx, vtcm_act_g,- activation_chunk, (int) n_rows,- params->k, params->act_stride, act_threads, params->k, vtcm_f32_act, L.act_f32_bytes);+ struct activation_transfer_params act_params = {+ .ctx = ctx,+ .dst = vtcm_act_g,+ .src = activation_chunk,+ .n_rows = (int) n_rows,+ .k_block = params->k,+ .k_stride = params->act_stride,+ .n_threads = act_threads,+ .act_threads_div = act_threads_div,+ .k_div = k_div,+ .k_valid = params->k,+ .vtcm_f32_act = vtcm_f32_act,+ .vtcm_f32_act_bytes = L.act_f32_bytes,+ };+ transfer_activation_chunk_threaded(&act_params);}// Prologue: Push A0 and A1 (if exists)@@ -2835,6 +3110,9 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,const struct mmid_row_mapping *matrix_rows,int cur_a,int mapping_stride) {+ struct htp_thread_trace * tr = &ctx->trace[0];+ htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);+const int cne1 = m;const int m_padded = hex_align_up(m, 32);@@ -2913,6 +3191,8 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00));+ htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);+hmx_matmul_job_t job;for (size_t mr = 0; mr < (size_t) m_padded; mr += m_chunk_n_rows) {@@ -2980,10 +3260,6 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_kconst int act_stride = (int)(src1->nb[1] / sizeof(float));const int wgt_stride = (int)(src0->nb[1] / sizeof(__fp16));- if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {- return HTP_STATUS_OK;- }-const float * src2_ptr = NULL;uint32_t src2_stride = 0;size_t src2_nb2 = 0;@@ -3027,6 +3303,8 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_kkparams->m_chunk, kparams->n_chunk,kparams->pipeline, n_threads,kparams->n_act_threads,+ &kparams->div_n_act_threads,+ &kparams->div_ne00_padded,kparams->vtcm_size);} else {ret = hmx_mm_2d_f32(@@ -3035,6 +3313,8 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k(int)(dst->nb[1] / sizeof(float)), src2_stride, (int)dst->ne[0],kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads,kparams->n_act_threads,+ &kparams->div_n_act_threads,+ &kparams->div_ne00_padded,kparams->tile_size, kparams->aligned_tile_size, kparams->vtcm_size);}@@ -3103,6 +3383,10 @@ static int hvx_mm_matmul_id(bool must_free_mapping) {htp_matmul_tensors_preamble;++ struct htp_thread_trace * tr = &octx->ctx->trace[0];+ htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);+const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;const struct htp_tensor * restrict ids = octx->src[2];const size_t src0_row_size = nb01;@@ -3175,8 +3459,9 @@ static int hvx_mm_matmul_id(mmctx->n_quant_tasks = n_quant_tasks;atomic_init(&mmctx->quant_barrier, n_quant_tasks);- const uint32_t n_matmul_jobs = octx->n_threads;- worker_pool_run_func(octx->ctx->worker_pool, matmul_id_job_func, mmctx, n_matmul_jobs);+ htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);++ worker_pool_run_func(octx->ctx->worker_pool, matmul_id_job_func, mmctx, octx->n_threads);if (must_free_mapping) free(mapping_buf);return HTP_STATUS_OK;@@ -3185,6 +3470,9 @@ static int hvx_mm_matmul_id(int op_matmul_id(struct htp_ops_context * octx) {htp_matmul_tensors_preamble;+ struct htp_thread_trace * tr = &octx->ctx->trace[0];+ htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);+struct htp_mm_context mmctx_struct = {0};struct htp_mm_context * mmctx = &mmctx_struct;mmctx->octx = octx;@@ -3262,10 +3550,7 @@ int op_matmul_id(struct htp_ops_context * octx) {}}- if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {- if (must_free_mapping) free(mapping_buf);- return HTP_STATUS_OK;- }+ htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);if (kparams->n_hmx) {return hmx_mm_op_matmul_id(octx, mmctx, matrix_row_counts, matrix_rows, mapping_buf, must_free_mapping);@@ -3275,6 +3560,9 @@ int op_matmul_id(struct htp_ops_context * octx) {}int op_matmul_qkv(struct htp_ops_context * octx) {+ struct htp_thread_trace * tr = &octx->ctx->trace[0];+ htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);+const struct htp_tensor * restrict src0 = octx->src[0]; // Wkconst struct htp_tensor * restrict src1 = octx->src[1]; // xconst struct htp_tensor * restrict src2 = octx->src[2]; // Wv@@ -3379,9 +3667,6 @@ int op_matmul_qkv(struct htp_ops_context * octx) {mmctx->vtcm_src3_size_per_thread = L.src3_bytes / octx->n_threads;mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads;- if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)- return HTP_STATUS_OK;-mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks;mmctx->quant_task_func = quant_task_func;mmctx->n_quant_tasks = n_quant_tasks;@@ -3413,12 +3698,18 @@ int op_matmul_qkv(struct htp_ops_context * octx) {} else {matmul_job_func = hvx_mm_qkv_2d;}++ htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);+worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs);return HTP_STATUS_OK;}int op_matmul_ffn(struct htp_ops_context * octx) {+ struct htp_thread_trace * tr = &octx->ctx->trace[0];+ htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);+const struct htp_tensor * restrict src0 = octx->src[0]; // Wgateconst struct htp_tensor * restrict src1 = octx->src[1]; // yconst struct htp_tensor * restrict src2 = octx->src[2]; // Wup@@ -3516,9 +3807,6 @@ int op_matmul_ffn(struct htp_ops_context * octx) {mmctx->vtcm_src2_size_per_thread = L.src2_bytes / octx->n_threads;mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads;- if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)- return HTP_STATUS_OK;-mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks;mmctx->quant_task_func = quant_task_func;mmctx->n_quant_tasks = n_quant_tasks;@@ -3550,6 +3838,9 @@ int op_matmul_ffn(struct htp_ops_context * octx) {} else {matmul_job_func = hvx_mm_ffn_2d;}++ htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);+worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs);return HTP_STATUS_OK;diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.hindex 2e131bc3d0..4450446609 100644--- a/ggml/src/ggml-hexagon/htp/matmul-ops.h+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h@@ -95,6 +95,8 @@ struct htp_mm_kernel_params {struct fastdiv_values div_r2;struct fastdiv_values div_r3;struct fastdiv_values div_ne11;+ struct fastdiv_values div_n_act_threads;+ struct fastdiv_values div_ne00_padded;};#if defined(__cplusplus)@@ -643,6 +645,136 @@ static inline size_t htp_mm_hmx_get_batched_vtcm_size(return L.total_bytes;}+static inline bool htp_mm_hmx_solve_batched_params(+ int wtype,+ uint32_t k,+ uint32_t ne01_padded,+ uint32_t ne11,+ uint32_t group_size,+ bool use_dma_activation,+ int n_threads,+ bool pipeline,+ size_t vtcm_budget,+ size_t * m_chunk_out,+ size_t * n_chunk_out,+ int * act_threads_out,+ size_t * vtcm_size_out+) {+ size_t best_mblocks = SIZE_MAX;+ int best_act_threads = 0;+ size_t best_m_chunk = 0;+ size_t best_n_chunk = 0;+ size_t best_vtcm_size = 0;++ int act_threads = n_threads;+ while (act_threads >= 1) {+ size_t group_overhead = 256;+ size_t group_size_per_n, group_size_per_m, group_size_per_mn;+ htp_mm_hmx_get_batched_chunk_costs(k, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn);++ size_t m_chunk_candidate = 0;+ size_t n_chunk_candidate = 0;+ size_t vtcm_size_candidate = 0;++ if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded,+ (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT,+ &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) {+ size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, use_dma_activation, pipeline, act_threads);+ if (exact_size <= vtcm_budget) {+ size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate;+ if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) {+ best_mblocks = mblocks;+ best_act_threads = act_threads;+ best_m_chunk = m_chunk_candidate;+ best_n_chunk = n_chunk_candidate;+ best_vtcm_size = exact_size;+ }+ }+ }+ if (act_threads == 1) {+ act_threads = 0;+ } else {+ act_threads /= 2;+ }+ }++ if (best_act_threads > 0) {+ *m_chunk_out = best_m_chunk;+ *n_chunk_out = best_n_chunk;+ *vtcm_size_out = best_vtcm_size;+ *act_threads_out = best_act_threads;+ return true;+ }+ return false;+}++static inline bool htp_mm_hmx_solve_2d_params(+ int wtype,+ uint32_t k,+ uint32_t m_id_rows,+ uint32_t ne01_padded,+ uint32_t ne11_padded,+ uint32_t m_for_cost,+ int n_threads,+ bool pipeline,+ bool is_matmul_id,+ uint32_t aligned_tile_size,+ size_t vtcm_budget,+ size_t * m_chunk_out,+ size_t * n_chunk_out,+ int * act_threads_out,+ size_t * vtcm_size_out+) {+ size_t best_mblocks = SIZE_MAX;+ int best_act_threads = 0;+ size_t best_m_chunk = 0;+ size_t best_n_chunk = 0;+ size_t best_vtcm_size = 0;++ const int m_for_chunks = is_matmul_id ? hex_align_up(m_id_rows, 32) : ne11_padded;++ int act_threads = n_threads;+ while (act_threads >= 1) {+ size_t simple_2d_overhead = 256;+ size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn;+ htp_mm_hmx_get_2d_chunk_costs(wtype, k, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn);++ size_t m_chunk_candidate = 0;+ size_t n_chunk_candidate = 0;+ size_t vtcm_size_candidate = 0;++ if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded,+ (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT,+ &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) {+ size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size);+ if (exact_size <= vtcm_budget) {+ size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate;+ if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) {+ best_mblocks = mblocks;+ best_act_threads = act_threads;+ best_m_chunk = m_chunk_candidate;+ best_n_chunk = n_chunk_candidate;+ best_vtcm_size = exact_size;+ }+ }+ }+ if (act_threads == 1) {+ act_threads = 0;+ } else {+ act_threads /= 2;+ }+ }++ if (best_act_threads > 0) {+ *m_chunk_out = best_m_chunk;+ *n_chunk_out = best_n_chunk;+ *vtcm_size_out = best_vtcm_size;+ *act_threads_out = best_act_threads;+ return true;+ }+ return false;+}+#ifdef __cplusplus}#endifdiff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.cindex d16dc7d38e..5bc7d74f5e 100644--- a/ggml/src/ggml-hexagon/htp/rope-ops.c+++ b/ggml/src/ggml-hexagon/htp/rope-ops.c@@ -18,6 +18,7 @@#include "htp-ctx.h"#include "htp-ops.h"#include "htp-ops.h"+#include "htp-tensor.h"// Redefined the rope type constants as we can't include ggml.h#define HTP_ROPE_TYPE_NORMAL 0@@ -712,17 +713,11 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {}int op_rope(struct htp_ops_context * octx) {- int err = HTP_STATUS_OK;-switch (octx->src[0]->type) {case HTP_TYPE_F32:- err = execute_op_rope_f32(octx);- break;+ return execute_op_rope_f32(octx);default:- err = HTP_STATUS_NO_SUPPORT;- break;+ return HTP_STATUS_NO_SUPPORT;}-- return err;}diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.cindex a71107f104..82792c610b 100644--- a/ggml/src/ggml-hexagon/htp/unary-ops.c+++ b/ggml/src/ggml-hexagon/htp/unary-ops.c@@ -19,6 +19,7 @@#include "ggml-common.h"#include "htp-ctx.h"#include "htp-ops.h"+#include "htp-tensor.h"#include "htp-vtcm.h"#include "hex-profile.h"@@ -397,7 +398,7 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * datstruct htp_ops_context * octx = uctx->octx; \const struct htp_tensor * src = octx->src[0]; \const struct htp_tensor * dst = octx->dst; \- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \+ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \\htp_unary_preamble; \\@@ -558,7 +559,7 @@ static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, voidstruct htp_ops_context * octx = uctx->octx; \const struct htp_tensor * src = octx->src[0]; \const struct htp_tensor * dst = octx->dst; \- struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \+ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \\htp_unary_preamble; \\@@ -922,17 +923,11 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {}int op_unary(struct htp_ops_context * octx) {- int err = HTP_STATUS_OK;-switch (octx->src[0]->type) {case HTP_TYPE_F32:- err = execute_op_unary_f32(octx);- break;+ return execute_op_unary_f32(octx);default:- err = HTP_STATUS_NO_SUPPORT;- break;+ return HTP_STATUS_NO_SUPPORT;}-- return err;}diff --git a/ggml/src/ggml-hexagon/htp/work-queue.c b/ggml/src/ggml-hexagon/htp/work-queue.cnew file mode 100644index 0000000000..bb73e205a4--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/work-queue.c@@ -0,0 +1,244 @@+#include "work-queue.h"+#include "hex-utils.h"++#include <qurt.h>+#include <qurt_hvx.h>++#include <stdatomic.h>+#include <stdint.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>++#include "HAP_farf.h"++#define LOWEST_USABLE_QURT_PRIO (254)++// internal structure kept in thread-local storage per instance of work queue+typedef struct {+ work_queue_t queue;+ unsigned int id;+} worker_context_t;++struct work_queue_task_s {+ work_queue_func_t func;+ void * data;+ unsigned int n_threads;+ atomic_uint barrier;+};++// internal structure kept in thread-local storage per instance of work queue+struct work_queue_s {+ atomic_uint seqn; // seqno used to detect new jobs+ atomic_uint idx_read; // Updated by producer (pop/reclaim)+ unsigned int idx_write; // Updated by producer (push)+ uint32_t idx_mask;+ uint32_t capacity;++ qurt_thread_t thread[WORK_QUEUE_MAX_N_THREADS]; // thread ID's of the workers+ worker_context_t context[WORK_QUEUE_MAX_N_THREADS]; // worker contexts+ void * stack[WORK_QUEUE_MAX_N_THREADS]; // thread stack pointers+ unsigned int n_threads; // total threads (workers + main)+ unsigned int n_workers; // number of active threads (just workers)++ atomic_bool active; // workers are polling/active+ atomic_bool killed; // threads need to exit+ bool external_mem; // memory owned externally++ struct work_queue_task_s queue[] __attribute__((aligned(HEX_L2_LINE_SIZE)));+};++static void work_queue_thread(void * context) {+ worker_context_t * me = (worker_context_t *) context;+ work_queue_t q = me->queue;++ FARF(HIGH, "work-queue: thread %u started", me->id);++ unsigned int prev_seqn = 0;++ while (!atomic_load_explicit(&q->killed, memory_order_relaxed)) {+ unsigned int seqn = atomic_load_explicit(&q->seqn, memory_order_acquire);+ if (seqn == prev_seqn) {+ if (atomic_load_explicit(&q->active, memory_order_relaxed)) {+ hex_pause();+ } else {+ qurt_futex_wait(&q->seqn, prev_seqn);+ }+ continue;+ }++ prev_seqn = seqn;++ // Process all active tasks in the queue+ unsigned int ir = atomic_load_explicit(&q->idx_read, memory_order_relaxed);+ unsigned int iw = q->idx_write;++ while (ir != iw) {+ struct work_queue_task_s * task = &q->queue[ir];++ unsigned int n = task->n_threads;+ unsigned int i = me->id;+ if (i < n) {+ task->func(n, i, task->data);++ atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release);+ } else {+ while (atomic_load_explicit(&task->barrier, memory_order_relaxed) > 0) {+ hex_pause();+ }+ }++ ir = (ir + 1) & q->idx_mask;+ }+ }++ FARF(HIGH, "work-queue: thread %u stopped", me->id);+}++bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) {+ if (n > q->n_threads) {+ FARF(ERROR, "work-queue: invalid number of jobs %u for n-threads %u", n, q->n_threads);+ return false;+ }++ unsigned int ir = atomic_load_explicit(&q->idx_read, memory_order_relaxed);+ unsigned int iw = q->idx_write;++ if (((iw + 1) & q->idx_mask) == ir) {+ FARF(ERROR, "work-queue-push: queue is full\n");+ return false;+ }++ struct work_queue_task_s * task = &q->queue[iw];+ task->func = func;+ task->data = data;+ task->n_threads = n;+ atomic_store_explicit(&task->barrier, n, memory_order_relaxed);++ q->idx_write = (iw + 1) & q->idx_mask;++ // publish job to workers (already awake and polling)+ atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release);++ // main thread runs job #0+ func(n, 0, data);++ atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release);++ while (atomic_load_explicit(&task->barrier, memory_order_relaxed) > 0) {+ hex_pause();+ }++ atomic_thread_fence(memory_order_acquire);++ atomic_store_explicit(&q->idx_read, (ir + 1) & q->idx_mask, memory_order_relaxed);++ return true;+}++size_t work_queue_sizeof(uint32_t n_threads, uint32_t capacity, uint32_t stack_size) {+ capacity = hex_ceil_pow2(capacity);+ uint32_t n_workers = n_threads > 1 ? n_threads - 1 : 0;+ size_t size_stacks = stack_size * n_workers;+ size_t size_q = hex_align_up(sizeof(struct work_queue_s) + capacity * sizeof(struct work_queue_task_s), HEX_L2_LINE_SIZE);+ return size_stacks + size_q;+}++size_t work_queue_alignof(void) {+ return 4096;+}++work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size) {+ capacity = hex_ceil_pow2(capacity);+ uint32_t n_workers = n_threads > 1 ? n_threads - 1 : 0;+ unsigned char * mem_blob = (unsigned char *) ptr;++ work_queue_t q = (work_queue_t) (mem_blob + stack_size * n_workers);+ memset(q, 0, sizeof(struct work_queue_s) + capacity * sizeof(struct work_queue_task_s));++ q->n_threads = n_threads;+ q->n_workers = n_workers;+ q->external_mem = true;+ q->capacity = capacity;++ for (unsigned int i = 0; i < n_workers; i++) {+ q->stack[i] = mem_blob; mem_blob += stack_size;+ q->thread[i] = 0;+ q->context[i].id = i + 1;+ q->context[i].queue = q;+ }++ atomic_init(&q->idx_read, 0);+ atomic_init(&q->seqn, 0);+ atomic_init(&q->active, false);+ q->idx_write = 0;+ q->idx_mask = capacity - 1;+ q->killed = 0;+ for (int i = 0; i < (int) capacity; i++) {+ atomic_init(&q->queue[i].barrier, 0);+ q->queue[i].func = NULL;+ q->queue[i].data = NULL;+ q->queue[i].n_threads = 0;+ }++ // launch the workers+ qurt_thread_attr_t attr;+ qurt_thread_attr_init(&attr);++ for (unsigned int i = 0; i < n_workers; i++) {+ qurt_thread_attr_set_stack_addr(&attr, q->stack[i]);+ qurt_thread_attr_set_stack_size(&attr, stack_size);++ char thread_name[32];+ snprintf(thread_name, sizeof(thread_name), "work-queue:%u", i);+ qurt_thread_attr_set_name(&attr, thread_name);++ // set up priority - by default, match the creating thread's prio+ int prio = qurt_thread_get_priority(qurt_thread_get_id());+ if (prio < 1) {+ prio = 1;+ }+ if (prio > LOWEST_USABLE_QURT_PRIO) {+ prio = LOWEST_USABLE_QURT_PRIO;+ }++ qurt_thread_attr_set_priority(&attr, prio);++ int err = qurt_thread_create(&q->thread[i], &attr, work_queue_thread, (void *) &q->context[i]);+ if (err) {+ FARF(ERROR, "Could not launch worker threads!");+ work_queue_free(q);+ return NULL;+ }+ }++ return q;+}++void work_queue_free(work_queue_t q) {+ if (!q) { return; }++ atomic_store_explicit(&q->killed, 1, memory_order_relaxed);+ atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release);+ qurt_futex_wake(&q->seqn, q->n_workers);++ for (unsigned int i = 0; i < q->n_workers; i++) {+ if (q->thread[i]) {+ int status;+ (void) qurt_thread_join(q->thread[i], &status);+ }+ }+}++void work_queue_wakeup(work_queue_t q) {+ if (!atomic_load_explicit(&q->active, memory_order_relaxed)) {+ atomic_store_explicit(&q->active, true, memory_order_release);+ // Increment seqn and wake workers to transition them out of sleep+ atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release);+ qurt_futex_wake(&q->seqn, q->n_workers);+ }+}++void work_queue_suspend(work_queue_t q) {+ atomic_store_explicit(&q->active, false, memory_order_release);+}diff --git a/ggml/src/ggml-hexagon/htp/work-queue.h b/ggml/src/ggml-hexagon/htp/work-queue.hnew file mode 100644index 0000000000..09ca4b1f43--- /dev/null+++ b/ggml/src/ggml-hexagon/htp/work-queue.h@@ -0,0 +1,38 @@+#ifndef HTP_WORK_QUEUE_H+#define HTP_WORK_QUEUE_H++#include <stdbool.h>+#include <stdint.h>+#include <stddef.h>++typedef void (*work_queue_func_t)(unsigned int n, unsigned int i, void *);++struct work_queue_s;+typedef struct work_queue_s * work_queue_t;++#define WORK_QUEUE_MAX_N_THREADS 10++size_t work_queue_sizeof(uint32_t n_threads, uint32_t capacity, uint32_t stack_size);+size_t work_queue_alignof(void);+work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size);+void work_queue_free(work_queue_t q);++void work_queue_wakeup(work_queue_t q);+void work_queue_suspend(work_queue_t q);++bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n);++static inline bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) {+ if (n <= 1) {+ func(n, 0, data);+ return true;+ }+ return work_queue_run_async(q, func, data, n);+}++// Legacy compatibility+typedef work_queue_func_t worker_callback_t;+#define worker_pool_run_func work_queue_run+#define worker_pool work_queue++#endif // #ifndef HTP_WORK_QUEUE_Hdiff --git a/ggml/src/ggml-hexagon/htp/worker-pool.c b/ggml/src/ggml-hexagon/htp/worker-pool.cdeleted file mode 100644index 50960d2c75..0000000000--- a/ggml/src/ggml-hexagon/htp/worker-pool.c+++ /dev/null@@ -1,305 +0,0 @@-#include "worker-pool.h"-#include "hex-utils.h"--#include <qurt.h>-#include <qurt_hvx.h>--#include <stdatomic.h>-#include <stdint.h>-#include <stdio.h>-#include <stdlib.h>-#include <string.h>--#include "HAP_farf.h"--#define LOWEST_USABLE_QURT_PRIO (254)--struct worker_pool_s;--// internal structure kept in thread-local storage per instance of worker pool-typedef struct {- struct worker_pool_s * pool;- unsigned int id;-} worker_context_t;--// internal structure kept in thread-local storage per instance of worker pool-typedef struct worker_pool_s {- worker_pool_job_t job[MAX_NUM_WORKERS]; // list of job descriptors- qurt_thread_t thread[MAX_NUM_WORKERS]; // thread ID's of the workers- worker_context_t context[MAX_NUM_WORKERS]; // worker contexts- void * stack[MAX_NUM_WORKERS]; // thread stack pointers- unsigned int n_threads; // number of workers in this pool-- atomic_uint seqn; // seqno used to detect new jobs- atomic_uint next_job; // next job index- atomic_uint n_pending; // number of pending jobs- atomic_uint n_jobs; // number of current jobs- atomic_bool killed; // threads need to exit-} worker_pool_t;--static void worker_pool_main(void * context) {- worker_context_t * me = (worker_context_t *) context;- worker_pool_t * pool = me->pool;-- FARF(HIGH, "worker-pool: thread %u started", me->id);-- unsigned int prev_seqn = 0;- unsigned int poll_cnt = WORKER_POOL_POLL_COUNT;- while (!atomic_load(&pool->killed)) {- unsigned int seqn = atomic_load(&pool->seqn);- if (seqn == prev_seqn) {- // drop HVX context while spinning- if (poll_cnt > 1 && poll_cnt == WORKER_POOL_POLL_COUNT) {- qurt_hvx_unlock();- }- if (--poll_cnt) {- hex_pause();- continue;- }- qurt_futex_wait(&pool->seqn, prev_seqn);- poll_cnt = WORKER_POOL_POLL_COUNT;- continue;- }-- prev_seqn = seqn;- poll_cnt = WORKER_POOL_POLL_COUNT;-- // New job- unsigned int n = atomic_load(&pool->n_jobs);- unsigned int i = atomic_fetch_add(&pool->next_job, 1);- if (i >= n) {- // Spurious wakeup- continue;- }-- pool->job[i].func(n, i, pool->job[i].data);-- atomic_fetch_sub(&pool->n_pending, 1);- }-- FARF(HIGH, "worker-pool: thread %u stopped", me->id);-}--AEEResult worker_pool_init_with_stack_size(worker_pool_context_t * context, uint32_t n_threads, uint32_t stack_size) {- int err = 0;-- if (NULL == context) {- FARF(ERROR, "NULL context passed to worker_pool_init().");- return AEE_EBADPARM;- }-- // Allocations- int size = (stack_size * n_threads) + (sizeof(worker_pool_t));-- unsigned char * mem_blob = (unsigned char *) malloc(size);- if (!mem_blob) {- FARF(ERROR, "Could not allocate memory for worker pool!!");- return AEE_ENOMEMORY;- }-- worker_pool_t * me = (worker_pool_t *) (mem_blob + stack_size * n_threads);-- // name for the first worker, useful in debugging threads- char name[19];- snprintf(name, 12, "0x%8x:", (int) me);- strcat(name, "worker0");- me->n_threads = n_threads;-- // initializations- for (unsigned int i = 0; i < me->n_threads; i++) {- me->stack[i] = NULL;- me->thread[i] = 0;-- me->context[i].id = i;- me->context[i].pool = me;- }-- // initialize job queue- me->n_pending = 0;- me->n_jobs = 0;- me->next_job = 0;- me->seqn = 0;- me->killed = 0;-- // launch the workers- qurt_thread_attr_t attr;- qurt_thread_attr_init(&attr);-- for (unsigned int i = 0; i < me->n_threads; i++) {- // set up stack- me->stack[i] = mem_blob;- mem_blob += stack_size;- qurt_thread_attr_set_stack_addr(&attr, me->stack[i]);- qurt_thread_attr_set_stack_size(&attr, stack_size);-- // set up name- qurt_thread_attr_set_name(&attr, name);- name[17] = (name[17] + 1);- // name threads context:worker0, context:worker1, .. (recycle at 9, but num threads should be less than that anyway)- if (name[17] > '9') {- name[17] = '0';- }-- // set up priority - by default, match the creating thread's prio- int prio = qurt_thread_get_priority(qurt_thread_get_id());-- if (prio < 1) {- prio = 1;- }- if (prio > LOWEST_USABLE_QURT_PRIO) {- prio = LOWEST_USABLE_QURT_PRIO;- }-- qurt_thread_attr_set_priority(&attr, prio);-- // launch- err = qurt_thread_create(&me->thread[i], &attr, worker_pool_main, (void *) &me->context[i]);- if (err) {- FARF(ERROR, "Could not launch worker threads!");- worker_pool_release((worker_pool_context_t *) &me);- return AEE_EQURTTHREADCREATE;- }- }- *context = (worker_pool_context_t *) me;- return AEE_SUCCESS;-}--AEEResult worker_pool_init(worker_pool_context_t * context, uint32_t n_threads) {- return worker_pool_init_with_stack_size(context, n_threads, WORKER_THREAD_STACK_SZ);-}--// clean up worker pool-void worker_pool_release(worker_pool_context_t * context) {- worker_pool_t * me = (worker_pool_t *) *context;-- // if no worker pool exists, return error.- if (NULL == me) {- return;- }-- atomic_store(&me->killed, 1);- atomic_fetch_add(&me->seqn, 1);- qurt_futex_wake(&me->seqn, me->n_threads);-- // de-initializations- for (unsigned int i = 0; i < me->n_threads; i++) {- if (me->thread[i]) {- int status;- (void) qurt_thread_join(me->thread[i], &status);- }- }-- // free allocated memory (were allocated as a single buffer starting at stack[0])- if (me->stack[0]) {- free(me->stack[0]);- }-- *context = NULL;-}--// run jobs-AEEResult worker_pool_run_jobs(worker_pool_context_t context, worker_pool_job_t * job, unsigned int n) {- worker_pool_t * me = (worker_pool_t *) context;- if (NULL == me) {- FARF(ERROR, "worker-pool: invalid context");- return AEE_EBADPARM;- }-- if (n > me->n_threads) {- FARF(ERROR, "worker-pool: invalid number of jobs %u for n-threads %u", n, me->n_threads);- return AEE_EBADPARM;- }-- memcpy(me->job, job, sizeof(worker_pool_job_t) * n);-- if (n > 1) {- atomic_store(&me->next_job, 1);- atomic_store(&me->n_jobs, n);- atomic_store(&me->n_pending, n - 1);-- // wake up workers- atomic_fetch_add(&me->seqn, 1);- qurt_futex_wake(&me->seqn, n - 1);- }-- // main thread runs job #0- me->job[0].func(n, 0, me->job[0].data);-- if (n > 1) {- while (atomic_load(&me->n_pending))- ;- }-- return 0;-}--// run func-AEEResult worker_pool_run_func(worker_pool_context_t context, worker_callback_t func, void * data, unsigned int n) {- worker_pool_job_t job[n];-- for (unsigned int i = 0; i < n; i++) {- job[i].func = func;- job[i].data = data;- }-- return worker_pool_run_jobs(context, job, n);-}--AEEResult worker_pool_set_thread_priority(worker_pool_context_t context, unsigned int prio) {- worker_pool_t * me = (worker_pool_t *) context;-- // if no worker pool exists, return error.- if (!me) {- return AEE_ENOMORE;- }-- int result = AEE_SUCCESS;- if (prio < 1) {- prio = 1;- }- if (prio > LOWEST_USABLE_QURT_PRIO) {- prio = LOWEST_USABLE_QURT_PRIO;- }-- for (unsigned int i = 0; i < me->n_threads; i++) {- int res = qurt_thread_set_priority(me->thread[i], (unsigned short) prio);- if (0 != res) {- result = AEE_EBADPARM;- FARF(ERROR, "QURT failed to set priority of thread %d, ERROR = %d", me->thread[i], res);- }- }-- return result;-}--AEEResult worker_pool_retrieve_thread_id(worker_pool_context_t context, unsigned int * tids) {- worker_pool_t * me = (worker_pool_t *) context;- if (!me) {- FARF(ERROR, "worker-pool: invalid context");- return AEE_EBADPARM;- ;- }-- for (int i = 0; i < me->n_threads; i++) {- tids[i] = me->thread[i];- }-- return AEE_SUCCESS;-}--AEEResult worker_pool_get_thread_priority(worker_pool_context_t context, unsigned int * prio) {- worker_pool_t * me = (worker_pool_t *) context;- if (!me) {- FARF(ERROR, "worker-pool: invalid context");- return AEE_EBADPARM;- }-- int priority = qurt_thread_get_priority(me->thread[0]);- if (priority > 0) {- *prio = priority;- return 0;- } else {- *prio = 0;- return AEE_EBADSTATE;- }-}diff --git a/ggml/src/ggml-hexagon/htp/worker-pool.h b/ggml/src/ggml-hexagon/htp/worker-pool.hdeleted file mode 100644index cba692126a..0000000000--- a/ggml/src/ggml-hexagon/htp/worker-pool.h+++ /dev/null@@ -1,65 +0,0 @@-#ifndef HTP_WORKER_POOL_H-#define HTP_WORKER_POOL_H--// MACRO enables function to be visible in shared-library case.-#define WORKERPOOL_API __attribute__((visibility("default")))--#include <AEEStdDef.h>-#include <AEEStdErr.h>-#include <stdint.h>--#ifdef __cplusplus-extern "C" {-#endif--/// signature of callbacks to be invoked by worker threads-typedef void (*worker_callback_t)(unsigned int n, unsigned int i, void *);--/// Typedef of worker_pool context-typedef void * worker_pool_context_t;--/// descriptor for requested callback-typedef struct {- worker_callback_t func;- void * data;-} worker_pool_job_t;--#define WORKER_THREAD_STACK_SZ (2 * 16384)--/// Maximum supported number of worker threads.-#define MAX_NUM_WORKERS 10--#if __HVX_ARCH__ > 79-#define WORKER_POOL_POLL_COUNT 2000-#else-#define WORKER_POOL_POLL_COUNT 1-#endif--// Initialize worker pool.-WORKERPOOL_API AEEResult worker_pool_init(worker_pool_context_t * context, uint32_t n_threads);--// Initialize worker pool with custom stack size-WORKERPOOL_API AEEResult worker_pool_init_with_stack_size(worker_pool_context_t * context,- uint32_t n_threads,- uint32_t stack_size);--// Kill worker threads and release worker pool resources-WORKERPOOL_API void worker_pool_release(worker_pool_context_t * context);--// Run jobs with the worker pool.-WORKERPOOL_API AEEResult worker_pool_run_jobs(worker_pool_context_t context, worker_pool_job_t * job, unsigned int n);--WORKERPOOL_API AEEResult worker_pool_run_func(worker_pool_context_t context,- worker_callback_t func,- void * data,- unsigned int n);--WORKERPOOL_API AEEResult worker_pool_set_thread_priority(worker_pool_context_t context, unsigned int prio);-WORKERPOOL_API AEEResult worker_pool_get_thread_priority(worker_pool_context_t context, unsigned int * prio);-WORKERPOOL_API AEEResult worker_pool_retrieve_thread_id(worker_pool_context_t context, unsigned int * tids);--#ifdef __cplusplus-}-#endif--#endif // #ifndef HTP_WORKER_POOL_Hdiff --git a/scripts/snapdragon/ggml-hexagon-profile.py b/scripts/snapdragon/ggml-hexagon-profile.pyindex 0f9240ddc6..2e096aaa93 100755--- a/scripts/snapdragon/ggml-hexagon-profile.py+++ b/scripts/snapdragon/ggml-hexagon-profile.py@@ -458,15 +458,11 @@ def main():if args.timeline:logger.info(f"\n# ASCII Timing {args.timeline.capitalize()}\n")- printed_cnt = 0for op in ops:if args.timeline == "summary":print_ascii_summary(op['name'], op['dims'], op['types'], op['usec'], op['cycles'], op['trace_events'], op.get('evt_val'))elif args.timeline == "diagram":print_ascii_timeline(op['name'], op['dims'], op['types'], op['usec'], op['cycles'], op['trace_events'], op.get('evt_val'))- printed_cnt += 1- if printed_cnt >= args.top:- breakelse:generate_report(ops, args.top, overrides, args.sort, pmu_name=final_pmu_name)