LLM inference in C/C++
C++ ggml
hex-dma: introduce direct-mapped DMA cache that is better suited for HVX FA mask handling (#29282)
Max Krasnyanskye6ab7c1a4105
diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.hindex d256e6bef4..c77359f892 100644--- a/ggml/src/ggml-hexagon/htp/dma-queue.h+++ b/ggml/src/ggml-hexagon/htp/dma-queue.h@@ -428,15 +428,16 @@ static inline bool dma_queue_push(dma_queue *q, dma_data ddata, size_t dst_strid#define DMA_CACHE_MAX_SIZE 256U+// Fully assoc LRU cachetypedef struct {uint8_t *base;uint32_t line_size;uint32_t capacity;dma_addr_t src[DMA_CACHE_MAX_SIZE];uint16_t age[DMA_CACHE_MAX_SIZE];-} dma_cache;+} dma_cache_fa;-static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_size, uint32_t capacity)+static inline void dma_cache_fa_init(dma_cache_fa *c, uint8_t *base, uint32_t line_size, uint32_t capacity){c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity;c->base = base;@@ -448,7 +449,7 @@ static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_siz}}-static inline bool dma_cache_push(dma_queue *q, dma_cache *c, dma_addr_t src_addr, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows)+static inline bool dma_cache_fa_push(dma_queue *q, dma_cache_fa *c, dma_addr_t src_addr, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows){uint32_t o_idx = 0;uint16_t o_age = 0;@@ -473,6 +474,40 @@ static inline bool dma_cache_push(dma_queue *q, dma_cache *c, dma_addr_t src_addreturn dma_queue_push_single_1d(q, dma_make_data(dst, src_addr), 0);}+// Direct mapped cache+typedef struct {+ uint8_t *base;+ uint32_t line_size;+ uint32_t capacity;+ uint32_t idx_mask;+ dma_addr_t src[DMA_CACHE_MAX_SIZE];+} dma_cache_dm;++static inline void dma_cache_dm_init(dma_cache_dm *c, uint8_t *base, uint32_t line_size, uint32_t capacity)+{+ c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity;+ c->idx_mask = c->capacity - 1;+ c->base = base;+ c->line_size = line_size;++ for (unsigned i=0; i < c->capacity; i++) {+ c->src[i] = 0;+ }+}++static inline bool dma_cache_dm_push(dma_queue *q, dma_cache_dm *c, uint32_t slot, dma_addr_t src_addr, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows)+{+ const uint32_t i = slot & c->idx_mask;+ uint8_t * dst = c->base + (i * c->line_size);++ if (c->src[i] == src_addr) {+ return dma_queue_push_single_1d(q, dma_make_data(dst, src_addr), 0); // dummy dma+ }++ c->src[i] = src_addr;+ return dma_queue_push(q, dma_make_data(dst, src_addr), dst_stride, src_stride, row_size, nrows);+}+#ifdef __cplusplus} // extern "C"#endifdiff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.cindex bfcf7cb0c1..f079f73892 100644--- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c+++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c@@ -160,7 +160,7 @@ struct hmx_fa_context {size_t col_vec_bytes;size_t d_tile_bytes;bool mask_broadcast; // true when mask->ne[2] == 1 (head-independent, single 2D DMA)- dma_cache m_cache;+ dma_cache_fa m_cache;};static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * data) {@@ -233,8 +233,8 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *uint8_t * spad_m = factx->spad_m + (mask ? factx->size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * ith;uint8_t * spad_a = factx->spad_a + factx->size_vkq_acc * ith;- dma_cache m_cache;- dma_cache_init(&m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE);+ dma_cache_dm m_cache;+ dma_cache_dm_init(&m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE);const size_t size_vkq_acc_single = hex_round_up(DV * sizeof(float), 128);@@ -328,7 +328,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *// Maskif (mask) {const dma_addr_t m_src = mp_base + ic_start * sizeof(__fp16);- dma_cache_push(dma_q, &m_cache, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1);+ dma_cache_dm_push(dma_q, &m_cache, ib, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1);}}@@ -537,7 +537,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *// Maskif (mask) {const dma_addr_t m_src = mp_base + next_ic_start * sizeof(__fp16);- dma_cache_push(dma_q, &m_cache, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1);+ dma_cache_dm_push(dma_q, &m_cache, next_ib, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1);}}} // end for g@@ -1796,7 +1796,7 @@ static inline void fa_prefetch_block(dma_queue * dma_q, const struct htp_tensorif (mask) {if (__builtin_expect(factx->mask_broadcast, true)) {const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + prefetch_start * sizeof(__fp16);- dma_cache_push(dma_q, &factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q);+ dma_cache_fa_push(dma_q, &factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q);} else {fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, prefetch_start, kv_head, G, m_line_bytes, prefetch_rows, n_rows_q, factx);}@@ -1975,7 +1975,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {const size_t m_line_bytes = L.m_line_bytes; // used by the mask DMAs in the KV loop- dma_cache_init(&factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE);+ dma_cache_fa_init(&factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE);// Head-dim padding: the K/V DMA staging buffers and the flat-Q buffer are laid out// with padded row strides (size_{k,v,q}_row_padded, covering D_pad columns) but the@@ -2057,7 +2057,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {if (factx.pipeline && mask) {if (__builtin_expect(factx.mask_broadcast, true)) {const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + 0;- dma_cache_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), n_rows_q);+ dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), n_rows_q);} else {fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, 0, kv_head, G, m_line_bytes, kv_rows0, n_rows_q, &factx);}@@ -2254,7 +2254,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {if (mask) {if (__builtin_expect(factx.mask_broadcast, true)) {const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + kv_start * sizeof(__fp16);- dma_cache_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q);+ dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q);} else {fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, kv_start, kv_head, G, m_line_bytes, kv_rows, n_rows_q, &factx);}@@ -2398,7 +2398,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {}if (__builtin_expect(factx.mask_broadcast, true)) {const dma_addr_t ms_src = mask->data + next_q_start * mask->nb[1] + next_im3 * mask->nb[3] + 0;- dma_cache_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), next_n_rows_q);+ dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), next_n_rows_q);} else {fa_push_mask_dma_gqa(dma_q, mask, next_q_start, next_im3, 0, next_kv_head, G, m_line_bytes, kv_rows0, next_n_rows_q, &factx);}