LLM inference in C/C++
C++ ggml
hexagon: enable alloc_buffer_n (#30126)
Max Krasnyansky9c2e0e491a82
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex 26b79582e8..f3ca731249 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -53,6 +53,7 @@#define GGML_COMMON_IMPL_CPP#include "ggml-backend-impl.h"+#include "ggml-alloc.h"#include "ggml-common.h"#include "ggml-hexagon.h"#include "ggml-impl.h"@@ -101,13 +102,16 @@ static size_t opt_ndev = 1;static size_t opt_nhvx = 0; // use allstatic int opt_nhmx = 1; // when set, enable HMX; when 0, use HVX onlystatic size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for buffer mappings-static size_t opt_mbuf = 1ul * 1024 * 1024 * 1024; // max buffer sizestatic int opt_etm = 0;static int opt_verbose = 0;static int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu)static bool opt_hostbuf = false;static bool opt_dma64 = false;+static size_t opt_mbuf_dyn = 512ul * 1024 * 1024; // max dynamic (compute) buffer size+static size_t opt_mbuf_static = 1ul * 1024 * 1024 * 1024; // max static (weight/KV) buffer size+static size_t opt_mbuf_total = 0; // total buffer space limit (0 = unconstrained)+static int opt_mm_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)static int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)static int opt_fa_head_split = 1; // 1 = partition flash_attn by KV heads in multicore (default on), 0 = token-based (original)@@ -121,7 +125,7 @@ static int opt_ar_scatter = 1; // 1 = reduce-scatter the fused ALLREDUCE+ADDstatic u32vec opt_pmu_evt { 0x3, 0x111, 0x100, 0x105, 0x240, 0x256, 0x7D, 0x8C };static int opt_opbatch = 1280; // max number of ops in a batch-static int opt_opqueue = 32; // max number of pending batches+static int opt_opqueue = 8; // max number of pending batchesstatic int opt_optrace = 0; // trace buffer size per thread (0 means default)static int opt_oppoll = 0; // polling for batch completionsstatic int opt_opfusion = 1; // enable/disable op fusion@@ -2820,8 +2824,30 @@ static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffeGGML_UNUSED(buft);}+static size_t parse_size(const char * str, size_t default_unit = 1024 * 1024) {+ if (!str || str[0] == '\0') {+ return 0;+ }+ char * end = NULL;+ double val = strtod(str, &end);+ if (val < 0) {+ return 0;+ }+ if (end && *end) {+ while (*end == ' ') end++;+ if (*end == 'k' || *end == 'K') {+ return (size_t) (val * 1024);+ } else if (*end == 'm' || *end == 'M') {+ return (size_t) (val * 1024 * 1024);+ } else if (*end == 'g' || *end == 'G') {+ return (size_t) (val * 1024 * 1024 * 1024);+ }+ }+ return (size_t) (val * default_unit);+}+static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {- return opt_mbuf;+ return opt_mbuf_dyn;GGML_UNUSED(buft);}@@ -2835,25 +2861,199 @@ static bool ggml_backend_hexagon_host_buffer_type_is_host(ggml_backend_buffer_tyGGML_UNUSED(buft);}+struct ggml_backend_hexagon_alloc_buffer_n_plan_item {+ size_t size;+ int first;+ int last;+};++using ggml_backend_hexagon_alloc_buffer_n_plan_t = std::vector<ggml_backend_hexagon_alloc_buffer_n_plan_item>;++static const char * ggml_hexagon_kv_layer_suffix(const struct ggml_tensor * t) {+ if (strncmp(t->name, "cache_", 6) != 0) {+ return NULL;+ }+ const char * p = strstr(t->name, "_l");+ if (!p || !isdigit((unsigned char)p[2])) {+ return NULL;+ }+ return p;+}++struct ggml_backend_hexagon_alloc_unit {+ size_t size;+ int first;+ int last;+};++static ggml_backend_hexagon_alloc_buffer_n_plan_t ggml_backend_hexagon_alloc_buffer_n_plan(+ ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) {+ ggml_backend_hexagon_alloc_buffer_n_plan_t plan;++ const size_t alignment = ggml_backend_buft_get_alignment(buft);+ const size_t max_size = opt_mbuf_static > 0 ? opt_mbuf_static : SIZE_MAX;++ std::vector<ggml_backend_hexagon_alloc_unit> units;++ int i = 0;+ while (i < n_tensors) {+ struct ggml_tensor * t = tensors[i];+ size_t unit_size = 0;+ int unit_first = i;+ int unit_last = i + 1;++ if (t->data == NULL && t->view_src == NULL) {+ unit_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);+ }++ const char * layer_suffix = ggml_hexagon_kv_layer_suffix(t);++ while (unit_last < n_tensors) {+ struct ggml_tensor * next = tensors[unit_last];++ if (next->view_src != NULL) {+ unit_last++;+ continue;+ }++ if (layer_suffix != NULL) {+ const char * next_suffix = ggml_hexagon_kv_layer_suffix(next);+ if (next_suffix != NULL && strcmp(layer_suffix, next_suffix) == 0) {+ if (next->data == NULL) {+ unit_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, next), alignment);+ }+ unit_last++;+ continue;+ }+ }++ break;+ }++ units.push_back({ unit_size, unit_first, unit_last });+ i = unit_last;+ }++ size_t cur_buf_size = 0;+ int cur_buf_first = 0;++ for (const auto & unit : units) {+ if (unit.size == 0) {+ continue;+ }++ if (cur_buf_size > 0 && (cur_buf_size + unit.size) > max_size) {+ plan.push_back({ cur_buf_size, cur_buf_first, unit.first });+ cur_buf_size = 0;+ cur_buf_first = unit.first;+ }++ cur_buf_size += unit.size;+ }++ if (cur_buf_size > 0) {+ plan.push_back({ cur_buf_size, cur_buf_first, n_tensors });+ }++ return plan;+}++static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer_n(+ ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) {+ const ggml_backend_hexagon_alloc_buffer_n_plan_t plan = ggml_backend_hexagon_alloc_buffer_n_plan(buft, tensors, n_tensors);++ std::vector<ggml_backend_buffer_t> buffers;+ buffers.reserve(plan.size());++ for (const auto & item : plan) {+ ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, item.size);+ if (buffer == NULL) {+ GGML_LOG_ERROR("%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(buft), item.size);+ for (ggml_backend_buffer_t b : buffers) {+ ggml_backend_buffer_free(b);+ }+ return NULL;+ }++ struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);++ struct ggml_tensor * t_failed = NULL;+ for (int j = item.first; j < item.last; j++) {+ struct ggml_tensor * t = tensors[j];+ if (t->data == NULL) {+ if (t->view_src == NULL) {+ if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) {+ t_failed = t;+ break;+ }+ } else if (t->buffer == NULL) {+ if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {+ t_failed = t;+ break;+ }+ }+ } else {+ if (t->view_src != NULL && t->buffer == NULL) {+ if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {+ t_failed = t;+ break;+ }+ }+ }+ }+ if (t_failed != NULL) {+ GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t_failed->name);+ for (ggml_backend_buffer_t b : buffers) {+ ggml_backend_buffer_free(b);+ }+ ggml_backend_buffer_free(buffer);+ return NULL;+ }++ buffers.push_back(buffer);+ }++ if (buffers.empty()) {+ return NULL;+ }++ if (buffers.size() == 1) {+ return buffers[0];+ }++ return ggml_backend_multi_buffer_alloc_buffer(buffers.data(), buffers.size());+}++static size_t ggml_backend_hexagon_buffer_type_get_alloc_size_n(+ ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) {+ const ggml_backend_hexagon_alloc_buffer_n_plan_t plan = ggml_backend_hexagon_alloc_buffer_n_plan(buft, tensors, n_tensors);++ size_t total = 0;+ for (const auto & item : plan) {+ total += item.size;+ }+ return total;+}+static ggml_backend_buffer_type_i ggml_backend_hexagon_buffer_type_interface = {/* .get_name = */ ggml_backend_hexagon_buffer_type_name,/* .alloc_buffer = */ ggml_backend_hexagon_buffer_type_alloc_buffer,- /* .alloc_buffer_n = */ NULL,+ /* .alloc_buffer_n = */ ggml_backend_hexagon_buffer_type_alloc_buffer_n,/* .get_alignment = */ ggml_backend_hexagon_buffer_type_get_alignment,/* .get_max_size = */ ggml_backend_hexagon_buffer_type_get_max_size,/* .get_alloc_size = */ ggml_backend_hexagon_buffer_type_get_alloc_size,- /* .get_alloc_size_n = */ NULL,+ /* .get_alloc_size_n = */ ggml_backend_hexagon_buffer_type_get_alloc_size_n,/* .is_host = */ ggml_backend_hexagon_buffer_type_is_host,};static ggml_backend_buffer_type_i ggml_backend_hexagon_host_buffer_type_interface = {/* .get_name = */ ggml_backend_hexagon_buffer_type_name,/* .alloc_buffer = */ ggml_backend_hexagon_host_buffer_type_alloc_buffer,- /* .alloc_buffer_n = */ NULL,+ /* .alloc_buffer_n = */ ggml_backend_hexagon_buffer_type_alloc_buffer_n,/* .get_alignment = */ ggml_backend_hexagon_buffer_type_get_alignment,/* .get_max_size = */ ggml_backend_hexagon_buffer_type_get_max_size,/* .get_alloc_size = */ ggml_backend_hexagon_buffer_type_get_alloc_size,- /* .get_alloc_size_n = */ NULL,+ /* .get_alloc_size_n = */ ggml_backend_hexagon_buffer_type_get_alloc_size_n,/* .is_host = */ ggml_backend_hexagon_host_buffer_type_is_host,};@@ -8545,8 +8745,8 @@ static const char * ggml_backend_hexagon_device_get_description(ggml_backend_dev}static void ggml_backend_hexagon_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {- *free = 0;- *total = *free;+ *free = opt_mbuf_total;+ *total = opt_mbuf_total;GGML_UNUSED(dev);}@@ -9295,7 +9495,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {size_t MiB = 1024 * 1024;// Update vmem default- opt_vmem = opt_arch >= 75 ? HTP_OP_MAX_VMEM_DEFAULT : 3000 * MiB;+ opt_vmem = opt_arch >= 75 ? HTP_OP_MAX_VMEM_DEFAULT : 3000 * MiB;opt_dma64 = opt_arch > 79 && (!str_dma64 || atoi(str_dma64) != 0);auto RE_ICASE = std::regex_constants::icase;@@ -9313,13 +9513,30 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {opt_nhmx = str_nhmx ? atoi(str_nhmx) : opt_nhmx;opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select;opt_fa_select = str_fa_select ? atoi(str_fa_select) : opt_fa_select;- opt_fa_head_split = str_fa_head_split ? atoi(str_fa_head_split) : opt_fa_head_split;- opt_gdn_select = str_gdn_select ? atoi(str_gdn_select) : opt_gdn_select;- opt_ar_select = str_ar_select ? atoi(str_ar_select) : opt_ar_select;- opt_ar_scatter = str_ar_scatter ? atoi(str_ar_scatter) : opt_ar_scatter;- opt_mbuf = str_mbuf ? strtoul(str_mbuf, NULL, 0) * MiB : opt_mbuf;- opt_vmem = str_vmem ? strtoul(str_vmem, NULL, 0) * MiB : opt_vmem;- opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) != 0 : opt_hostbuf;+ opt_fa_head_split = str_fa_head_split ? atoi(str_fa_head_split) : opt_fa_head_split;+ opt_gdn_select = str_gdn_select ? atoi(str_gdn_select) : opt_gdn_select;+ opt_ar_select = str_ar_select ? atoi(str_ar_select) : opt_ar_select;+ opt_ar_scatter = str_ar_scatter ? atoi(str_ar_scatter) : opt_ar_scatter;++ if (str_mbuf) {+ const char * p = str_mbuf;+ for (int idx = 0; idx < 3 && p && *p; idx++) {+ while (*p == ' ') p++;+ const char * comma = strchr(p, ',');+ size_t len = comma ? (size_t)(comma - p) : strlen(p);+ while (len > 0 && p[len - 1] == ' ') len--;+ if (len > 0) {+ std::string token(p, len);+ if (idx == 0) opt_mbuf_dyn = parse_size(token.c_str());+ if (idx == 1) opt_mbuf_static = parse_size(token.c_str());+ if (idx == 2) opt_mbuf_total = parse_size(token.c_str());+ }+ if (!comma) break;+ p = comma + 1;+ }+ }+ opt_vmem = str_vmem ? parse_size(str_vmem) : opt_vmem;+ opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) != 0 : opt_hostbuf;// Parse device configurationconst char * str_devices = getenv("GGML_HEXAGON_DEVICES");diff --git a/scripts/snapdragon/run.py b/scripts/snapdragon/run.pyindex 14e53aee61..382879f7ba 100755--- a/scripts/snapdragon/run.py+++ b/scripts/snapdragon/run.py@@ -152,6 +152,7 @@ def main():parser.add_argument("--profile", help="Profiling flag (enables Hexagon profiling and OpenCL autotuning)")parser.add_argument("--sched-debug", action="store_true", help="Enable GGML/llama.cpp scheduler debug output (GGML_SCHED_DEBUG=2)")parser.add_argument("--mtmd-device", help="Specify the backend device ID for Multi-Threaded Multi-Device setup (MTMD_BACKEND_DEVICE)")+ parser.add_argument("--no-embd-offload", action="store_true", help="Keep token embeddings and output projection on CPU (-ot token_embd.weight=CPU,output.weight=CPU)")# Hexagon specific parametersparser.add_argument("--hex-verbose", help="Enable verbose logging (GGML_HEXAGON_VERBOSE)")@@ -166,7 +167,7 @@ def main():parser.add_argument("--hex-opfilter", help="Regex pattern to filter/select which operators are offloaded to NPU (GGML_HEXAGON_OPFILTER)")parser.add_argument("--hex-opfusion", help="NPU graph node fusion optimization level (0: disabled, 1: enabled) (GGML_HEXAGON_OPFUSION)")parser.add_argument("--hex-vmem", help="Maximum NPU VMEM size limit in MB to allocate (GGML_HEXAGON_VMEM)")- parser.add_argument("--hex-mbuf", help="Maximum host buffer size limit in MB to allocate (GGML_HEXAGON_MBUF)")+ parser.add_argument("--hex-mbuf", help="Host buffer size limits in MB (supports K/M/G suffix): <dyn>[,<static>[,<total>]] (default: 512,1024,0) (GGML_HEXAGON_MBUF)")parser.add_argument("--hex-mm-select", help="Select MUL_MAT and MUL_MAT_ID kernel (GGML_HEXAGON_MM_SELECT) 2:HMX,1:HVX,0:disable")parser.add_argument("--hex-fa-select", help="Select Flash Attention kernel (GGML_HEXAGON_FA_SELECT) 2:HMX,1:HVX,0:disable")parser.add_argument("--hex-fa-head-split", help="Enable (1) or disable (0) head-parallel flash_attn partitioning (GGML_HEXAGON_FA_HEAD_SPLIT)")@@ -422,6 +423,9 @@ def main():if basename in ("llama-cli", "llama-completion", "llama-server", "llama-bench"):if "-t" not in cmd_args and "--threads" not in cmd_args:cmd_args += ["-t", "6"]+ if getattr(args, "no_embd_offload", False):+ if not any("token_embd" in arg or "output.weight" in arg for arg in cmd_args):+ cmd_args += ["-ot", r"^(token_embd|output)\.weight$=CPU"]# Resolve target directory on devicetarget_dir = args.target_dir