LLM inference in C/C++
C++ ggml
hexagon: add configurable HTP DCVS/HMX power mode (#30423)
Pranav Dhinakarec0bfac1794a
diff --git a/docs/backend/snapdragon/README.md b/docs/backend/snapdragon/README.mdindex ca79ca8852..b8347a224a 100644--- a/docs/backend/snapdragon/README.md+++ b/docs/backend/snapdragon/README.md@@ -330,6 +330,31 @@ on 4 physical NPUs, or `--devices 'HTP0[0-1:0],HTP1[0-1:1]'` on 2 physical NPUs- `GGML_HEXAGON_DMA64=0` (default: enabled on v81+)Disables 64-bit DMA for model weights. Set to `1` to enable it explicitly on a supported architecture.+- `GGML_HEXAGON_POWER_MODE=default`+ Controls the DCVS/HMX power mode requested from the HTP DSP for the session. Modes are listed+ from lowest to highest power. All modes except `burst` leave DCVS enabled and keep the HTP awake.++ - `low_power_saver`+ - `power_saver`+ - `high_power_saver`+ - `low_balanced`+ - `balanced`+ - `high_performance`+ - `sustained_high_performance`+ - `burst` (maximum clocks, disables DCVS; may thermal-throttle on sustained workloads)+ - `default` (currently aliases to `burst`)++ Unknown values are logged as a warning and fall back to `burst`.++ This must be set in the environment before the process initializes the backend registry, so+ there is no `llama-*` CLI flag for it. The `run.py` runner exposes it as `--hex-power-mode`.++ Examples:++ `GGML_HEXAGON_POWER_MODE=balanced llama-completion ...`+ `./scripts/snapdragon/run.py --hex-power-mode sustained_high_performance ...`++- `GGML_HEXAGON_VERBOSE=1`Enables verbose logging of Ops from the backend. Example output:diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex 1c1c92da20..08c15ff786 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -108,6 +108,41 @@ static int opt_verbose = 0;static int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu)static bool opt_hostbuf = false;static bool opt_dma64 = false;+static int opt_power_mode = HTP_POWER_MODE_BURST; // DCVS/HMX power profile requested at session start++static const std::pair<const char *, int> htp_power_mode_names[] = {+ { "low_power_saver", HTP_POWER_MODE_LOW_POWER_SAVER },+ { "power_saver", HTP_POWER_MODE_POWER_SAVER },+ { "high_power_saver", HTP_POWER_MODE_HIGH_POWER_SAVER },+ { "low_balanced", HTP_POWER_MODE_LOW_BALANCED },+ { "balanced", HTP_POWER_MODE_BALANCED },+ { "high_performance", HTP_POWER_MODE_HIGH_PERFORMANCE },+ { "sustained_high_performance", HTP_POWER_MODE_SUSTAINED_HIGH_PERFORMANCE },+ { "burst", HTP_POWER_MODE_BURST },+ { "default", HTP_POWER_MODE_BURST }, // flip this to change the default mode+};++static int htp_power_mode_from_str(const char * s) {+ for (auto & e : htp_power_mode_names) {+#if defined(_MSC_VER)+ if (_stricmp(s, e.first) == 0) {+#else+ if (strcasecmp(s, e.first) == 0) {+#endif+ return e.second;+ }+ }+ return -1;+}++static const char * htp_power_mode_to_str(int mode) {+ for (auto & e : htp_power_mode_names) {+ if (e.second == mode) {+ return e.first;+ }+ }+ return "unknown";+}static size_t opt_mbuf_dyn = 512ul * 1024 * 1024; // max dynamic (compute) buffer sizestatic size_t opt_mbuf_static = 1ul * 1024 * 1024 * 1024; // max static (weight/KV) buffer size@@ -4956,12 +4991,13 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) nthis->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem);// Start dspqueue/opbatch processing- err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem);+ err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem, (uint32_t) opt_power_mode);if (err != 0) {GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err);throw std::runtime_error("ggml-hex: iface start failed (see log for details)");}this->valid_iface = true;+ GGML_LOG_INFO("ggml-hex: %s power mode: %s\n", this->c_name(), htp_power_mode_to_str(opt_power_mode));if (opt_profile) {htp_iface_pmu_conf pmu_conf{};@@ -9959,6 +9995,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {const char * str_optrace = getenv("GGML_HEXAGON_OPTRACE");const char * str_hostbuf = getenv("GGML_HEXAGON_HOSTBUF");const char * str_dma64 = getenv("GGML_HEXAGON_DMA64");+ const char * str_power_mode = getenv("GGML_HEXAGON_POWER_MODE");// Init Arch first since it affects other defaultsif (!str_arch) {@@ -9999,6 +10036,14 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {opt_opfusion = str_opfusion ? atoi(str_opfusion) : opt_opfusion;opt_profile = str_profile ? atoi(str_profile) : 0;opt_etm = str_etm ? atoi(str_etm) : 0;+ if (str_power_mode) {+ int m = htp_power_mode_from_str(str_power_mode);+ if (m < 0) {+ GGML_LOG_WARN("ggml-hex: unknown GGML_HEXAGON_POWER_MODE '%s', falling back to 'burst'\n", str_power_mode);+ } else {+ opt_power_mode = m;+ }+ }opt_nhvx = str_nhvx ? strtoul(str_nhvx, NULL, 0) : opt_nhvx;opt_nhmx = str_nhmx ? atoi(str_nhmx) : opt_nhmx;opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select;diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.hindex 5405c83b65..70d56e181a 100644--- a/ggml/src/ggml-hexagon/htp/htp-ops.h+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h@@ -197,6 +197,20 @@ enum htp_profiler_mode {HTP_PROF_TRACE = 3,};+// DCVS/HMX power profile requested for the session.+enum htp_power_mode {+ HTP_POWER_MODE_LOW_POWER_SAVER = 0,+ HTP_POWER_MODE_POWER_SAVER = 1,+ HTP_POWER_MODE_HIGH_POWER_SAVER = 2,+ HTP_POWER_MODE_LOW_BALANCED = 3,+ HTP_POWER_MODE_BALANCED = 4,+ HTP_POWER_MODE_HIGH_PERFORMANCE = 5,+ HTP_POWER_MODE_SUSTAINED_HIGH_PERFORMANCE = 6,+ HTP_POWER_MODE_BURST = 7,++ HTP_POWER_MODE_COUNT+};+enum htp_trace_event_id {HTP_TRACE_EVT_DMA = 0,HTP_TRACE_EVT_L2FLUSH = 1,diff --git a/ggml/src/ggml-hexagon/htp/htp_iface.idl b/ggml/src/ggml-hexagon/htp/htp_iface.idlindex b46e252965..303dbfb0aa 100644--- a/ggml/src/ggml-hexagon/htp/htp_iface.idl+++ b/ggml/src/ggml-hexagon/htp/htp_iface.idl@@ -11,7 +11,7 @@ struct htp_iface_pmu_conf {};interface htp_iface : remote_handle64 {- AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 max_vmem);+ AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 max_vmem, in uint32 power_mode);AEEResult stop();AEEResult mmap(in uint32 fd, in uint64 size);AEEResult munmap(in uint32 fd);diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.cindex aff1c6c0f7..df7c4a5bd1 100644--- a/ggml/src/ggml-hexagon/htp/main.c+++ b/ggml/src/ggml-hexagon/htp/main.c@@ -363,12 +363,91 @@ static void htp_main_thread(void * context);static void htp_packet_callback(dspqueue_t queue, int error, void * context);static void htp_error_callback(dspqueue_t queue, int error, void * context);-AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t max_vmem) {+// Non-BURST modes leave DCVS enabled and keep the HTP awake.+// BURST pins all corners to MAX and disables DCVS.+// Corner constants are from HAP_power.h (HAP_dcvs_voltage_corner_t / HAP_dcvs_exp_vcorner_t).+struct htp_power_profile {+ boolean dcvs_enable;+ uint32_t bus_min_corner;+ uint32_t bus_max_corner;+ uint32_t bus_target_corner;+ uint32_t core_min_corner;+ uint32_t core_max_corner;+ uint32_t core_target_corner;+ boolean sleep_disable;+ uint32_t hmx_min_corner; // only used when __HVX_ARCH__ >= 75+ uint32_t hmx_max_corner;+ uint32_t hmx_target_corner;+ uint32_t hmx_perf_mode;+};++#define HTP_POWER_PROFILE_ENTRY(min_c, max_c, hmx_c) \+ { \+ .dcvs_enable = TRUE, \+ .bus_min_corner = (min_c), \+ .bus_max_corner = (max_c), \+ .bus_target_corner = (max_c), \+ .core_min_corner = (min_c), \+ .core_max_corner = (max_c), \+ .core_target_corner = (max_c), \+ .sleep_disable = TRUE, \+ .hmx_min_corner = (hmx_c), \+ .hmx_max_corner = (hmx_c), \+ .hmx_target_corner = (hmx_c), \+ .hmx_perf_mode = HAP_CLK_PERF_HIGH, \+ }++static const struct htp_power_profile htp_power_profiles[HTP_POWER_MODE_COUNT] = {+ [HTP_POWER_MODE_LOW_POWER_SAVER] =+ HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_SVS2, HAP_DCVS_VCORNER_SVS2, HAP_DCVS_EXP_VCORNER_MIN_SVS),+ [HTP_POWER_MODE_POWER_SAVER] =+ HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_SVS, HAP_DCVS_VCORNER_SVS_PLUS, HAP_DCVS_EXP_VCORNER_SVS),+ [HTP_POWER_MODE_HIGH_POWER_SAVER] =+ HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_SVS_PLUS, HAP_DCVS_VCORNER_NOM, HAP_DCVS_EXP_VCORNER_SVS_L2),+ [HTP_POWER_MODE_LOW_BALANCED] =+ HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_NOM, HAP_DCVS_VCORNER_NOM_PLUS, HAP_DCVS_EXP_VCORNER_NOM),+ [HTP_POWER_MODE_BALANCED] =+ HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_NOM_PLUS, HAP_DCVS_VCORNER_TURBO, HAP_DCVS_EXP_VCORNER_NOM_L1),+ [HTP_POWER_MODE_HIGH_PERFORMANCE] =+ HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_TURBO, HAP_DCVS_VCORNER_TURBO_PLUS, HAP_DCVS_EXP_VCORNER_TUR_L2),+ [HTP_POWER_MODE_SUSTAINED_HIGH_PERFORMANCE] =+ HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_NOM_PLUS, HAP_DCVS_VCORNER_TURBO_PLUS, HAP_DCVS_EXP_VCORNER_TUR_L2),+ [HTP_POWER_MODE_BURST] = {+ .dcvs_enable = FALSE,+ .bus_min_corner = HAP_DCVS_VCORNER_MAX,+ .bus_max_corner = HAP_DCVS_VCORNER_MAX,+ .bus_target_corner = HAP_DCVS_VCORNER_MAX,+ .core_min_corner = HAP_DCVS_VCORNER_MAX,+ .core_max_corner = HAP_DCVS_VCORNER_MAX,+ .core_target_corner = HAP_DCVS_VCORNER_MAX,+ .sleep_disable = TRUE,+ .hmx_min_corner = HAP_DCVS_EXP_VCORNER_MAX,+ .hmx_max_corner = HAP_DCVS_EXP_VCORNER_MAX,+ .hmx_target_corner = HAP_DCVS_EXP_VCORNER_MAX,+ .hmx_perf_mode = HAP_CLK_PERF_HIGH,+ },+};++#undef HTP_POWER_PROFILE_ENTRY++AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t max_vmem, uint32_t power_mode) {struct htp_handle * h = (struct htp_handle *) handle;if (!h) {return AEE_EBADPARM;}+ if (power_mode >= HTP_POWER_MODE_COUNT) {+ FARF(ERROR, "invalid power-mode %u, falling back to burst", (unsigned) power_mode);+ power_mode = HTP_POWER_MODE_BURST;+ }+ const struct htp_power_profile * pwr = &htp_power_profiles[power_mode];+ FARF(ALWAYS, "power-mode %u : dcvs-enable %d bus[%u,%u,%u] core[%u,%u,%u] sleep-disable %d hmx[%u,%u,%u]",+ (unsigned) power_mode, (int) pwr->dcvs_enable,+ (unsigned) pwr->bus_min_corner, (unsigned) pwr->bus_max_corner, (unsigned) pwr->bus_target_corner,+ (unsigned) pwr->core_min_corner, (unsigned) pwr->core_max_corner, (unsigned) pwr->core_target_corner,+ (int) pwr->sleep_disable,+ (unsigned) pwr->hmx_min_corner, (unsigned) pwr->hmx_max_corner, (unsigned) pwr->hmx_target_corner);+if (h->ctx) {FARF(ERROR, "Queue already open");return AEE_EITEMBUSY;@@ -499,17 +578,17 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsprequest.type = HAP_power_set_DCVS_v3;request.dcvs_v3.set_dcvs_enable = TRUE;- request.dcvs_v3.dcvs_enable = FALSE;+ request.dcvs_v3.dcvs_enable = pwr->dcvs_enable;request.dcvs_v3.set_bus_params = TRUE;- request.dcvs_v3.bus_params.min_corner = HAP_DCVS_VCORNER_MAX;- request.dcvs_v3.bus_params.max_corner = HAP_DCVS_VCORNER_MAX;- request.dcvs_v3.bus_params.target_corner = HAP_DCVS_VCORNER_MAX;+ request.dcvs_v3.bus_params.min_corner = pwr->bus_min_corner;+ request.dcvs_v3.bus_params.max_corner = pwr->bus_max_corner;+ request.dcvs_v3.bus_params.target_corner = pwr->bus_target_corner;request.dcvs_v3.set_core_params = TRUE;- request.dcvs_v3.core_params.min_corner = HAP_DCVS_VCORNER_MAX;- request.dcvs_v3.core_params.max_corner = HAP_DCVS_VCORNER_MAX;- request.dcvs_v3.core_params.target_corner = HAP_DCVS_VCORNER_MAX;+ request.dcvs_v3.core_params.min_corner = pwr->core_min_corner;+ request.dcvs_v3.core_params.max_corner = pwr->core_max_corner;+ request.dcvs_v3.core_params.target_corner = pwr->core_target_corner;request.dcvs_v3.set_sleep_disable = TRUE;- request.dcvs_v3.sleep_disable = TRUE;+ request.dcvs_v3.sleep_disable = pwr->sleep_disable;#if (__HEXAGON_ARCH__ >= 79)HAP_set_dcvs_v3_protected_bus_corners(&request, 1);@@ -537,10 +616,10 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsprequest.hmx_v2.set_power = TRUE;request.hmx_v2.power_up = TRUE;request.hmx_v2.set_clock = TRUE;- request.hmx_v2.target_corner = HAP_DCVS_EXP_VCORNER_MAX;- request.hmx_v2.min_corner = HAP_DCVS_EXP_VCORNER_MAX;- request.hmx_v2.max_corner = HAP_DCVS_EXP_VCORNER_MAX;- request.hmx_v2.perf_mode = HAP_CLK_PERF_HIGH;+ request.hmx_v2.target_corner = pwr->hmx_target_corner;+ request.hmx_v2.min_corner = pwr->hmx_min_corner;+ request.hmx_v2.max_corner = pwr->hmx_max_corner;+ request.hmx_v2.perf_mode = pwr->hmx_perf_mode;FARF(ALWAYS, "Setting HMX clock\n");err = HAP_power_set((void *) ctx, &request);if (err != AEE_SUCCESS) {diff --git a/scripts/snapdragon/run.py b/scripts/snapdragon/run.pyindex 382879f7ba..dec4b996fc 100755--- a/scripts/snapdragon/run.py+++ b/scripts/snapdragon/run.py@@ -159,6 +159,7 @@ def main():parser.add_argument("--hex-profile", help="Enable NPU/Hexagon profiling and performance metrics print (GGML_HEXAGON_PROFILE)")parser.add_argument("--hex-nhvx", help="Number of HVX units to use (GGML_HEXAGON_NHVX)")parser.add_argument("--hex-nhmx", help="Number of HMX units to use. 0 disables HMX power-up (GGML_HEXAGON_NHMX)")+ parser.add_argument("--hex-power-mode", help="HTP DCVS/HMX power mode: low_power_saver, power_saver, high_power_saver, low_balanced, balanced, high_performance, sustained_high_performance, burst (GGML_HEXAGON_POWER_MODE)")parser.add_argument("--hex-hostbuf", help="Enable host buffers (GGML_HEXAGON_HOSTBUF)")parser.add_argument("--hex-dma64", nargs="?", const="1", help="Enable (1) or disable (0) 64-bit DMA for model weights (GGML_HEXAGON_DMA64)")parser.add_argument("--hex-opbatch", help="Maximum number of operations to batch into a single HTP execution (GGML_HEXAGON_OPBATCH)")@@ -302,6 +303,7 @@ def main():# Other Hexagon environment variablesset_env("GGML_HEXAGON_NHVX", args.hex_nhvx)set_env("GGML_HEXAGON_NHMX", args.hex_nhmx)+ set_env("GGML_HEXAGON_POWER_MODE", args.hex_power_mode)set_env("GGML_HEXAGON_HOSTBUF", args.hex_hostbuf)set_env("GGML_HEXAGON_DMA64", args.hex_dma64)set_env("GGML_HEXAGON_OPBATCH", args.hex_opbatch)