LLM inference in C/C++
C++ ggml
src : add n_expert_used_max function (#28323)
Daniel Bevenius9a4843cf2f1a
diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cppindex 7df82ffe21..34b3c68801 100644--- a/src/llama-hparams.cpp+++ b/src/llama-hparams.cpp@@ -87,6 +87,15 @@ uint32_t llama_hparams::n_expert_used(uint32_t il) const {GGML_ABORT("fatal error");}+uint32_t llama_hparams::n_expert_used_max() const {+ uint32_t val = 0;+ for (uint32_t il = 0; il < n_layer_all; ++il) {+ val = std::max(val, n_expert_used(il));+ }++ return val;+}+uint32_t llama_hparams::n_gqa(uint32_t il) const {const uint32_t n_head = this->n_head(il);const uint32_t n_head_kv = this->n_head_kv(il);diff --git a/src/llama-hparams.h b/src/llama-hparams.hindex 2f238a1744..e9029ff34b 100644--- a/src/llama-hparams.h+++ b/src/llama-hparams.h@@ -392,6 +392,9 @@ struct llama_hparams {uint32_t n_expert_used(uint32_t il = 0) const;+ // return the maximum n_expert_used across all layers+ uint32_t n_expert_used_max() const;+uint32_t n_gqa(uint32_t il = 0) const;uint32_t n_rot(uint32_t il = 0) const;diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cppindex 49f3c4f8ea..91bb5e7cc8 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -951,7 +951,7 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * wcase GGML_OP_MUL_MAT_ID:{// Used for either MoE expert routing or embedded adapter routing- const int n_ids_used = hparams.router_layer >= 0 ? 1 : hparams.n_expert_used();+ const int n_ids_used = hparams.router_layer >= 0 ? 1 : hparams.n_expert_used_max();GGML_ASSERT(n_ids_used > 0);ggml_tensor * b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_ids_used, 512);ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_ids_used, 512);@@ -964,7 +964,7 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w} break;case GGML_OP_ADD_ID:{- const int n_expert_used = hparams.n_expert_used();+ const int n_expert_used = hparams.n_expert_used_max();GGML_ASSERT(n_expert_used > 0);ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512);ggml_tensor * c = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512);diff --git a/src/llama-model.cpp b/src/llama-model.cppindex 6344f2d8ae..f22e35ed93 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -1254,10 +1254,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {}// models may route a different number of experts per layer, so validate the maximum- uint32_t n_expert_used_max = 0;- for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {- n_expert_used_max = std::max(n_expert_used_max, hparams.n_expert_used(il));- }+ uint32_t n_expert_used_max = hparams.n_expert_used_max();GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS);GGML_ASSERT(n_expert_used_max <= hparams.n_expert);@@ -1509,10 +1506,9 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {// TODO: move to a separate functionconst auto tn = LLM_TN(arch);- const int64_t n_expert = hparams.n_expert;- const int64_t n_expert_used = hparams.n_expert_used();+ const int64_t n_expert = hparams.n_expert;- if (n_expert > 0 && n_expert_used == 0) {+ if (n_expert > 0 && hparams.n_expert_used_max() == 0) {throw std::runtime_error("model has expert layers but no expert layers are used");}