LLM inference in C/C++
C++ ggml
model: add NVIDIA Nemotron-3-Puzzle-75B-A9B (NemotronHPuzzle) support (#25444)
Yaniss Amazouzc61b98b875ea
diff --git a/conversion/__init__.py b/conversion/__init__.pyindex 254a3e6c82..ba73192efa 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -188,6 +188,7 @@ TEXT_MODEL_MAP: dict[str, str] = {"NanbeigeForCausalLM": "nanbeige","NemotronForCausalLM": "nemotron","NemotronHForCausalLM": "nemotron",+ "NemotronHPuzzleForCausalLM": "nemotron","NeoBERT": "bert","NeoBERTForSequenceClassification": "bert","NeoBERTLMHead": "bert",diff --git a/conversion/nemotron.py b/conversion/nemotron.pyindex 07fbc65314..c7adb2e27a 100644--- a/conversion/nemotron.py+++ b/conversion/nemotron.py@@ -5,6 +5,7 @@ from typing import Any, Callable, Iterable, TYPE_CHECKINGimport torchif TYPE_CHECKING:+ from pathlib import Pathfrom torch import Tensorfrom .base import MmprojModel, ModelBase, TextModel, gguf, logger@@ -201,6 +202,7 @@ class NemotronHModel(GraniteHybridModel):model_arch = gguf.MODEL_ARCH.NEMOTRON_His_moe: bool = Falsesupports_mtp_export = True+ _experts: list[dict[str, Tensor]] | None = None_SSM_LAYER_TYPES = {"mamba", "linear_attention"}_ATTN_LAYER_TYPES = {"attention", "full_attention"}@@ -513,3 +515,88 @@ class NemotronHModel(GraniteHybridModel):experts = [k for d in self._experts for k in d.keys()]if len(experts) > 0:raise ValueError(f"Unprocessed experts: {experts}")++[email protected]("NemotronHPuzzleForCausalLM")[email protected]("nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16")+class NemotronHPuzzleModel(NemotronHModel):+ """NVIDIA Puzzle: NemotronH with a per-block MoE config (block_configs).++ The checkpoint also ships an MTP draft head (mtp.safetensors). It is skipped+ here: there is no Puzzle MTP inference path in tree, and the head is laid out+ by mtp_block_configs rather than the mtp.layers.* form NemotronHModel maps."""++ model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE+ is_moe: bool = True+ supports_mtp_export = False++ def __init__(self, dir_model: "Path", *args, **kwargs):+ hparams = dict(kwargs.pop("hparams", None) or ModelBase.load_hparams(dir_model, self.is_mistral_format))++ self.block_configs: list[dict] = hparams["block_configs"]+ self.n_layer_trunk = len(self.block_configs)++ # block_configs carries the per-block MoE shape, and is the authority on the+ # block pattern too: the layers_block_type the HF config wrapper computes is+ # not sized to it.+ hparams["num_hidden_layers"] = self.n_layer_trunk+ hparams["layers_block_type"] = [bc["block_type"] for bc in self.block_configs]++ self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE++ # Bypass NemotronHModel.__init__: it assumes a flat num_experts_per_tok /+ # moe_intermediate_size and a layers_block_type sized to block_count, neither+ # of which hold for Puzzle's per-block config.+ GraniteHybridModel.__init__(self, dir_model, *args, hparams=hparams, **kwargs)++ self.head_dim = self.find_hparam(["head_dim", "attention_head_dim"])+ self.d_inner = self.find_hparam(["num_heads"]) * self.d_model++ # NemotronHModel.__init__ folds an MTP block into block_count when the+ # config carries num_nextn_predict_layers; Puzzle's config does, but its+ # head has a different layout and no inference path, so stay opted out.+ self._mtp_bid = None++ def set_gguf_parameters(self):+ GraniteHybridModel.set_gguf_parameters(self)++ head_dim = self.head_dim+ if head_dim is None:+ raise ValueError("Could not find the attention head dim in config")+ self.gguf_writer.add_key_length(head_dim)+ self.gguf_writer.add_value_length(head_dim)++ ffn_lengths = [bc.get("moe_intermediate_size") or 0 for bc in self.block_configs]+ experts_used = [bc.get("num_experts_per_tok") or 0 for bc in self.block_configs]++ self.gguf_writer.add_feed_forward_length(ffn_lengths)+ self.gguf_writer.add_expert_feed_forward_length(ffn_lengths)+ self.gguf_writer.add_expert_used_count(experts_used)++ self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["moe_shared_expert_intermediate_size"])+ self.gguf_writer.add_expert_count(self.hparams["n_routed_experts"])+ self.gguf_writer.add_expert_shared_count(self.hparams["n_shared_experts"])+ self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])+ self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])+ self.gguf_writer.add_expert_group_count(self.hparams["n_group"])+ self.gguf_writer.add_moe_latent_size(self.hparams["moe_latent_size"])++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ # The official BF16 checkpoint (NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16)+ # names the trunk "model.*" (model.layers.*, model.embeddings, model.norm_f)+ # where the original release used the NemotronH-style "backbone.*", and spells+ # the router bias "e_score_correction_bias" instead of "e_score_correction.bias";+ # normalize so both convert identically.+ if name.startswith("model."):+ name = "backbone." + name[len("model."):]+ if name.endswith("mixer.gate.e_score_correction_bias"):+ name = name[: -len("e_score_correction_bias")] + "e_score_correction.bias"++ yield from super().modify_tensors(data_torch, name, bid)++ @classmethod+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:+ # Drop the MTP head unconditionally; see the class docstring.+ if item[0].startswith("mtp."):+ return None+ return super().filter_tensors(item)diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.pyindex d95fe9b1ac..689c2fca11 100644--- a/gguf-py/gguf/gguf_writer.py+++ b/gguf-py/gguf/gguf_writer.py@@ -733,8 +733,11 @@ class GGUFWriter:else:self.add_array(Keys.LLM.FEED_FORWARD_LENGTH.format(arch=self.arch), length)- def add_expert_feed_forward_length(self, length: int) -> None:- self.add_uint32(Keys.LLM.EXPERT_FEED_FORWARD_LENGTH.format(arch=self.arch), length)+ def add_expert_feed_forward_length(self, length: int | Sequence[int]) -> None:+ if isinstance(length, int):+ self.add_uint32(Keys.LLM.EXPERT_FEED_FORWARD_LENGTH.format(arch=self.arch), length)+ else:+ self.add_array(Keys.LLM.EXPERT_FEED_FORWARD_LENGTH.format(arch=self.arch), length)def add_expert_shared_feed_forward_length(self, length: int) -> None:self.add_uint32(Keys.LLM.EXPERT_SHARED_FEED_FORWARD_LENGTH.format(arch=self.arch), length)@@ -860,8 +863,11 @@ class GGUFWriter:def add_expert_count(self, count: int) -> None:self.add_uint32(Keys.LLM.EXPERT_COUNT.format(arch=self.arch), count)- def add_expert_used_count(self, count: int) -> None:- self.add_uint32(Keys.LLM.EXPERT_USED_COUNT.format(arch=self.arch), count)+ def add_expert_used_count(self, count: int | Sequence[int]) -> None:+ if isinstance(count, int):+ self.add_uint32(Keys.LLM.EXPERT_USED_COUNT.format(arch=self.arch), count)+ else:+ self.add_array(Keys.LLM.EXPERT_USED_COUNT.format(arch=self.arch), count)def add_expert_shared_count(self, count: int) -> None:self.add_uint32(Keys.LLM.EXPERT_SHARED_COUNT.format(arch=self.arch), count)diff --git a/src/llama-graph.cpp b/src/llama-graph.cppindex 6b7eec14fa..274a626433 100644--- a/src/llama-graph.cpp+++ b/src/llama-graph.cpp@@ -1466,7 +1466,7 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) :n_embd_head_v (hparams.n_embd_head_v()),n_embd_v_gqa (hparams.n_embd_v_gqa()),n_expert (hparams.n_expert),- n_expert_used (cparams.warmup ? hparams.n_expert : hparams.n_expert_used),+ n_expert_used (cparams.warmup ? hparams.n_expert : hparams.n_expert_used()),freq_base (cparams.rope_freq_base),freq_scale (cparams.rope_freq_scale),ext_factor (cparams.yarn_ext_factor),@@ -2270,25 +2270,26 @@ ggml_tensor * llm_graph_context::build_moe_ffn(assert(n_expert_used > 0);// order the views before the adds- for (uint32_t i = 0; i < hparams.n_expert_used; ++i) {+ // Use per-layer n_expert_used to bound the graph even during warmup (avoids+ // the large-add-nodes issue for uniform arches; for Puzzle the per-layer+ // value is correct). ref: https://github.com/ggml-org/llama.cpp/pull/14753+ const uint32_t n_expert_used_il = hparams.n_expert_used(il);+ for (uint32_t i = 0; i < n_expert_used_il; ++i) {cur_experts[i] = ggml_view_2d(ctx0, experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1]);ggml_build_forward_expand(gf, cur_experts[i]);}// aggregate experts- // note: here we explicitly use hparams.n_expert_used instead of n_expert_used- // to avoid potentially a large number of add nodes during warmup- // ref: https://github.com/ggml-org/llama.cpp/pull/14753ggml_tensor * moe_out = cur_experts[0];- for (uint32_t i = 1; i < hparams.n_expert_used; ++i) {+ for (uint32_t i = 1; i < n_expert_used_il; ++i) {moe_out = ggml_add(ctx0, moe_out, cur_experts[i]);ggml_build_forward_expand(gf, moe_out);}- if (hparams.n_expert_used == 1) {+ if (n_expert_used_il == 1) {// avoid returning a non-contiguous tensormoe_out = ggml_cont(ctx0, moe_out);}diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cppindex 6a820c61c9..7df82ffe21 100644--- a/src/llama-hparams.cpp+++ b/src/llama-hparams.cpp@@ -71,6 +71,22 @@ uint32_t llama_hparams::n_ff(uint32_t il) const {GGML_ABORT("fatal error");}+uint32_t llama_hparams::n_ff_exp(uint32_t il) const {+ if (il < n_layer_all) {+ return n_ff_exp_arr[il];+ }++ GGML_ABORT("fatal error");+}++uint32_t llama_hparams::n_expert_used(uint32_t il) const {+ if (il < n_layer_all) {+ return n_expert_used_arr[il];+ }++ GGML_ABORT("fatal error");+}+uint32_t llama_hparams::n_gqa(uint32_t il) const {const uint32_t n_head = this->n_head(il);const uint32_t n_head_kv = this->n_head_kv(il);diff --git a/src/llama-hparams.h b/src/llama-hparams.hindex 390d3543d7..2f238a1744 100644--- a/src/llama-hparams.h+++ b/src/llama-hparams.h@@ -62,7 +62,6 @@ struct llama_hparams {// per-token adapter selection. -1 when the model has no such layer.int32_t router_layer = -1;uint32_t n_expert = 0;- uint32_t n_expert_used = 0;uint32_t n_rel_attn_bkts = 0;// TODO: this needs to be reworked@@ -92,10 +91,14 @@ struct llama_hparams {std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_kv_arr;std::array<uint32_t, LLAMA_MAX_LAYERS> n_ff_arr;+ // per-layer expert feed-forward size+ std::array<uint32_t, LLAMA_MAX_LAYERS> n_ff_exp_arr;+ // per-layer top-k expert routing count+ std::array<uint32_t, LLAMA_MAX_LAYERS> n_expert_used_arr;+uint32_t n_layer_dense_lead = 0;uint32_t n_lora_q = 0;uint32_t n_lora_kv = 0;- uint32_t n_ff_exp = 0;uint32_t n_ff_shexp = 0;uint32_t n_ff_chexp = 0;uint32_t n_expert_shared = 0;@@ -385,6 +388,10 @@ struct llama_hparams {uint32_t n_ff(uint32_t il = 0) const;+ uint32_t n_ff_exp(uint32_t il = 0) const;++ uint32_t n_expert_used(uint32_t il = 0) const;+uint32_t n_gqa(uint32_t il = 0) const;uint32_t n_rot(uint32_t il = 0) const;diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cppindex 7663797ba0..d940b1b616 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -951,7 +951,7 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * wcase GGML_OP_MUL_MAT_ID:{// Used for either MoE expert routing or embedded adapter routing- const int n_ids_used = hparams.router_layer >= 0 ? 1 : hparams.n_expert_used;+ const int n_ids_used = hparams.router_layer >= 0 ? 1 : hparams.n_expert_used();GGML_ASSERT(n_ids_used > 0);ggml_tensor * b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_ids_used, 512);ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_ids_used, 512);@@ -964,7 +964,7 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w} break;case GGML_OP_ADD_ID:{- const int n_expert_used = hparams.n_expert_used;+ const int n_expert_used = hparams.n_expert_used();GGML_ASSERT(n_expert_used > 0);ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512);ggml_tensor * c = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512);diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cppindex 8860bd3f43..919e90eccc 100644--- a/src/llama-model-saver.cpp+++ b/src/llama-model-saver.cpp@@ -222,7 +222,7 @@ void llama_model_saver::add_kv_from_model() {add_kv(LLM_KV_BLOCK_COUNT, hparams.n_layer_all);add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);add_kv(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, true);- add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp());add_kv(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent);add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);add_kv(LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH, hparams.n_ff_chexp);@@ -233,7 +233,7 @@ void llama_model_saver::add_kv_from_model() {add_kv(LLM_KV_USE_PARALLEL_RESIDUAL, hparams.use_par_res);// add_kv(LLM_KV_TENSOR_DATA_LAYOUT, ???);add_kv(LLM_KV_EXPERT_COUNT, hparams.n_expert);- add_kv(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used);+ add_kv(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used());add_kv(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);add_kv(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups);add_kv(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used);diff --git a/src/llama-model.cpp b/src/llama-model.cppindex bfce09de0c..408e76e322 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -634,7 +634,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str// the FFN is the same for Qwen 3 Next and Qwen 3.5:if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp(il);GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);return {{n_ff_exp, 2}};}@@ -657,7 +657,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const strreturn {{tensor->ne[axis], 1}};}if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp(il);GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);return {{n_ff_exp, 2}};}@@ -943,6 +943,7 @@ const char * llm_type_name(llm_type type) {case LLM_TYPE_31B_A3_5B: return "31B.A3.5B";case LLM_TYPE_35B_A3B: return "35B.A3B";case LLM_TYPE_48B_A3B: return "48B.A3B";+ case LLM_TYPE_75B_A9B: return "75B.A9B";case LLM_TYPE_80B_A3B: return "80B.A3B";case LLM_TYPE_A3B: return "A3B";case LLM_TYPE_100B_A6B: return "100B.A6B";@@ -1226,14 +1227,15 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);GGML_ASSERT(hparams.n_layer_nextn <= hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);- ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);+ std::fill(hparams.n_expert_used_arr.begin(), hparams.n_expert_used_arr.end(), 0);+ ml.get_key_or_arr(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used_arr, hparams.n_layer_all, false);ml.get_key(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups, false);ml.get_key(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used, false);if (arch == LLM_ARCH_HUNYUAN_VL || arch == LLM_ARCH_HUNYUAN_DENSE) {if (hparams.n_expert <= 1) {- hparams.n_expert = 0;- hparams.n_expert_used = 0;+ hparams.n_expert = 0;+ std::fill(hparams.n_expert_used_arr.begin(), hparams.n_expert_used_arr.end(), 0);}}@@ -1251,10 +1253,16 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {GGML_ASSERT(hparams.convnext.n_layer <= hparams.n_layer_all);}+ // models may route a different number of experts per layer, so validate the maximum+ uint32_t n_expert_used_max = 0;+ for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {+ n_expert_used_max = std::max(n_expert_used_max, hparams.n_expert_used(il));+ }+GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS);- GGML_ASSERT(hparams.n_expert_used <= hparams.n_expert);+ GGML_ASSERT(n_expert_used_max <= hparams.n_expert);if (hparams.n_expert > 0) {- GGML_ASSERT(hparams.n_expert_used > 0);+ GGML_ASSERT(n_expert_used_max > 0);GGML_ASSERT(hparams.n_expert_groups < hparams.n_expert);if (hparams.n_expert_groups > 1) {GGML_ASSERT(hparams.n_expert % hparams.n_expert_groups == 0);@@ -1262,13 +1270,14 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {GGML_ASSERT(hparams.n_group_used < hparams.n_expert_groups);}} else {- GGML_ASSERT(hparams.n_expert_used == 0);+ GGML_ASSERT(n_expert_used_max == 0);GGML_ASSERT(hparams.n_expert_groups == 0);}- std::fill(hparams.n_head_arr.begin(), hparams.n_head_arr.end(), 0);- std::fill(hparams.n_head_kv_arr.begin(), hparams.n_head_kv_arr.end(), 0);- std::fill(hparams.n_ff_arr.begin(), hparams.n_ff_arr.end(), 0);+ std::fill(hparams.n_head_arr.begin(), hparams.n_head_arr.end(), 0);+ std::fill(hparams.n_head_kv_arr.begin(), hparams.n_head_kv_arr.end(), 0);+ std::fill(hparams.n_ff_arr.begin(), hparams.n_ff_arr.end(), 0);+ std::fill(hparams.n_ff_exp_arr.begin(), hparams.n_ff_exp_arr.end(), 0);std::fill(hparams.rope_sections.begin(), hparams.rope_sections.end(), 0);std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), 1);@@ -1501,7 +1510,7 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {const auto tn = LLM_TN(arch);const int64_t n_expert = hparams.n_expert;- const int64_t n_expert_used = hparams.n_expert_used;+ const int64_t n_expert_used = hparams.n_expert_used();if (n_expert > 0 && n_expert_used == 0) {throw std::runtime_error("model has expert layers but no expert layers are used");@@ -1957,7 +1966,7 @@ void llama_model::print_info() const {LLAMA_LOG_INFO("%s: f_attn_value_scale = %.4f\n", __func__, hparams.f_attn_value_scale);LLAMA_LOG_INFO("%s: n_ff = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_ff(il); }, hparams.n_layer_all).c_str());LLAMA_LOG_INFO("%s: n_expert = %u\n", __func__, hparams.n_expert);- LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used);+ LLAMA_LOG_INFO("%s: n_expert_used = %u\n", __func__, hparams.n_expert_used());LLAMA_LOG_INFO("%s: n_expert_groups = %d\n", __func__, hparams.n_expert_groups);LLAMA_LOG_INFO("%s: n_group_used = %d\n", __func__, hparams.n_group_used);LLAMA_LOG_INFO("%s: causal attn = %d\n", __func__, hparams.causal_attn);@@ -2032,7 +2041,7 @@ void llama_model::print_info() const {if (arch == LLM_ARCH_DEEPSEEK) {LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp());LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);}@@ -2045,7 +2054,7 @@ void llama_model::print_info() const {LLAMA_LOG_INFO("%s: n_lora_kv = %d\n", __func__, hparams.n_lora_kv);LLAMA_LOG_INFO("%s: n_embd_head_k_mla = %d\n", __func__, hparams.n_embd_head_k_mla());LLAMA_LOG_INFO("%s: n_embd_head_v_mla = %d\n", __func__, hparams.n_embd_head_v_mla());- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp());LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm);@@ -2053,7 +2062,7 @@ void llama_model::print_info() const {}if (arch == LLM_ARCH_QWEN2MOE) {- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp());LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);}@@ -2063,7 +2072,7 @@ void llama_model::print_info() const {arch == LLM_ARCH_OPENAI_MOE ||arch == LLM_ARCH_QWEN3VLMOE ||arch == LLM_ARCH_RND1) {- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp());}if (arch == LLM_ARCH_MINICPM ||@@ -2080,7 +2089,7 @@ void llama_model::print_info() const {if (arch == LLM_ARCH_BAILINGMOE) {LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp());LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);LLAMA_LOG_INFO("%s: expert_weights_norm = %d\n", __func__, hparams.expert_weights_norm);@@ -2088,7 +2097,7 @@ void llama_model::print_info() const {if (arch == LLM_ARCH_BAILINGMOE2 || arch == LLM_ARCH_BAILINGMOE3) {LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp());LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);LLAMA_LOG_INFO("%s: n_expert_shared = %d\n", __func__, hparams.n_expert_shared);LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);@@ -2098,12 +2107,12 @@ void llama_model::print_info() const {}if (arch == LLM_ARCH_SMALLTHINKER || arch == LLM_ARCH_LFM2MOE) {- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp());LLAMA_LOG_INFO("%s: expert_gating_func = %s\n", __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func));}if (arch == LLM_ARCH_GROVEMOE) {- LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);+ LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp());LLAMA_LOG_INFO("%s: n_ff_chexp = %d\n", __func__, hparams.n_ff_chexp);LLAMA_LOG_INFO("%s: n_group_experts = %d\n", __func__, hparams.n_group_experts);LLAMA_LOG_INFO("%s: expert_group_scale = %.2f\n", __func__, hparams.expert_group_scale);diff --git a/src/llama-model.h b/src/llama-model.hindex ee6bb5ac37..4c4a30e018 100644--- a/src/llama-model.h+++ b/src/llama-model.h@@ -128,6 +128,7 @@ enum llm_type {LLM_TYPE_31B_A3_5B,LLM_TYPE_35B_A3B, // Qwen3.5LLM_TYPE_48B_A3B, // Kimi Linear+ LLM_TYPE_75B_A9B, // Nemotron 3 PuzzleLLM_TYPE_80B_A3B, // Qwen3 NextLLM_TYPE_A3B, // Qwen3.8 Flash NextLLM_TYPE_100B_A6B,@@ -839,7 +840,7 @@ const char * llm_type_name(llm_type type);const int64_t n_token_types = vocab.n_token_types(); GGML_UNUSED(n_token_types); \const int64_t n_rot = hparams.n_rot(); GGML_UNUSED(n_rot); \const int64_t n_expert = hparams.n_expert; GGML_UNUSED(n_expert); \- const int64_t n_expert_used = hparams.n_expert_used; GGML_UNUSED(n_expert_used); \+ const int64_t n_expert_used = hparams.n_expert_used(); GGML_UNUSED(n_expert_used); \const int64_t n_ctx_train = hparams.n_ctx_train; GGML_UNUSED(n_ctx_train);// For internal test usediff --git a/src/models/afmoe.cpp b/src/models/afmoe.cppindex 063b214256..cf02203671 100644--- a/src/models/afmoe.cpp+++ b/src/models/afmoe.cpp@@ -3,7 +3,7 @@void llama_model_afmoe::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);@@ -52,7 +52,7 @@ void llama_model_afmoe::load_arch_tensors(llama_model_loader &) {output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);}- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();for (int i = 0; i < n_layer; ++i) {auto & layer = layers[i];diff --git a/src/models/bailingmoe.cpp b/src/models/bailingmoe.cppindex 7faf73c835..9d1073ae14 100644--- a/src/models/bailingmoe.cpp+++ b/src/models/bailingmoe.cpp@@ -3,7 +3,7 @@void llama_model_bailingmoe::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);@@ -19,7 +19,7 @@ void llama_model_bailingmoe::load_arch_tensors(llama_model_loader &) {LLAMA_LOAD_LOCALS;const int64_t n_expert_shared = hparams.n_expert_shared;- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/bailingmoe2.cpp b/src/models/bailingmoe2.cppindex 8fc0ea752b..24fc4e0226 100644--- a/src/models/bailingmoe2.cpp+++ b/src/models/bailingmoe2.cpp@@ -3,7 +3,7 @@void llama_model_bailingmoe2::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);@@ -21,7 +21,7 @@ void llama_model_bailingmoe2::load_arch_tensors(llama_model_loader &) {LLAMA_LOAD_LOCALS;const int64_t n_expert_shared = hparams.n_expert_shared;- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cppindex 5ebedaecbb..1f2592cfa1 100644--- a/src/models/bailingmoe3.cpp+++ b/src/models/bailingmoe3.cpp@@ -15,7 +15,7 @@ void llama_model_bailingmoe3::load_arch_hparams(llama_model_loader & ml) {hparams.kda_safe_gate = true;}ml.get_key(LLM_KV_KDA_GATE_LOWER_BOUND, hparams.kda_gate_lower_bound);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);@@ -26,7 +26,7 @@ void llama_model_bailingmoe3::load_arch_hparams(llama_model_loader & ml) {ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, false);if (hparams.n_ff_shexp == 0) {- hparams.n_ff_shexp = hparams.n_ff_exp * std::max(1u, hparams.n_expert_shared);+ hparams.n_ff_shexp = hparams.n_ff_exp() * std::max(1u, hparams.n_expert_shared);}GGML_ASSERT(hparams.kda_safe_gate);@@ -115,9 +115,9 @@ void llama_model_bailingmoe3::load_arch_tensors(llama_model_loader & ml) {} else {layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, trunk_flags);layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", il), { n_expert }, trunk_flags);- layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", il), { n_embd, hparams.n_ff_exp, n_expert }, trunk_flags);- layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", il), { n_embd, hparams.n_ff_exp, n_expert }, trunk_flags);- layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { hparams.n_ff_exp, n_embd, n_expert }, trunk_flags);+ layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", il), { n_embd, hparams.n_ff_exp(), n_expert }, trunk_flags);+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", il), { n_embd, hparams.n_ff_exp(), n_expert }, trunk_flags);+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { hparams.n_ff_exp(), n_embd, n_expert }, trunk_flags);layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, hparams.n_ff_shexp }, trunk_flags);layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, hparams.n_ff_shexp }, trunk_flags);layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { hparams.n_ff_shexp, n_embd }, trunk_flags);@@ -145,9 +145,9 @@ void llama_model_bailingmoe3::load_arch_tensors(llama_model_loader & ml) {layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", il), { n_embd }, flags);layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, flags);layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", il), { n_expert }, flags);- layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", il), { n_embd, hparams.n_ff_exp, n_expert }, flags);- layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", il), { n_embd, hparams.n_ff_exp, n_expert }, flags);- layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { hparams.n_ff_exp, n_embd, n_expert }, flags);+ layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", il), { n_embd, hparams.n_ff_exp(), n_expert }, flags);+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", il), { n_embd, hparams.n_ff_exp(), n_expert }, flags);+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { hparams.n_ff_exp(), n_embd, n_expert }, flags);layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, hparams.n_ff_shexp }, flags);layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, hparams.n_ff_shexp }, flags);layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { hparams.n_ff_shexp, n_embd }, flags);diff --git a/src/models/bert.cpp b/src/models/bert.cppindex 53ce29f23c..ca0281d306 100644--- a/src/models/bert.cpp+++ b/src/models/bert.cpp@@ -182,7 +182,7 @@ llama_model_bert::graph::graph(const llama_model & model, const llm_graph_paramsnullptr,model.layers[il].ffn_down_exps,nullptr,- hparams.n_expert, hparams.n_expert_used,+ hparams.n_expert, hparams.n_expert_used(),LLM_FFN_GELU, false,hparams.expert_weights_scale,LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cppindex c50910edc5..5e02cd56e7 100644--- a/src/models/cohere2moe.cpp+++ b/src/models/cohere2moe.cpp@@ -13,7 +13,7 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);@@ -89,7 +89,7 @@ void llama_model_cohere2moe::load_arch_tensors(llama_model_loader & ml) {layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }, flags);layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, n_ff }, flags);} else {- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff;+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff;layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, flags);layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff_exp, n_embd, n_expert }, flags);@@ -113,7 +113,7 @@ void llama_model_cohere2moe::load_arch_tensors(llama_model_loader & ml) {create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, flags);layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, flags);- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff;+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff;// Routed expertslayer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, flags);diff --git a/src/models/deepseek.cpp b/src/models/deepseek.cppindex f52ec9518b..a47a9c3daf 100644--- a/src/models/deepseek.cpp+++ b/src/models/deepseek.cpp@@ -3,11 +3,11 @@void llama_model_deepseek::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);- switch (hparams.n_ff_exp) {+ switch (hparams.n_ff_exp()) {case 1408: type = LLM_TYPE_16B; break;case 1792: type = LLM_TYPE_20B; break;default: type = LLM_TYPE_UNKNOWN;@@ -19,7 +19,7 @@ void llama_model_deepseek::load_arch_tensors(llama_model_loader &) {const int64_t n_expert_shared = hparams.n_expert_shared;- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cppindex 3a76187aa5..4628ff4daf 100644--- a/src/models/deepseek2.cpp+++ b/src/models/deepseek2.cpp@@ -15,7 +15,7 @@ void llama_model_deepseek2::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl, false);ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl, false);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);@@ -79,7 +79,7 @@ void llama_model_deepseek2::load_arch_tensors(llama_model_loader & ml) {const int64_t q_lora_rank = hparams.n_lora_q;const int64_t kv_lora_rank = hparams.n_lora_kv;- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/deepseek2ocr.cpp b/src/models/deepseek2ocr.cppindex 65d31c31b9..1c5c452e96 100644--- a/src/models/deepseek2ocr.cpp+++ b/src/models/deepseek2ocr.cpp@@ -4,7 +4,7 @@ void llama_model_deepseek2ocr::load_arch_hparams(llama_model_loader & ml) {// similar to deepseek2, but without MLAml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);@@ -25,7 +25,7 @@ void llama_model_deepseek2ocr::load_arch_tensors(llama_model_loader &) {const int64_t n_expert_shared = hparams.n_expert_shared;// similar to deepseek2, but without MLA- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cppindex 079bdfc302..60cc17c497 100644--- a/src/models/deepseek32.cpp+++ b/src/models/deepseek32.cpp@@ -4,7 +4,7 @@#include "llama-kv-cache-dsa.h"void llama_model_deepseek32::load_arch_hparams(llama_model_loader & ml) {- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);hparams.f_norm_eps = 1e-6; // eps for layer normml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false);@@ -20,7 +20,7 @@ void llama_model_deepseek32::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl, false);ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl, false);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);// DSA parameters@@ -71,7 +71,7 @@ void llama_model_deepseek32::load_arch_tensors(llama_model_loader & ml) {const int64_t q_lora_rank = hparams.n_lora_q;const int64_t kv_lora_rank = hparams.n_lora_kv;- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();const int64_t n_expert_shared = hparams.n_expert_shared;tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cppindex 680516b230..5bdf14b486 100644--- a/src/models/deepseek4.cpp+++ b/src/models/deepseek4.cpp@@ -29,7 +29,7 @@ void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm);@@ -83,7 +83,7 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader & ml) {LLAMA_LOAD_LOCALS;const int64_t q_lora_rank = hparams.n_lora_q;- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();const int64_t n_expert_shared = hparams.n_expert_shared;const int64_t n_embd_head = hparams.n_embd_head_k();@@ -1298,7 +1298,7 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_player.ffn_gate_exps,layer.ffn_down_exps,exp_probs_b,- n_expert, hparams.n_expert_used,+ n_expert, hparams.n_expert_used(),LLM_FFN_SILU, hparams.expert_weights_norm,hparams.expert_weights_scale,(llama_expert_gating_func_type) hparams.expert_gating_func,@@ -1455,7 +1455,7 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llmlayer.ffn_gate_exps,layer.ffn_down_exps,layer.ffn_exp_probs_b,- n_expert, hparams.n_expert_used,+ n_expert, hparams.n_expert_used(),LLM_FFN_SILU, hparams.expert_weights_norm,hparams.expert_weights_scale,(llama_expert_gating_func_type) hparams.expert_gating_func,diff --git a/src/models/dflash.cpp b/src/models/dflash.cppindex 036bcc14aa..da84f30b63 100644--- a/src/models/dflash.cpp+++ b/src/models/dflash.cpp@@ -40,7 +40,7 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) {if (hparams.dsv4_hc_mult > 0) {ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm);@@ -159,7 +159,7 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {if (hparams.dsv4_hc_mult > 0) {const int64_t q_lora_rank = hparams.n_lora_q;- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();const int64_t n_expert_shared = hparams.n_expert_shared;const int64_t n_embd_head = hparams.n_embd_head_k();const int64_t o_groups = hparams.dsv4_o_group_count;@@ -948,7 +948,7 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_layer.ffn_gate_exps,layer.ffn_down_exps,layer.ffn_exp_probs_b,- n_expert, hparams.n_expert_used,+ n_expert, hparams.n_expert_used(),LLM_FFN_SILU, hparams.expert_weights_norm,hparams.expert_weights_scale,(llama_expert_gating_func_type) hparams.expert_gating_func,diff --git a/src/models/dots1.cpp b/src/models/dots1.cppindex 07d6ab1b7c..a3a85748ef 100644--- a/src/models/dots1.cpp+++ b/src/models/dots1.cpp@@ -3,7 +3,7 @@void llama_model_dots1::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);@@ -19,7 +19,7 @@ void llama_model_dots1::load_arch_tensors(llama_model_loader &) {LLAMA_LOAD_LOCALS;const int64_t n_expert_shared = hparams.n_expert_shared;- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cppindex 7656562b05..0991c488e4 100644--- a/src/models/dots3note.cpp+++ b/src/models/dots3note.cpp@@ -11,7 +11,7 @@ void llama_model_dots3note::load_arch_hparams(llama_model_loader & ml) {// MoE parametersml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);@@ -56,7 +56,7 @@ void llama_model_dots3note::load_arch_tensors(llama_model_loader & ml) {const int64_t n_embd_head_qk_rope = hparams.n_rot();const int64_t q_lora_rank = hparams.n_lora_q;- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();const int64_t n_expert_shared = hparams.n_expert_shared;tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/ernie4-5.cpp b/src/models/ernie4-5.cppindex 895cf690bd..7bf7be648a 100644--- a/src/models/ernie4-5.cpp+++ b/src/models/ernie4-5.cpp@@ -6,7 +6,7 @@ void llama_model_ernie4_5::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);if (arch == LLM_ARCH_ERNIE4_5_MOE) {- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);ml.get_key(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, hparams.n_moe_layer_step);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);@@ -47,7 +47,7 @@ void llama_model_ernie4_5::load_arch_tensors(llama_model_loader &) {layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);if (arch == LLM_ARCH_ERNIE4_5_MOE && static_cast<uint32_t>(i) >= hparams.n_layer_dense_lead) { // MoE layers- int n_ff_exp = hparams.n_ff_exp;+ int n_ff_exp = hparams.n_ff_exp();layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cppindex 86e5a3a984..976ee050ad 100644--- a/src/models/exaone-moe.cpp+++ b/src/models/exaone-moe.cpp@@ -13,7 +13,7 @@ void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);@@ -30,7 +30,7 @@ void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) {void llama_model_exaone_moe::load_arch_tensors(llama_model_loader &) {LLAMA_LOAD_LOCALS;- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();const int64_t n_ff_shexp = hparams.n_ff_shexp > 0 ? hparams.n_ff_shexp : n_ff_exp;const int64_t head_dim = hparams.n_embd_head_k();const int64_t n_qo_dim = n_head * head_dim;diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cppindex aa518c6df5..c6dd7d1bf9 100644--- a/src/models/gemma4.cpp+++ b/src/models/gemma4.cpp@@ -11,7 +11,7 @@ void llama_model_gemma4::load_arch_hparams(llama_model_loader & ml) {hparams.f_attention_scale = 1.0f; // Gemma4 uses self.scaling = 1.0 (no pre-attn scaling)ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.n_embd_per_layer);@@ -32,7 +32,7 @@ void llama_model_gemma4::load_arch_tensors(llama_model_loader &) {LLAMA_LOAD_LOCALS;const uint32_t n_embd_per_layer = hparams.n_embd_per_layer;- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();if (n_embd_head_k != n_embd_head_v) {throw std::runtime_error("Gemma 4 requires n_embd_head_k == n_embd_head_v");diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cppindex 543b15cf35..44d8832748 100644--- a/src/models/glm-dsa.cpp+++ b/src/models/glm-dsa.cpp@@ -27,7 +27,7 @@ const std::array<uint32_t, LLAMA_MAX_LAYERS> GLM_5_2_DEFAULT_INDEXER_TYPES = {};void llama_model_glm_dsa::load_arch_hparams(llama_model_loader & ml) {- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false);@@ -42,7 +42,7 @@ void llama_model_glm_dsa::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl, false);ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl, false);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);// DSA parameters@@ -104,7 +104,7 @@ void llama_model_glm_dsa::load_arch_tensors(llama_model_loader & ml) {const int64_t q_lora_rank = hparams.n_lora_q;const int64_t kv_lora_rank = hparams.n_lora_kv;- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/glm4-moe.cpp b/src/models/glm4-moe.cppindex 1d2ac65fd7..d6ae5783cb 100644--- a/src/models/glm4-moe.cpp+++ b/src/models/glm4-moe.cpp@@ -1,7 +1,7 @@#include "models.h"void llama_model_glm4_moe::load_arch_hparams(llama_model_loader & ml) {- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false);@@ -40,7 +40,7 @@ void llama_model_glm4_moe::load_arch_tensors(llama_model_loader & ml) {}GGML_ASSERT(hparams.n_expert > 0 && "n_expert must be > 0 for GLM4_MOE MoE layers");- GGML_ASSERT(hparams.n_expert_used > 0 && "n_expert_used must be > 0 for GLM4_MOE MoE layers");+ GGML_ASSERT(hparams.n_expert_used() > 0 && "n_expert_used must be > 0 for GLM4_MOE MoE layers");tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);@@ -82,7 +82,7 @@ void llama_model_glm4_moe::load_arch_tensors(llama_model_loader & ml) {layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), { n_expert }, flags);// MoE branch- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert }, flags);diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cppindex 3aa2b63b23..08d9e8a542 100644--- a/src/models/granite-swa.cpp+++ b/src/models/granite-swa.cpp@@ -11,7 +11,7 @@ void llama_model_granite_swa::load_arch_hparams(llama_model_loader & ml) {// MoE expert configurationml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);- ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);+ ml.get_key_or_arr(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used_arr, hparams.n_layer_all, false);// iSWA configurationml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);diff --git a/src/models/grok.cpp b/src/models/grok.cppindex 42f38af672..cb6afc3a70 100644--- a/src/models/grok.cpp+++ b/src/models/grok.cpp@@ -12,7 +12,7 @@ void llama_model_grok::load_arch_hparams(llama_model_loader & ml) {hparams.f_final_logit_softcapping = 0.0f;ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale, false);ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false);ml.get_key(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale, false);@@ -50,7 +50,7 @@ void llama_model_grok::load_arch_tensors(llama_model_loader &) {output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);}- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff/* / n_expert_used*/; // grok-1 n_ff_exp == n_ff+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff/* / n_expert_used*/; // grok-1 n_ff_exp == n_fffor (int i = 0; i < n_layer; ++i) {auto & layer = layers[i];diff --git a/src/models/grovemoe.cpp b/src/models/grovemoe.cppindex 643a448e59..f32f3e9ed9 100644--- a/src/models/grovemoe.cpp+++ b/src/models/grovemoe.cpp@@ -1,7 +1,7 @@#include "models.h"void llama_model_grovemoe::load_arch_hparams(llama_model_loader & ml) {- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH, hparams.n_ff_chexp, false);ml.get_key(LLM_KV_EXPERT_GROUP_SCALE, hparams.expert_group_scale);ml.get_key(LLM_KV_EXPERTS_PER_GROUP, hparams.n_group_experts);@@ -46,7 +46,7 @@ void llama_model_grovemoe::load_arch_tensors(llama_model_loader &) {layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);// MoE branch- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;const int64_t n_ff_chexp = hparams.n_ff_chexp ? hparams.n_ff_chexp : n_embd_head_k;const int64_t n_chunk_expert = n_expert / hparams.n_group_experts;diff --git a/src/models/hunyuan-moe.cpp b/src/models/hunyuan-moe.cppindex 4d55f5e7f3..cedc3b53eb 100644--- a/src/models/hunyuan-moe.cpp+++ b/src/models/hunyuan-moe.cpp@@ -2,7 +2,7 @@void llama_model_hunyuan_moe::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);switch (hparams.n_layer()) {diff --git a/src/models/hy-v3.cpp b/src/models/hy-v3.cppindex 3c45331b11..f6b72d843c 100644--- a/src/models/hy-v3.cpp+++ b/src/models/hy-v3.cpp@@ -2,7 +2,7 @@void llama_model_hy_v3::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);@@ -45,7 +45,7 @@ void llama_model_hy_v3::load_arch_tensors(llama_model_loader & ml) {auto load_block = [&](int i, int flags) {auto & layer = layers[i];- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / (n_expert_used > 0 ? n_expert_used : 1);+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / (n_expert_used > 0 ? n_expert_used : 1);const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff_exp;layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags);diff --git a/src/models/kimi-k3.cpp b/src/models/kimi-k3.cppindex 7b46bccdb0..b061093ebb 100644--- a/src/models/kimi-k3.cpp+++ b/src/models/kimi-k3.cpp@@ -30,7 +30,7 @@ void llama_model_kimi_k3::load_arch_hparams(llama_model_loader & ml) {hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0;}- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);@@ -139,7 +139,7 @@ void llama_model_kimi_k3::load_arch_tensors(llama_model_loader &) {layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);} else {- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);@@ -584,7 +584,7 @@ ggml_tensor * llama_model_kimi_k3::graph::build_latent_moe(layer.ffn_down_exps,layer.ffn_exp_probs_b,hparams.n_expert,- hparams.n_expert_used,+ hparams.n_expert_used(),LLM_FFN_SITU, hparams.expert_weights_norm,hparams.expert_weights_scale,(llama_expert_gating_func_type) hparams.expert_gating_func,diff --git a/src/models/kimi-linear.cpp b/src/models/kimi-linear.cppindex bda3cd9b04..601d1d9beb 100644--- a/src/models/kimi-linear.cpp+++ b/src/models/kimi-linear.cpp@@ -19,7 +19,7 @@ void llama_model_kimi_linear::load_arch_hparams(llama_model_loader & ml) {}// MoE parameters - Kimi uses moe_intermediate_size = 1024- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);@@ -137,7 +137,7 @@ void llama_model_kimi_linear::load_arch_tensors(llama_model_loader &) {layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);// MoE intermediate size (different from dense FFN)- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();// Kimi uses n_layer_dense_lead to determine which layers use dense FFN vs MoE// first_k_dense_replace = 1 means layer 0 uses dense FFN, layers 1+ use MoE@@ -504,7 +504,7 @@ llama_model_kimi_linear::graph::graph(const llama_model & model, const llm_graphlayer.ffn_down_exps,layer.ffn_exp_probs_b,hparams.n_expert,- hparams.n_expert_used,+ hparams.n_expert_used(),LLM_FFN_SILU, true,hparams.expert_weights_scale,(llama_expert_gating_func_type) hparams.expert_gating_func,diff --git a/src/models/laguna.cpp b/src/models/laguna.cppindex 82c9a9538c..556400bfce 100644--- a/src/models/laguna.cpp+++ b/src/models/laguna.cpp@@ -9,7 +9,7 @@void llama_model_laguna::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);@@ -24,7 +24,7 @@ void llama_model_laguna::load_arch_hparams(llama_model_loader & ml) {// Weightless fixtures (test-llama-archs) omit this key; derive a nonzero// size so the shared expert is still built. Real GGUFs always carry the// exact value (routed and shared FF lengths may differ).- hparams.n_ff_shexp = hparams.n_ff_exp * hparams.n_expert_shared;+ hparams.n_ff_shexp = hparams.n_ff_exp() * hparams.n_expert_shared;}// Sliding-window attention is OPTIONAL. XS.2 is hybrid (full / SWA / SWA /@@ -76,7 +76,7 @@ void llama_model_laguna::load_arch_tensors(llama_model_loader & ml) {output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);}- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();const int64_t n_ff_shexp = hparams.n_ff_shexp;for (int i = 0; i < n_layer; ++i) {diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cppindex 9a42955570..07b71ccd3a 100644--- a/src/models/lfm2.cpp+++ b/src/models/lfm2.cpp@@ -53,9 +53,9 @@ void llama_model_lfm2::load_arch_tensors(llama_model_loader &) {if (is_moe_layer) {GGML_ASSERT(n_expert && n_expert_used);layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);- layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, hparams.n_ff_exp, n_expert}, 0);- layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {hparams.n_ff_exp, n_embd, n_expert}, 0);- layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, hparams.n_ff_exp, n_expert}, 0);+ layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, hparams.n_ff_exp(), n_expert}, 0);+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {hparams.n_ff_exp(), n_embd, n_expert}, 0);+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, hparams.n_ff_exp(), n_expert}, 0);layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);} else { // denselayer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);diff --git a/src/models/lfm2moe.cpp b/src/models/lfm2moe.cppindex 490f5c223e..f8d47f9b85 100644--- a/src/models/lfm2moe.cpp+++ b/src/models/lfm2moe.cpp@@ -6,7 +6,7 @@ void llama_model_lfm2moe::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);for (uint32_t il = 0; il < hparams.n_layer(); ++il) {@@ -42,9 +42,9 @@ void llama_model_lfm2moe::load_arch_tensors(llama_model_loader &) {if (is_moe_layer) {GGML_ASSERT(n_expert && n_expert_used);layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);- layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, hparams.n_ff_exp, n_expert}, 0);- layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {hparams.n_ff_exp, n_embd, n_expert}, 0);- layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, hparams.n_ff_exp, n_expert}, 0);+ layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, hparams.n_ff_exp(), n_expert}, 0);+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {hparams.n_ff_exp(), n_embd, n_expert}, 0);+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, hparams.n_ff_exp(), n_expert}, 0);layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);} else { // denselayer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);diff --git a/src/models/llada-moe.cpp b/src/models/llada-moe.cppindex 2ae8938644..0ee9ce1bec 100644--- a/src/models/llada-moe.cpp+++ b/src/models/llada-moe.cpp@@ -1,7 +1,7 @@#include "models.h"void llama_model_llada_moe::load_arch_hparams(llama_model_loader & ml) {- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);// diffusion language model uses non-causal attention@@ -39,7 +39,7 @@ void llama_model_llada_moe::load_arch_tensors(llama_model_loader &) {layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);diff --git a/src/models/llama4.cpp b/src/models/llama4.cppindex 7194c72a58..8a812beffa 100644--- a/src/models/llama4.cpp+++ b/src/models/llama4.cpp@@ -2,7 +2,7 @@void llama_model_llama4::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, hparams.n_moe_layer_step);const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);@@ -75,7 +75,7 @@ void llama_model_llama4::load_arch_tensors(llama_model_loader &) {layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));if (is_moe_layer) {- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);diff --git a/src/models/mellum.cpp b/src/models/mellum.cppindex 28823018bc..872a9c8f55 100644--- a/src/models/mellum.cpp+++ b/src/models/mellum.cpp@@ -2,7 +2,7 @@void llama_model_mellum::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);if (hparams.n_swa > 0) {@@ -61,7 +61,7 @@ void llama_model_mellum::load_arch_tensors(llama_model_loader &) {throw std::runtime_error("n_expert_used must be > 0 for Mellum");}- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);diff --git a/src/models/mimo2.cpp b/src/models/mimo2.cppindex 1dc554220d..8772319f46 100644--- a/src/models/mimo2.cpp+++ b/src/models/mimo2.cpp@@ -5,7 +5,7 @@ void llama_model_mimo2::load_arch_hparams(llama_model_loader & ml) {hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);@@ -62,7 +62,7 @@ void llama_model_mimo2::load_arch_tensors(llama_model_loader & ml) {layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED | flags);// MoE branch- int64_t n_ff_exp = hparams.n_ff_exp;+ int64_t n_ff_exp = hparams.n_ff_exp();layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED | flags);layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED | flags);layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, TENSOR_NOT_REQUIRED | flags);diff --git a/src/models/minimax-m2.cpp b/src/models/minimax-m2.cppindex 86a8ae2b1d..c2e69bfaa8 100644--- a/src/models/minimax-m2.cpp+++ b/src/models/minimax-m2.cpp@@ -2,7 +2,7 @@void llama_model_minimax_m2::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);switch (hparams.n_layer()) {diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cppindex 1ba699d016..80260a6295 100644--- a/src/models/minimax-m3.cpp+++ b/src/models/minimax-m3.cpp@@ -13,7 +13,7 @@void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);@@ -36,7 +36,7 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) {void llama_model_minimax_m3::load_arch_tensors(llama_model_loader &) {LLAMA_LOAD_LOCALS;const int64_t n_expert_shared = hparams.n_expert_shared;- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cppindex 55640c996a..d2c48f125e 100644--- a/src/models/nemotron-h.cpp+++ b/src/models/nemotron-h.cpp@@ -1,5 +1,7 @@#include "models.h"+#include <algorithm> // std::max+void llama_model_nemotron_h::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);@@ -16,7 +18,8 @@ void llama_model_nemotron_h::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); // MTP head final_layernorm- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);+ // Puzzle models set a different expert FFN size per layer+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);@@ -26,7 +29,17 @@ void llama_model_nemotron_h::load_arch_hparams(llama_model_loader & ml) {switch (hparams.n_layer()) {case 52: type = LLM_TYPE_31B_A3_5B; break; // Nemotron-H_MOE 31Bcase 56: type = LLM_TYPE_9B; break;- case 88: type = LLM_TYPE_120B_A12B; break;+ case 88:+ {+ // Nemotron 3 Super (uniform MoE) and Nemotron 3 Puzzle (per-layer+ // heterogeneous MoE) both have 88 layers; the per-layer top-k array+ // is the discriminator.+ bool heterogeneous = false;+ for (uint32_t i = 1; i < hparams.n_layer(); ++i) {+ heterogeneous |= hparams.n_expert_used_arr[i] != hparams.n_expert_used_arr[0];+ }+ type = heterogeneous ? LLM_TYPE_75B_A9B : LLM_TYPE_120B_A12B;+ } break;default: type = LLM_TYPE_UNKNOWN;}}@@ -94,7 +107,10 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) {layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);} else {if (n_expert != 0) {- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ // Use per-layer n_ff_exp; fall back to n_ff/n_expert_used if absent (existing GGUFs).+ const int64_t n_ff_exp_i = hparams.n_ff_exp(i)+ ? (int64_t)hparams.n_ff_exp(i)+ : hparams.n_ff(i) / (int64_t)hparams.n_expert_used(i);const int64_t n_ff_shexp = hparams.n_ff_shexp;layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert}, trunk_flags);@@ -104,8 +120,8 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) {layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", i), {n_embd, moe_n_embd}, TENSOR_NOT_REQUIRED);layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", i), {moe_n_embd, n_embd}, TENSOR_NOT_REQUIRED);- layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, moe_n_embd, n_expert}, trunk_flags);- layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp, n_expert}, trunk_flags);+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp_i, moe_n_embd, n_expert}, trunk_flags);+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp_i, n_expert}, trunk_flags);// Shared expert branchlayer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, trunk_flags);@@ -129,7 +145,7 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) {const int64_t n_head_i = hparams.n_head(i);const int64_t n_embd_k_gqa_i = hparams.n_embd_k_gqa(i);const int64_t n_embd_v_gqa_i = hparams.n_embd_v_gqa(i);- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ const int64_t n_ff_exp = hparams.n_ff_exp(i) ? (int64_t)hparams.n_ff_exp(i) : n_ff / (int64_t)hparams.n_expert_used(i);const int64_t n_ff_shexp = hparams.n_ff_shexp;// NextN input-fusion tensors@@ -280,7 +296,7 @@ ggml_tensor * llama_model_nemotron_h::graph::build_ffn_layer(ggml_tensor * cur,nullptr, // no gatemodel.layers[il].ffn_down_exps,model.layers[il].ffn_exp_probs_b,- n_expert, n_expert_used,+ n_expert, (int64_t)hparams.n_expert_used(il),LLM_FFN_RELU_SQR, hparams.expert_weights_norm,hparams.expert_weights_scale,LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,diff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cppindex c91bae1c35..c9f9b677d0 100644--- a/src/models/openai-moe.cpp+++ b/src/models/openai-moe.cpp@@ -2,7 +2,7 @@void llama_model_openai_moe::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;@@ -24,7 +24,7 @@ void llama_model_openai_moe::load_arch_hparams(llama_model_loader & ml) {void llama_model_openai_moe::load_arch_tensors(llama_model_loader &) {LLAMA_LOAD_LOCALS;- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/qwen2moe.cpp b/src/models/qwen2moe.cppindex e831ed11aa..8bcb1017bf 100644--- a/src/models/qwen2moe.cpp+++ b/src/models/qwen2moe.cpp@@ -1,7 +1,7 @@#include "models.h"void llama_model_qwen2moe::load_arch_hparams(llama_model_loader & ml) {- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);@@ -42,7 +42,7 @@ void llama_model_qwen2moe::load_arch_tensors(llama_model_loader &) {}// MoE branch- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cppindex 9bf4ea4328..ed4083f12b 100644--- a/src/models/qwen35moe.cpp+++ b/src/models/qwen35moe.cpp@@ -2,7 +2,7 @@#include "llama-memory-recurrent.h"void llama_model_qwen35moe::load_arch_hparams(llama_model_loader & ml) {- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);@@ -54,7 +54,7 @@ void llama_model_qwen35moe::load_arch_tensors(llama_model_loader & ml) {auto load_block_trunk = [&](int il, int flags) {auto & layer = layers[il];- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff;// Calculate dimensions from hyperparameters@@ -106,7 +106,7 @@ void llama_model_qwen35moe::load_arch_tensors(llama_model_loader & ml) {auto load_block_mtp = [&](int il) {auto & layer = layers[il];- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff;// MTP block looks like a full-attention Qwen3.5 decoder block with MoE FFN.diff --git a/src/models/qwen3moe.cpp b/src/models/qwen3moe.cppindex 6f6df5390e..a6a3381e5e 100644--- a/src/models/qwen3moe.cpp+++ b/src/models/qwen3moe.cpp@@ -1,7 +1,7 @@#include "models.h"void llama_model_qwen3moe::load_arch_hparams(llama_model_loader & ml) {- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);switch (hparams.n_layer()) {@@ -47,7 +47,7 @@ void llama_model_qwen3moe::load_arch_tensors(llama_model_loader &) {}// MoE branch- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cppindex b2b8809c7f..eb823b8ead 100644--- a/src/models/qwen3next.cpp+++ b/src/models/qwen3next.cpp@@ -2,7 +2,7 @@#include "llama-memory-recurrent.h"void llama_model_qwen3next::load_arch_hparams(llama_model_loader & ml) {- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);@@ -50,7 +50,7 @@ void llama_model_qwen3next::load_arch_tensors(llama_model_loader & ml) {output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED);}- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;// Calculate dimensions from hyperparametersconst int64_t head_k_dim = hparams.ssm_d_state;diff --git a/src/models/qwen3vlmoe.cpp b/src/models/qwen3vlmoe.cppindex 7c41592f77..e7a81e32cf 100644--- a/src/models/qwen3vlmoe.cpp+++ b/src/models/qwen3vlmoe.cpp@@ -3,7 +3,7 @@void llama_model_qwen3vlmoe::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.n_deepstack_layers, false);ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true);- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);switch (hparams.n_layer()) {@@ -49,7 +49,7 @@ void llama_model_qwen3vlmoe::load_arch_tensors(llama_model_loader &) {}// MoE branch- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cppindex 8f0e47b1fe..773204a5a1 100644--- a/src/models/qwen4exp.cpp+++ b/src/models/qwen4exp.cpp@@ -24,7 +24,7 @@ static void qwen4exp_require_arr_len(llama_model_loader & ml, llm_kv kid, uint32}void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) {- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);@@ -191,7 +191,7 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {for (int il = 0; il < n_layer; ++il) {auto & layer = layers[il];- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff;const int64_t head_k_dim = hparams.ssm_d_state;diff --git a/src/models/rnd1.cpp b/src/models/rnd1.cppindex fc276ce591..553a757302 100644--- a/src/models/rnd1.cpp+++ b/src/models/rnd1.cpp@@ -1,7 +1,7 @@#include "models.h"void llama_model_rnd1::load_arch_hparams(llama_model_loader & ml) {- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);switch (hparams.n_layer()) {@@ -49,7 +49,7 @@ void llama_model_rnd1::load_arch_tensors(llama_model_loader &) {}// MoE branch- const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);diff --git a/src/models/smallthinker.cpp b/src/models/smallthinker.cppindex a8e3d957f1..680ffb8fda 100644--- a/src/models/smallthinker.cpp+++ b/src/models/smallthinker.cpp@@ -18,7 +18,7 @@ void llama_model_smallthinker::load_arch_hparams(llama_model_loader & ml) {hparams.n_no_rope_layer_step = hparams.n_layer();}- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);@@ -57,7 +57,7 @@ void llama_model_smallthinker::load_arch_tensors(llama_model_loader &) {GGML_ASSERT(n_expert_used > 0 && "n_expert_used must be > 0 for SMALLTHINKER");// MoE branch- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, 0);layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert }, 0);layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff_exp, n_embd, n_expert }, 0);diff --git a/src/models/step35.cpp b/src/models/step35.cppindex d101d115ef..53f3179c63 100644--- a/src/models/step35.cpp+++ b/src/models/step35.cpp@@ -9,7 +9,7 @@ void llama_model_step35::load_arch_hparams(llama_model_loader & ml) {hparams.n_rot_full = hparams.n_rot_full / 2;// MoE + SWA parameters- ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);@@ -99,7 +99,7 @@ void llama_model_step35::load_arch_tensors(llama_model_loader & ml) {layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED);// MoE routed experts + selection bias (router_bias)- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED);layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED);layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, TENSOR_NOT_REQUIRED);@@ -150,7 +150,7 @@ void llama_model_step35::load_arch_tensors(llama_model_loader & ml) {layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED);// MoE routed experts + selection bias (router_bias)- const int64_t n_ff_exp = hparams.n_ff_exp;+ const int64_t n_ff_exp = hparams.n_ff_exp();layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED);layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED);layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, TENSOR_NOT_REQUIRED);