LLM inference in C/C++
C++ ggml
convert: fix get block count error for Nemotron 3 Ultra (#27101)
Rock Chena3b1effcda84
diff --git a/conversion/nemotron.py b/conversion/nemotron.pyindex 3e37c7b469..e5d1671851 100644--- a/conversion/nemotron.py+++ b/conversion/nemotron.py@@ -207,7 +207,9 @@ class NemotronHModel(GraniteHybridModel):# calling the parent __init__. This is because the parent constructor# uses self.model_arch to build the tensor name map, and all MoE-specific# mappings would be missed if it were called with the default non-MoE arch.- hparams = ModelBase.load_hparams(args[0], self.is_mistral_format)+ hparams = kwargs.pop("hparams", None)+ if hparams is None:+ hparams = ModelBase.load_hparams(args[0], self.is_mistral_format)has_moe_params = ("num_experts_per_tok" in hparamsor (isinstance(hparams.get("llm_config"), dict) and "num_experts_per_tok" in hparams["llm_config"])@@ -215,8 +217,11 @@ class NemotronHModel(GraniteHybridModel):if has_moe_params:self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOEself.is_moe = True+ layers_block_type = hparams.get("layers_block_type")+ if layers_block_type is not None:+ hparams["num_hidden_layers"] = len(layers_block_type)- super().__init__(*args, **kwargs)+ super().__init__(*args, hparams=hparams, **kwargs)# Save the top-level head_dim for laterself.head_dim = self.hparams.get("head_dim", self.hparams.get("attention_head_dim"))