LLM inference in C/C++
C++ ggml
conversion : fix Nemotron 3.5 Lightning layers (#27729)
Daniel Beveniusdac869b0a05d
diff --git a/conversion/nemotron.py b/conversion/nemotron.pyindex e5d1671851..cd0d48c8f0 100644--- a/conversion/nemotron.py+++ b/conversion/nemotron.py@@ -202,6 +202,10 @@ class NemotronHModel(GraniteHybridModel):is_moe: bool = Falsesupports_mtp_export = True+ _SSM_LAYER_TYPES = {"mamba", "linear_attention"}+ _ATTN_LAYER_TYPES = {"attention", "full_attention"}+ _MLP_LAYER_TYPES = {"moe"}+def __init__(self, *args, **kwargs):# We have to determine the correct model architecture (MoE vs non-MoE) before# calling the parent __init__. This is because the parent constructor@@ -242,8 +246,8 @@ class NemotronHModel(GraniteHybridModel):self._ssm_layers = [i for i, val in enumerate(pattern) if val == "M"]self._mlp_layers = [i for i, val in enumerate(pattern) if val == ("E" if self.is_moe else "-")]else:- self._ssm_layers = [i for i, val in enumerate(pattern) if val == "mamba"]- self._mlp_layers = [i for i, val in enumerate(pattern) if val == "moe"]+ self._ssm_layers = [i for i, val in enumerate(pattern) if val in self._SSM_LAYER_TYPES]+ self._mlp_layers = [i for i, val in enumerate(pattern) if val in self._MLP_LAYER_TYPES]# `--no-mtp` drops it entirely; `--mtp` exports only the MTP headself._mtp_bid: int | None = None@@ -272,7 +276,7 @@ class NemotronHModel(GraniteHybridModel):if isinstance(pattern, str):return [i for i, val in enumerate(pattern) if val == "*"]- return [i for i, val in enumerate(pattern) if val == "attention"]+ return [i for i, val in enumerate(pattern) if val in self._ATTN_LAYER_TYPES]@classmethoddef filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: