LLM inference in C/C++
C++ ggml
models : add Nemotron Super 3.5 VL and Nemotron Bolt 74B A9B support (#30520)
Kartik Guliacd0a5b6a180d
diff --git a/common/speculative.cpp b/common/speculative.cppindex 7010933082..d8afd40954 100644--- a/common/speculative.cpp+++ b/common/speculative.cpp@@ -1338,7 +1338,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {GGML_ASSERT(lattice && "DFlash2 selector produced no lattice");int32_t predecessor = 0;- for (int32_t i = 1; i < n_block_tokens; ++i) {+ for (int32_t i = 1; i < (is_dflash2 ? n_block_tokens : params.n_max + 1); ++i) {const float * row = lattice + (size_t) (beg + i) * n_embd_dec;const int32_t top_k = is_dflash2 ? selector_top_k : lilicorr_top_k;const float * scores = row + top_k + (size_t) predecessor * top_k;diff --git a/conversion/__init__.py b/conversion/__init__.pyindex d5b480a9bf..8ce4cb7337 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -353,6 +353,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {"MiniCPMV4_6ForConditionalGeneration": "minicpm","MiniCPMV4_7ForConditionalGeneration": "minicpm","Mistral3ForConditionalGeneration": "llava",+ "NemotronH_Omni_Reasoning_V3": "nemotron","NemotronH_Nano_VL_V2": "nemotron","MuseGlimmerForConditionalGeneration": "muse_glimmer","PaddleOCRVisionModel": "ernie",diff --git a/conversion/base.py b/conversion/base.pyindex efb7ea72e2..a5a80f27da 100644--- a/conversion/base.py+++ b/conversion/base.py@@ -568,7 +568,7 @@ class ModelBase:tensors_to_remove.append(base_name + "_zero_point")else:raise NotImplementedError(f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported")- elif quant_method == "modelopt":+ elif quant_method in ("modelopt", "modelopt_mixed"):dequant_fp8()elif quant_method is not None:raise NotImplementedError(f"Quant method is not yet supported: {quant_method!r}")diff --git a/conversion/nemotron.py b/conversion/nemotron.pyindex 581f41a023..c900481943 100644--- a/conversion/nemotron.py+++ b/conversion/nemotron.py@@ -1,6 +1,6 @@from __future__ import annotations-from typing import Any, Callable, Iterable, TYPE_CHECKING+from typing import Any, Callable, Iterable, Sequence, TYPE_CHECKINGimport numpy as npimport torch@@ -89,19 +89,22 @@ class LiLiCorrDraftModel(DFlashModel):self.gguf_writer.add_lilicorr_vector_eps(float(dflash_config["lilicorr_vector_eps"]))self.gguf_writer.add_lilicorr_logit_scale(float(dflash_config["lilicorr_logit_scale"]))+ def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", ".bias")) -> str:+ if name.startswith("model.lilicorr.layers."):+ parts = name.split(".", 4)+ if not parts[4].startswith("attn.in_proj."):+ parts[3] = str(self.block_count + int(parts[3]))+ name = ".".join(parts)+ elif name.startswith("model.lilicorr.feature_norm."):+ feature_bid = self.block_count + self.lilicorr_geometry["num_layers"]+ name = name.replace("model.lilicorr.feature_norm.", f"model.lilicorr.feature_mlp.{feature_bid}.norm.")+ elif name.startswith("model.lilicorr.feature_mlp."):+ feature_bid = self.block_count + self.lilicorr_geometry["num_layers"]+ name = name.replace("model.lilicorr.feature_mlp.", f"model.lilicorr.feature_mlp.{feature_bid}.")+ return super().map_tensor_name(name, try_suffixes)+def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:if name.startswith("model.lilicorr."):- if name.startswith("model.lilicorr.layers."):- parts = name.split(".", 4)- if not parts[4].startswith("attn.in_proj."):- parts[3] = str(self.block_count + int(parts[3]))- name = ".".join(parts)- elif name.startswith("model.lilicorr.feature_norm."):- feature_bid = self.block_count + self.lilicorr_geometry["num_layers"]- name = name.replace("model.lilicorr.feature_norm.", f"model.lilicorr.feature_mlp.{feature_bid}.norm.")- elif name.startswith("model.lilicorr.feature_mlp."):- feature_bid = self.block_count + self.lilicorr_geometry["num_layers"]- name = name.replace("model.lilicorr.feature_mlp.", f"model.lilicorr.feature_mlp.{feature_bid}.")yield self.map_tensor_name(name), data_torchreturnyield from super().modify_tensors(data_torch, name, bid)@@ -251,6 +254,113 @@ class NemotronNanoV2VLModel(MmprojModel):yield mapped_name, tensor[email protected]("NemotronH_Omni_Reasoning_V3")+class NemotronOmniVLModel(NemotronNanoV2VLModel):+ _omni_tensor_prefixes = (+ "vision_model.embeddings.",+ "vision_model.encoder.layer.",+ "vision_projector.mlp1.",+ )++ def get_vision_config(self) -> dict[str, Any] | None:+ vision_config = dict(self.global_config["vision_config"])+ vision_config["intermediate_size"] = int(vision_config["hidden_size"] * vision_config["mlp_ratio"])+ vision_config["image_size"] = 512 # warmup only; the full position table is resized at runtime+ return vision_config++ def set_gguf_parameters(self):+ preprocessor_config = self.preprocessor_config++ preprocessor_config["image_mean"] = preprocessor_config["norm_mean"]+ preprocessor_config["image_std"] = preprocessor_config["norm_std"]++ MmprojModel.set_gguf_parameters(self)+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.NEMOTRON_OMNI)+ self.gguf_writer.add_vision_attention_layernorm_eps(1e-6)+ self.gguf_writer.add_vision_use_gelu(True)+ downsample_ratio = self.global_config.get("downsample_ratio", 0.5)+ self.gguf_writer.add_vision_projector_scale_factor(int(1.0 / downsample_ratio))+ patch_area = self.hparams["patch_size"] ** 2+ self.gguf_writer.add_vision_min_pixels(preprocessor_config["min_num_patches"] * patch_area)+ self.gguf_writer.add_vision_max_pixels(preprocessor_config["max_num_patches"] * patch_area)++ @classmethod+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:+ name, gen = item++ # Transformers can rename RADIO and projector tensors in Omni checkpoints.+ # Accept both checkpoint layouts.+ if name == "vision_model.summary_idxs":+ # This buffer is only used by RadioModel.summary. The multimodal+ # projector consumes RadioModel.features for both images and video.+ return None+ if name.startswith(cls._omni_tensor_prefixes):+ return name, gen++ if "radio_model.model.patch_generator.video_embedder" in name or name.startswith("vision_projector.vision_final_layernorm."):+ return MmprojModel.filter_tensors((name, gen))++ return super().filter_tensors(item)++ def tensor_force_quant(self, name, new_name, bid, n_dims):+ if new_name in ("v.position_embd.weight", "v.class_embd"):+ return gguf.GGMLQuantizationType.F32+ return super().tensor_force_quant(name, new_name, bid, n_dims)++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ if name.startswith(self._omni_tensor_prefixes):+ if name == "vision_model.embeddings.video_patch_projection.weight":+ data_torch = data_torch.reshape(+ self.hparams["hidden_size"],+ 6,+ self.hparams["patch_size"],+ self.hparams["patch_size"],+ )+ yield gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch+ return++ if name.startswith("vision_projector.mlp1."):+ projector_mapping = {+ "vision_projector.mlp1.norm.weight": "mlp1.0.weight",+ "vision_projector.mlp1.linear1.weight": "mlp1.1.weight",+ "vision_projector.mlp1.linear2.weight": "mlp1.3.weight",+ }+ name = projector_mapping[name]+ elif name == "vision_model.embeddings.patch_projection.weight":+ data_torch = data_torch.reshape(+ self.hparams["hidden_size"],+ 3,+ self.hparams["patch_size"],+ self.hparams["patch_size"],+ )+ elif name == "vision_model.embeddings.position_embedding":+ data_torch = data_torch.squeeze(0)+ name += ".weight"+ elif ".layer_scale" in name and name.endswith(".lambda1"):+ name = name.removesuffix("lambda1") + "weight"++ yield from MmprojModel.modify_tensors(self, data_torch, name, bid)+ return++ if "patch_generator.pos_embed" in name:+ # Keep the full table for runtime interpolation.+ yield from MmprojModel.modify_tensors(self, data_torch, name, bid)+ return++ if "patch_generator.video_embedder" in name:+ patch_size = self.hparams["patch_size"]+ n_embd = self.hparams["hidden_size"]+ temporal_patch_size = self.global_config["video_temporal_patch_size"]+ if temporal_patch_size != 2:+ raise ValueError("Current implementation only supports a temporal patch size of 2")+ data_torch = data_torch.reshape(n_embd, temporal_patch_size * 3, patch_size, patch_size)+ new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1"+ yield new_name, data_torch+ return++ yield from super().modify_tensors(data_torch, name, bid)++@ModelBase.register("NemotronForCausalLM")@ModelBase.example("nvidia/Minitron-4B-Base")class NemotronModel(TextModel):@@ -382,6 +492,8 @@ class NemotronHModel(GraniteHybridModel):@classmethoddef filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:name, gen = item+ if name.startswith("language_model."):+ name = name[len("language_model."):]if name.startswith("mtp."):# --no-mtp: drop the MTP head entirelyif cls.no_mtp:@@ -531,6 +643,14 @@ class NemotronHModel(GraniteHybridModel):"mtp.layers.1.final_layernorm.weight": "model.layers.{bid}.shared_head.norm.weight",}+ def _generate_nvfp4_tensors(self):+ if self._mtp_bid is not None:+ for name in list(self.model_tensors):+ if name.startswith("mtp.layers.1.mixer"):+ new_name = f"backbone.layers.{self._mtp_bid}.{name.split('.', 3)[3]}"+ self.model_tensors[new_name] = self.model_tensors.pop(name)+ super()._generate_nvfp4_tensors()+def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# mtp.layers.0: NextN input fusion + attention# mtp.layers.1: MoE + final head normdiff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 0c9872228a..a39d046fe1 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -6265,6 +6265,7 @@ class VisionProjectorType:GLM5V = "glm5v"YOUTUVL = "youtuvl"NEMOTRON_V2_VL = "nemotron_v2_vl"+ NEMOTRON_OMNI = "nemotron_omni"QWEN3TTS_SPKENC = "qwen3tts_spkenc" # audio: ECAPA-TDNN speaker encoderQWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictorPOCKETTTS_SPKENC = "pockettts_spkenc" # audio: mimi encoder as voice-prompt encoderdiff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.pyindex 5eab5b1fdc..63301cc114 100644--- a/gguf-py/gguf/tensor_mapping.py+++ b/gguf-py/gguf/tensor_mapping.py@@ -1735,6 +1735,7 @@ class TensorNameMap:"vision_model.class_embedding", # llama 4"model.vision.patch_embedding.cls_embedding", # cogvlm"vision_model.radio_model.model.patch_generator.cls_token.token", # Nemotron Nano v2 VL+ "vision_model.embeddings.cls_register_token", # Nemotron Omni"model.vision_model.embeddings.class_embedding", # Deepseek-OCR),@@ -1756,6 +1757,7 @@ class TensorNameMap:"model.vision_model.embeddings.patch_embedding", # Deepseek-OCR CLIP"siglip2.vision_model.embeddings.patch_embedding","vision_model.radio_model.model.patch_generator.embedder", # Nemotron Nano v2 VL+ "vision_model.embeddings.patch_projection", # Nemotron Omni"model.vision_tower.patch_embedder.input_proj", # gemma4"vision_tower.patch_embed.patchifier.proj", # dots.ocr"vision_encoder.patch_embed.proj", # dots3note@@ -1788,6 +1790,7 @@ class TensorNameMap:"model.vision.patch_embedding.position_embedding", # cogvlm"visual.embeddings.position_embedding", # glm4v"vision_model.radio_model.model.patch_generator.pos_embed", # Nemotron Nano v2 VL+ "vision_model.embeddings.position_embedding", # Nemotron Omni"model.vision_tower.patch_embedder.position_embedding_table", # gemma4"vision_model.positional_embedding", # Step3-VL"model.vision_embedder.pos_embedding", # gemma4 unified@@ -1818,6 +1821,7 @@ class TensorNameMap:),MODEL_TENSOR.V_ENC_ATTN_Q: (+ "vision_model.encoder.layer.{bid}.attention.attention.query", # Nemotron Omni"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.q_proj", # Granite4Vision"vision_tower.vision_model.encoder.layers.{bid}.self_attn.q_proj","model.vision_tower.encoder.layers.{bid}.self_attn.q_proj", # minicpmv4_6@@ -1846,6 +1850,7 @@ class TensorNameMap:),MODEL_TENSOR.V_ENC_ATTN_K: (+ "vision_model.encoder.layer.{bid}.attention.attention.key", # Nemotron Omni"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.k_proj", # Granite4Vision"vision_tower.vision_model.encoder.layers.{bid}.self_attn.k_proj","model.vision_tower.encoder.layers.{bid}.self_attn.k_proj", # minicpmv4_6@@ -1874,6 +1879,7 @@ class TensorNameMap:),MODEL_TENSOR.V_ENC_ATTN_V: (+ "vision_model.encoder.layer.{bid}.attention.attention.value", # Nemotron Omni"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.v_proj", # Granite4Vision"vision_tower.vision_model.encoder.layers.{bid}.self_attn.v_proj","model.vision_tower.encoder.layers.{bid}.self_attn.v_proj", # minicpmv4_6@@ -1894,6 +1900,7 @@ class TensorNameMap:),MODEL_TENSOR.V_ENC_INPUT_NORM: (+ "vision_model.encoder.layer.{bid}.norm1", # Nemotron Omni"vision.blocks.{bid}.norm1", # deepseek4v"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", # Granite4Vision"vision_tower.vision_model.encoder.layers.{bid}.layer_norm1",@@ -1920,6 +1927,7 @@ class TensorNameMap:),MODEL_TENSOR.V_ENC_ATTN_O: (+ "vision_model.encoder.layer.{bid}.attention.output.dense", # Nemotron Omni"vision.blocks.{bid}.attn.wo", # deepseek4v"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", # Granite4Vision"vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj",@@ -1952,6 +1960,7 @@ class TensorNameMap:),MODEL_TENSOR.V_ENC_POST_ATTN_NORM: (+ "vision_model.encoder.layer.{bid}.norm2", # Nemotron Omni"vision.blocks.{bid}.norm2", # deepseek4v"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", # Granite4Vision"vision_tower.vision_model.encoder.layers.{bid}.layer_norm2",@@ -1979,6 +1988,7 @@ class TensorNameMap:),MODEL_TENSOR.V_ENC_FFN_UP: (+ "vision_model.encoder.layer.{bid}.mlp.fc1", # Nemotron Omni"vision.blocks.{bid}.mlp.w1_up", # deepseek4v (split from fused w1)"vision_encoder.blocks.{bid}.mlp.fc3", # dots3note"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision@@ -2016,6 +2026,7 @@ class TensorNameMap:),MODEL_TENSOR.V_ENC_FFN_DOWN: (+ "vision_model.encoder.layer.{bid}.mlp.fc2", # Nemotron Omni"vision.blocks.{bid}.mlp.w2", # deepseek4v"vision_encoder.blocks.{bid}.mlp.fc2", # dots3note"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision@@ -2074,12 +2085,14 @@ class TensorNameMap:),MODEL_TENSOR.V_LAYER_SCALE_1: (+ "vision_model.encoder.layer.{bid}.layer_scale1", # Nemotron Omni"vision_tower.vision_model.encoder.layers.{bid}.ls1", # InternVL"model.vision_tower.encoder.layer.{bid}.lambda_1", # Intern-S1"vision_model.transformer.resblocks.{bid}.ls_1", # Step3-VL),MODEL_TENSOR.V_LAYER_SCALE_2: (+ "vision_model.encoder.layer.{bid}.layer_scale2", # Nemotron Omni"vision_tower.vision_model.encoder.layers.{bid}.ls2", # InternVL"model.vision_tower.encoder.layer.{bid}.lambda_2", # Intern-S1"vision_model.transformer.resblocks.{bid}.ls_2", # Step3-VL@@ -2114,6 +2127,7 @@ class TensorNameMap:"siglip2.vision_model.post_layernorm","model.qwen2_model.model.model.norm", # Deepseek-OCR-2 qwen2"model.vision_tower.ln_post", # muse-glimmer+ "vision_projector.vision_final_layernorm", # Nemotron Omni Reasoning V3),MODEL_TENSOR.V_MM_POST_NORM: (diff --git a/src/models/dflash.cpp b/src/models/dflash.cppindex 982351dc86..6f28d2f1db 100644--- a/src/models/dflash.cpp+++ b/src/models/dflash.cpp@@ -930,7 +930,7 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_grares->t_inp_tokens = inp_tokens;ggml_tensor * inp_anchor = nullptr;- if (model.lilicorr_pass_hidden_proj && ubatch.embd_state) {+ if (model.lilicorr_pass_hidden_proj && ubatch.embd_state && n_tokens == (int64_t) hparams.dflash_block_size * ubatch.n_seqs_unq) {// LiliCorr sets inp_anchor to pass the context features to the lattice scorerauto inp = std::make_unique<llm_graph_input_embd_h>(n_embd, n_embd);inp->tokens = inp_tokens;@@ -1100,7 +1100,7 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_graif (model.dflash_selector_hidden) {build_dflash2_selector(*this, model, inp_tokens);- } else if (model.lilicorr_pass_hidden_proj && ubatch.embd_state) {+ } else if (model.lilicorr_pass_hidden_proj && ubatch.embd_state && n_tokens == (int64_t) hparams.dflash_block_size * ubatch.n_seqs_unq) {build_lilicorr_scores(*this, model, tok_embd, inp_anchor);}}diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cppindex b14913f2a8..e9c61308fe 100644--- a/src/models/nemotron-h.cpp+++ b/src/models/nemotron-h.cpp@@ -109,21 +109,26 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) {layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);} else {if (n_expert != 0) {+ const auto * router_meta = ml.get_tensor_meta(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i).str().c_str());+ const int64_t n_expert_layer = router_meta ? router_meta->ne[1] : n_expert;+ if (n_expert_layer < hparams.n_expert_used(i) || n_expert_layer > n_expert) {+ throw std::runtime_error("Invalid per-layer Nemotron expert count");+ }// Use per-layer n_ff_exp; fall back to n_ff/n_expert_used if absent (existing GGUFs).const int64_t n_ff_exp_i = hparams.n_ff_exp(i)? (int64_t)hparams.n_ff_exp(i): hparams.n_ff(i) / (int64_t)hparams.n_expert_used(i);const int64_t n_ff_shexp = hparams.n_ff_shexp;- layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert}, trunk_flags);- layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert }, trunk_flags);+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert_layer}, trunk_flags);+ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert_layer}, trunk_flags);// MoE branchlayer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", i), {n_embd, moe_n_embd}, TENSOR_NOT_REQUIRED);layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", i), {moe_n_embd, n_embd}, TENSOR_NOT_REQUIRED);- layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp_i, moe_n_embd, n_expert}, trunk_flags);- layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp_i, n_expert}, trunk_flags);+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp_i, moe_n_embd, n_expert_layer}, trunk_flags);+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp_i, n_expert_layer}, trunk_flags);// Shared expert branchlayer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, trunk_flags);@@ -178,6 +183,15 @@ void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) {layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp, n_expert}, mtp_flags);layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, mtp_flags);layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, mtp_flags);++ layer.ffn_down_exps_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED);+ layer.ffn_up_exps_s = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED);+ layer.ffn_down_exps_in_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "input_scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED);+ layer.ffn_up_exps_in_s = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "input_scale", i), {n_expert}, mtp_flags | TENSOR_NOT_REQUIRED);+ layer.ffn_down_shexp_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED);+ layer.ffn_up_shexp_s = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED);+ layer.ffn_down_shexp_in_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "input_scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED);+ layer.ffn_up_shexp_in_s = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "input_scale", i), {1}, mtp_flags | TENSOR_NOT_REQUIRED);}}@@ -304,7 +318,7 @@ ggml_tensor * llama_model_nemotron_h::graph::build_ffn_layer(ggml_tensor * cur,nullptr, // no gatemodel.layers[il].ffn_down_exps,model.layers[il].ffn_exp_probs_b,- n_expert, (int64_t)hparams.n_expert_used(il),+ model.layers[il].ffn_up_exps->ne[2], (int64_t)hparams.n_expert_used(il),LLM_FFN_RELU_SQR, hparams.expert_weights_norm,hparams.expert_weights_scale,LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txtindex 20098937ce..82f74bdc96 100644--- a/tools/mtmd/CMakeLists.txt+++ b/tools/mtmd/CMakeLists.txt@@ -46,6 +46,7 @@ add_library(mtmdmodels/kimivl.cppmodels/kimik25.cppmodels/nemotron-v2-vl.cpp+ models/nemotron-omni.cppmodels/muse-glimmer.cppmodels/llama4.cppmodels/llava.cppdiff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.hindex 30cb307891..3d6c6f8c48 100644--- a/tools/mtmd/clip-impl.h+++ b/tools/mtmd/clip-impl.h@@ -498,6 +498,7 @@ enum projector_type {PROJECTOR_TYPE_YASA2,PROJECTOR_TYPE_KIMIK25,PROJECTOR_TYPE_NEMOTRON_V2_VL,+ PROJECTOR_TYPE_NEMOTRON_OMNI,PROJECTOR_TYPE_HUNYUANVL,PROJECTOR_TYPE_PARAKEET,PROJECTOR_TYPE_EXAONE4_5,@@ -569,6 +570,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {{ PROJECTOR_TYPE_YASA2, "yasa2"},{ PROJECTOR_TYPE_KIMIK25, "kimik25"},{ PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"},+ { PROJECTOR_TYPE_NEMOTRON_OMNI, "nemotron_omni"},{ PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"},{ PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"},{ PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"},@@ -904,11 +906,13 @@ static void clip_log_internal(enum ggml_log_level level, const char * format, ..struct clip_image_f32_batch {std::vector<clip_image_f32> entries;bool is_audio = false;+ bool is_video = false;clip_image_f32_batch clone() const {clip_image_f32_batch new_batch{/* entries */ {},/* is_audio */ is_audio,+ /* is_video */ is_video,};new_batch.entries.reserve(entries.size());for (const auto & entry : entries) {diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.hindex 657671ded5..2c4ea398cc 100644--- a/tools/mtmd/clip-model.h+++ b/tools/mtmd/clip-model.h@@ -566,7 +566,7 @@ struct clip_model {// embeddingsggml_tensor * class_embedding = nullptr;ggml_tensor * patch_embeddings_0 = nullptr;- ggml_tensor * patch_embeddings_1 = nullptr; // second Conv2D kernel when we decouple Conv3D along temporal dimension (Qwen2VL)+ ggml_tensor * patch_embeddings_1 = nullptr; // additional patch kernel; interpretation is model-specific (Qwen2VL, Nemotron Omni)ggml_tensor * patch_bias = nullptr;ggml_tensor * position_embeddings = nullptr;ggml_tensor * norm_embd_w = nullptr;diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex fccb00c3aa..4c1185a431 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -1042,6 +1042,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const{builder = std::make_unique<clip_graph_nemotron_v2_vl>(ctx, img);} break;+ case PROJECTOR_TYPE_NEMOTRON_OMNI:+ {+ builder = std::make_unique<clip_graph_nemotron_omni>(ctx, img);+ } break;case PROJECTOR_TYPE_LLAMA4:{builder = std::make_unique<clip_graph_llama4>(ctx, img);@@ -1523,6 +1527,16 @@ struct clip_model_loader {{get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);} break;+ case PROJECTOR_TYPE_NEMOTRON_OMNI:+ {+ hparams.ffn_op = FFN_GELU_ERF;+ log_ffn_op = "gelu_erf";+ hparams.image_resize_pad = PAD_NONE;+ get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);+ get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);+ get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);+ hparams.warmup_image_size = hparams.image_size;+ } break;case PROJECTOR_TYPE_PARAKEET:{get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor);@@ -3269,6 +3283,7 @@ struct clip_model_loader {model.mm_3_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "bias"));} break;case PROJECTOR_TYPE_NEMOTRON_V2_VL:+ case PROJECTOR_TYPE_NEMOTRON_OMNI:{model.mm_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));model.mm_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));@@ -4320,6 +4335,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {case PROJECTOR_TYPE_COHERE2V:case PROJECTOR_TYPE_INTERNVL:case PROJECTOR_TYPE_NEMOTRON_V2_VL:+ case PROJECTOR_TYPE_NEMOTRON_OMNI:case PROJECTOR_TYPE_LLAMA4:{// both X and Y are downscaled by the scale factor@@ -5428,6 +5444,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {case PROJECTOR_TYPE_COHERE2V:case PROJECTOR_TYPE_INTERNVL:case PROJECTOR_TYPE_NEMOTRON_V2_VL:+ case PROJECTOR_TYPE_NEMOTRON_OMNI:case PROJECTOR_TYPE_QWEN2A:case PROJECTOR_TYPE_QWEN3A:case PROJECTOR_TYPE_GLMA:@@ -6149,6 +6166,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {return ctx->model.mm_3_w->ne[1]; // out_proj output dimcase PROJECTOR_TYPE_INTERNVL:case PROJECTOR_TYPE_NEMOTRON_V2_VL:+ case PROJECTOR_TYPE_NEMOTRON_OMNI:return ctx->model.mm_3_w->ne[1];case PROJECTOR_TYPE_LLAMA4:return ctx->model.mm_model_proj->ne[1];@@ -6225,6 +6243,7 @@ int clip_model_n_temporal_merge(const struct clip_ctx * ctx) {case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL:case PROJECTOR_TYPE_QWEN3VL:+ case PROJECTOR_TYPE_NEMOTRON_OMNI:case PROJECTOR_TYPE_LING3VL:return 2;default:diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.hindex adb5ede309..2f4676a543 100644--- a/tools/mtmd/models/models.h+++ b/tools/mtmd/models/models.h@@ -109,6 +109,12 @@ struct clip_graph_nemotron_v2_vl : clip_graph {ggml_cgraph * build() override;};+struct clip_graph_nemotron_omni : clip_graph {+ clip_graph_nemotron_omni(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}+ ggml_cgraph * build() override;+ ggml_tensor * build_inp_with_temporal_merge();+};+struct clip_graph_llama4 : clip_graph {clip_graph_llama4(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}ggml_cgraph * build() override;diff --git a/tools/mtmd/models/nemotron-omni.cpp b/tools/mtmd/models/nemotron-omni.cppnew file mode 100644index 0000000000..c4b1bff406--- /dev/null+++ b/tools/mtmd/models/nemotron-omni.cpp@@ -0,0 +1,56 @@+#include "models.h"++ggml_tensor * clip_graph_nemotron_omni::build_inp_with_temporal_merge() {+ if (n_batch == 1 && !img_batch->is_video) {+ return build_inp();+ }++ GGML_ASSERT(n_batch == 1 || n_batch == 2);+ GGML_ASSERT(model.patch_embeddings_1 != nullptr);+ GGML_ASSERT(model.patch_embeddings_1->ne[2] == 6);++ ggml_tensor * inp_raw = build_inp_raw();+ if (n_batch == 1) {+ // pad the last video frame inside the graph+ inp_raw = ggml_concat(ctx0, inp_raw, inp_raw, 2);+ } else {+ inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.ny(), 6, 1);+ }++ ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1);+ inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, 1);+ return ggml_cont(ctx0, ggml_transpose(ctx0, inp));+}++ggml_cgraph * clip_graph_nemotron_omni::build() {+ GGML_ASSERT(model.class_embedding != nullptr);+ GGML_ASSERT(model.position_embeddings != nullptr);++ const int n_registers = model.class_embedding->ne[1];+ const int n_pos = n_patches + n_registers;++ ggml_tensor * inp = build_inp_with_temporal_merge();++ inp = ggml_add(ctx0, inp, resize_position_embeddings(GGML_SCALE_MODE_BILINEAR));+ cb(inp, "inp_pos", -1);++ inp = ggml_concat(ctx0, model.class_embedding, inp, 1);++ ggml_tensor * cur = build_vit(inp, n_pos, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, nullptr);++ cur = ggml_view_2d(ctx0, cur,+ n_embd, n_patches,+ ggml_row_size(cur->type, n_embd),+ n_registers * ggml_row_size(cur->type, n_embd));++ cur = build_patch_merge_permute(cur, model.hparams.n_merge);++ {+ cur = build_norm(cur, model.mm_0_w, nullptr, NORM_TYPE_RMS, 1e-5, -1);+ cur = build_ffn(cur, model.mm_1_w, nullptr, nullptr, nullptr, model.mm_3_w, nullptr, FFN_RELU_SQR, -1);+ }++ ggml_build_forward_expand(gf, cur);++ return gf;+}diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cppindex 2c2b069361..2fa02f83eb 100644--- a/tools/mtmd/mtmd.cpp+++ b/tools/mtmd/mtmd.cpp@@ -898,6 +898,12 @@ struct mtmd_context {{image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v);} break;+ case PROJECTOR_TYPE_NEMOTRON_OMNI:+ {+ img_beg = "<img>";+ img_end = "</img>";+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);+ } break;case PROJECTOR_TYPE_LFM2:{// multi-tile:@@ -1607,6 +1613,7 @@ struct mtmd_tokenizer {clip_image_f32_batch batch_f32;batch_f32.is_audio = false;+ batch_f32.is_video = bitmaps[0]->mergeable;batch_f32.entries = std::move(preproc_out.entries);// do NOT use preproc_out from this point on, it's moved