LLM inference in C/C++
C++ ggml
model : support Nemotron-3(.5)-Embed (#30877)
Takamichi Ohara52ffa0274556
diff --git a/conversion/__init__.py b/conversion/__init__.pyindex efa537b0f2..bddfe3c1f5 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -200,7 +200,9 @@ TEXT_MODEL_MAP: dict[str, str] = {"MiniMaxM3SparseForCausalLM": "minimax","MiniMaxM3SparseForConditionalGeneration": "minimax","Ministral3ForCausalLM": "mistral3",+ "Ministral3Model": "mistral3","Mistral3ForConditionalGeneration": "mistral3",+ "Mistral3Model": "mistral3","VoxtralRealtimeForConditionalGeneration": "mistral3","MistralForCausalLM": "llama","MixtralForCausalLM": "llama",@@ -358,6 +360,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {"MiniCPMV4_6ForConditionalGeneration": "minicpm","MiniCPMV4_7ForConditionalGeneration": "minicpm","Mistral3ForConditionalGeneration": "llava",+ "Mistral3Model": "llava","NemotronH_Omni_Reasoning_V3": "nemotron","NemotronH_Nano_VL_V2": "nemotron","MuseGlimmerForConditionalGeneration": "muse_glimmer",diff --git a/conversion/llava.py b/conversion/llava.pyindex 98a004f986..e193f0b95d 100644--- a/conversion/llava.py+++ b/conversion/llava.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport json-from typing import Iterable, TYPE_CHECKING+from typing import Callable, Iterable, TYPE_CHECKINGif TYPE_CHECKING:from torch import Tensor@@ -128,3 +128,24 @@ class LlavaVisionModel(MmprojModel):yield from super().modify_tensors(img_break_embd, name, bid)return # skip other tensors++[email protected]("Mistral3Model")[email protected]("nvidia/Nemotron-3.5-Embed-1B-BF16")+class Mistral3EmbeddingVisionModel(LlavaVisionModel):+ def __init__(self, dir_model, *args, **kwargs):+ hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)+ text_config = hparams.get("text_config", hparams)+ if text_config.get("is_causal") is not False:+ raise NotImplementedError("Expected a Nemotron Embed model for Mistral3Model, but is_causal is not False")+ super().__init__(dir_model, *args, **kwargs)++ @classmethod+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:+ if item[0].endswith("embed_tokens.weight"):+ return ModelBase.filter_tensors(item)+ return super().filter_tensors(item)++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self.gguf_writer.add_vision_preproc_image_size(self.preprocessor_config["size"]["longest_edge"])diff --git a/conversion/mistral3.py b/conversion/mistral3.pyindex 661475251a..106c329e72 100644--- a/conversion/mistral3.py+++ b/conversion/mistral3.py@@ -5,7 +5,7 @@ from typing import TYPE_CHECKINGif TYPE_CHECKING:from torch import Tensor-from .base import ModelBase, TextModel, gguf+from .base import ModelBase, TextModel, get_model_architecture, gguffrom .deepseek import DeepseekV2Modelfrom .llama import LlamaModel@@ -13,9 +13,11 @@ from .llama import LlamaModel@ModelBase.register("Mistral3ForConditionalGeneration",+ "Mistral3Model","Ministral3ForCausalLM",+ "Ministral3Model",)[email protected]("mistralai/Mistral-Small-3.1-24B-Instruct-2503", "hf-tiny-v2/tiny-random-Ministral3ForCausalLM")[email protected]("mistralai/Mistral-Small-3.1-24B-Instruct-2503", "hf-tiny-v2/tiny-random-Ministral3ForCausalLM", "nvidia/Nemotron-3-Embed-1B-BF16", "nvidia/Nemotron-3.5-Embed-1B-BF16")class Mistral3Model(TextModel):class Ministral3Model(LlamaModel):model_arch = gguf.MODEL_ARCH.MISTRAL3@@ -29,6 +31,18 @@ class Mistral3Model(TextModel):self.gguf_writer.add_rope_scaling_yarn_log_mul(rope_params["mscale_all_dim"])self.gguf_writer.add_attn_temperature_scale(rope_params["llama_4_scaling_beta"])+ class Mistral3EmbedNemotronModel(Ministral3Model):+ model_arch = gguf.MODEL_ARCH.MISTRAL3++ def set_vocab(self):+ super().set_vocab()+ self.gguf_writer.add_add_bos_token(False)+ self.gguf_writer.add_add_eos_token(False)++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self._try_set_pooling_type()+class Mistral4Model(DeepseekV2Model):model_arch = gguf.MODEL_ARCH.MISTRAL4skip_mtp = False # model contains no MTP layers, so no need to skip@@ -42,12 +56,20 @@ class Mistral3Model(TextModel):model_arch = gguf.MODEL_ARCH.MISTRAL3 # unusedimpl: TextModel- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)+ def __init__(self, dir_model, *args, **kwargs):+ hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)+ hf_arch = get_model_architecture(hparams, self.model_type)+ if hf_arch in ("Ministral3Model", "Mistral3Model"):+ text_config = hparams.get("text_config", hparams)+ if text_config.get("is_causal") is not False:+ raise NotImplementedError(f"Expected a Nemotron Embed model for {hf_arch}, but is_causal is not False")+ super().__init__(dir_model, *args, **kwargs)if self.hparams.get("model_type") == "mistral4":- self.impl = Mistral3Model.Mistral4Model(*args, **kwargs)+ self.impl = Mistral3Model.Mistral4Model(dir_model, *args, **kwargs)+ elif self.hf_arch in ("Ministral3Model", "Mistral3Model"):+ self.impl = Mistral3Model.Mistral3EmbedNemotronModel(dir_model, *args, **kwargs)else:- self.impl = Mistral3Model.Ministral3Model(*args, **kwargs)+ self.impl = Mistral3Model.Ministral3Model(dir_model, *args, **kwargs)def set_vocab(self):self.impl.set_vocab()diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.hindex 64e006e8d6..5caa93b421 100644--- a/tools/mtmd/clip-model.h+++ b/tools/mtmd/clip-model.h@@ -66,6 +66,7 @@ struct clip_hparams {resize_algo image_resize_algo = RESIZE_ALGO_BICUBIC;pad_style image_resize_pad = PAD_CEIL; // padding style when resizingbool image_round_half_even = false; // smart_resize rounds .5 to even, like Python round() in the HF processor+ bool image_floor_before_align = false;std::array<uint8_t, 3> image_pad_color = {0, 0, 0};// (preprocessor) for llava-uhd style modelsdiff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex 26941c9a85..df61f82f00 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -1625,7 +1625,23 @@ struct clip_model_loader {hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;hparams.rope_theta = 10000.0f;get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);- hparams.set_limit_image_tokens(8, 1024);+ get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);+ if (hparams.image_longest_edge > 0) {+ const int64_t patch_size = int64_t(hparams.patch_size) * hparams.n_merge;+ if (patch_size <= 0) {+ throw std::runtime_error("Pixtral merged patch size must be positive");+ }+ const int64_t n_tokens_per_side =+ (hparams.image_longest_edge + patch_size - 1) / patch_size;+ if (n_tokens_per_side * patch_size > std::sqrt(std::numeric_limits<int>::max())) {+ throw std::runtime_error("Pixtral preprocessing image size is too large");+ }+ hparams.image_resize_pad = PAD_NONE;+ hparams.image_floor_before_align = true;+ hparams.set_limit_image_tokens(0, n_tokens_per_side * n_tokens_per_side);+ } else {+ hparams.set_limit_image_tokens(8, 1024);+ }hparams.set_warmup_n_tokens(256); // avoid OOM on warmup} break;case PROJECTOR_TYPE_LIGHTONOCR:diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cppindex e0cd0bf1d0..ab6067b97e 100644--- a/tools/mtmd/mtmd-image.cpp+++ b/tools/mtmd/mtmd-image.cpp@@ -873,16 +873,28 @@ mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const clip_imagclip_image_size mtmd_image_preprocessor_longest_edge::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) {GGML_ASSERT(hparams.image_longest_edge > 0);+ clip_image_size size = original_size;+ if (hparams.image_floor_before_align) {+ const double ratio = double(std::max(size.width, size.height)) / hparams.image_longest_edge;+ if (ratio > 1.0) {+ size.width = static_cast<int>(std::floor(size.width / ratio));+ size.height = static_cast<int>(std::floor(size.height / ratio));+ if (size.width <= 0 || size.height <= 0) {+ throw std::runtime_error("image dimensions must be positive after resizing");+ }+ }+ }+// the original pixtral model doesn't have n_mergeconst int cur_merge = hparams.n_merge == 0 ? 1 : hparams.n_merge;return img_tool::calc_size_preserved_ratio(- original_size,+ size,{/* align_size */ hparams.patch_size * cur_merge,/* min_pixels */ std::max(0, hparams.image_min_pixels),/* max_pixels */ std::max(0, hparams.image_max_pixels),/* longest_edge */ std::min(hparams.image_longest_edge,- std::max(original_size.width, original_size.height)),+ std::max(size.width, size.height)),});}diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cppindex 26d5ee4105..b3ac1979ec 100644--- a/tools/mtmd/mtmd.cpp+++ b/tools/mtmd/mtmd.cpp@@ -804,7 +804,11 @@ struct mtmd_context {{// https://github.com/huggingface/transformers/blob/1cd110c6cb6a6237614130c470e9a902dbc1a4bd/docs/source/en/model_doc/pixtral.mdimg_end = "[IMG_END]";- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);+ if (clip_get_hparams(ctx_v)->image_longest_edge > 0) {+ image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(ctx_v);+ } else {+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);+ }} break;case PROJECTOR_TYPE_PHI4:{