LLM inference in C/C++
C++ ggml
model : add Ling 3.0 VL support (#29151)
Tobyf830688e9121
diff --git a/conversion/__init__.py b/conversion/__init__.pyindex f966373f1d..85db1d6431 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -28,6 +28,7 @@ TEXT_MODEL_MAP: dict[str, str] = {"BailingMoeForCausalLM": "bailingmoe","BailingMoeV2ForCausalLM": "bailingmoe","BailingMoeV3ForCausalLM": "bailingmoe3",+ "BailingMoeV3VLForConditionalGeneration": "bailingmoe3","BambaForCausalLM": "granite","BertForMaskedLM": "bert","BertForSequenceClassification": "bert",@@ -302,6 +303,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {"Gemma4ForConditionalGeneration": "gemma","Gemma4UnifiedForConditionalGeneration": "gemma","Glm4vForConditionalGeneration": "qwen3vl",+ "BailingMoeV3VLForConditionalGeneration": "bailingmoe3","Glm4vMoeForConditionalGeneration": "qwen3vl","Glm5vForConditionalGeneration": "kimivl","GlmOcrForConditionalGeneration": "qwen3vl",diff --git a/conversion/bailingmoe3.py b/conversion/bailingmoe3.pyindex 20bba23e51..36b9315649 100644--- a/conversion/bailingmoe3.py+++ b/conversion/bailingmoe3.py@@ -9,7 +9,9 @@ import torchif TYPE_CHECKING:from torch import Tensor-from .base import ModelBase, TextModel, gguf+from .base import ModelBase, MmprojModel, TextModel, gguf++from .qwen3vl import Qwen3VLVisionModel@ModelBase.register("BailingMoeV3ForCausalLM")@@ -74,7 +76,7 @@ class BailingMoeV3Model(TextModel):self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["moe_shared_expert_intermediate_size"])- self.gguf_writer.add_expert_shared_count(self.hparams["num_shared_experts"])+ self.gguf_writer.add_expert_shared_count(self.hparams.get("num_shared_experts", 1))self.gguf_writer.add_leading_dense_block_count(self.hparams["first_k_dense_replace"])self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])@@ -191,3 +193,111 @@ class BailingMoeV3Model(TextModel):experts = [name for layer in self._experts for name in layer]if experts:raise ValueError(f"Unprocessed experts: {experts}")++[email protected]("BailingMoeV3VLForConditionalGeneration")[email protected]("inclusionAI/Ling-3.0-flash-VL")+class BailingMoeV3VLModel(BailingMoeV3Model):+ model_arch = gguf.MODEL_ARCH.BAILINGMOE3++ def index_tensors(self, remote_hf_model_id: str | None = None):+ # hoist text_config before the shared BailingMoeV3 logic runs:+ # ModelBase.__init__ calls this with the raw VL config, where the text+ # dims still live under text_config+ if "text_config" in self.hparams:+ self.hparams = {**self.hparams, **self.hparams["text_config"]}+ return super().index_tensors(remote_hf_model_id=remote_hf_model_id)++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ mrope_section = self.hparams.get("mrope_section")+ if mrope_section is None:+ raise ValueError("BailingMoeV3VL requires mrope_section in the config")+ if sum(mrope_section[:3]) * 2 != self.hparams["qk_rope_head_dim"]:+ raise ValueError(+ f"mrope_section {mrope_section[:3]} counts rope pairs and must sum to"+ f" qk_rope_head_dim / 2 = {self.hparams['qk_rope_head_dim'] // 2}"+ )+ # mrope_section is [t, h, w]; pad to the 4-wide sections array+ self.gguf_writer.add_rope_dimension_sections(list(mrope_section[:3]) + [0])++ @classmethod+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:+ name, gen = item++ # Skip projector tensors; the vision tower is skipped by TextModel.filter_tensors+ if name.startswith("linear_proj"):+ return None++ return super().filter_tensors(item)++[email protected]("BailingMoeV3VLForConditionalGeneration")[email protected]("inclusionAI/Ling-3.0-flash-VL")+class BailingMoeV3VLVisionModel(Qwen3VLVisionModel):+ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ assert self.hparams_vision is not None++ if self.hparams_vision.get("disable_merger_proj") is not True:+ raise ValueError("BailingMoeV3VL requires disable_merger_proj=true")++ # out_hidden_size is the vision encoder output (post spatial merge, pre linear_proj)+ self.image_emb_dim = self.hparams_vision.get("out_hidden_size")+ if self.image_emb_dim is None:+ raise ValueError("BailingMoeV3VL vision config requires out_hidden_size")++ def set_gguf_parameters(self):+ assert self.hparams_vision is not None+ MmprojModel.set_gguf_parameters(self) # skip Qwen3VLVisionModel parameters+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.LING3VL)+ self.gguf_writer.add_vision_use_gelu(True)++ merge_size = self.hparams_vision.get("spatial_merge_size")+ if merge_size is not None:+ self.gguf_writer.add_vision_spatial_merge_size(int(merge_size))++ rms_norm_eps = self.global_config.get("text_config", {}).get("rms_norm_eps", 1e-6)+ self.gguf_writer.add_vision_attention_layernorm_eps(rms_norm_eps)++ @classmethod+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:+ name, gen = item++ if name.startswith("lm_head."):+ return None++ if name.startswith("linear_proj"):+ # top-level projector MLP: linear_proj.0 -> mm.0, linear_proj.2 -> mm.2+ parts = name.split(".")+ if len(parts) != 3:+ raise ValueError(f"Unexpected linear_proj tensor: {name}")+ idx, suffix = int(parts[1]), parts[2]+ name = f"mm.{idx}.{suffix}"+ # the qwen3vl filter keeps only visual.*; skip it for the renamed projector tensors+ return MmprojModel.filter_tensors((name, gen))++ if name.startswith("model.visual."):+ name = name.replace("model.visual.", "visual.", 1)++ if not name.startswith("visual."):+ return None++ return super().filter_tensors((name, gen))++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ assert self.hparams_vision is not None++ if name.startswith("mm.0.") or name.startswith("mm.2."):+ # top-level projector MLP (linear_proj.0 / linear_proj.2, renamed by filter_tensors)+ yield (name, data_torch)+ return++ if name == "visual.merger.norm.weight" or name == "visual.merger.norm.bias":+ # the merger is norm-only for Ling: per-patch LayerNorm before the spatial merge+ new_name = f"mm.input_norm.{name.split('.')[-1]}"+ yield (new_name, data_torch)+ return++ # Ling has no patch bias; the Conv3D split below matches the stock qwen3vl path+ yield from Qwen3VLVisionModel.modify_tensors(self, data_torch, name, bid)diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 27c83516ec..f585adcee3 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -650,6 +650,7 @@ class VISION_PROJECTOR_TYPE(IntEnum):GEMMA3N = auto()GEMMA3 = auto()QWEN3VL = auto()+ LING3VL = auto()STEP3VL = auto()COGVLM = auto()@@ -1407,6 +1408,7 @@ VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = {VISION_PROJECTOR_TYPE.MERGER: "qwen2vl_merger",VISION_PROJECTOR_TYPE.GEMMA3: "gemma3",VISION_PROJECTOR_TYPE.QWEN3VL: "qwen3vl_merger",+ VISION_PROJECTOR_TYPE.LING3VL: "ling3vl",VISION_PROJECTOR_TYPE.STEP3VL: "step3vl",}@@ -5829,6 +5831,7 @@ class VisionProjectorType:QWEN25VL = "qwen2.5vl_merger"EXAONE4_5 = "exaone4_5"QWEN3VL = "qwen3vl_merger"+ LING3VL = "ling3vl"STEP3VL = "step3vl"ULTRAVOX = "ultravox"INTERNVL = "internvl"diff --git a/src/llama-model.cpp b/src/llama-model.cppindex e195f50d0b..3801e5cbe7 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -2955,7 +2955,6 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {case LLM_ARCH_GRANITE_SWA:case LLM_ARCH_CHAMELEON:case LLM_ARCH_BAILINGMOE:- case LLM_ARCH_BAILINGMOE3:case LLM_ARCH_NEO_BERT:case LLM_ARCH_SMOLLM3:case LLM_ARCH_ARCEE:@@ -2970,6 +2969,10 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {case LLM_ARCH_DOTS3NOTE:case LLM_ARCH_NANBEIGE:case LLM_ARCH_POCKETTTS:+ return LLAMA_ROPE_TYPE_NORM;+ case LLM_ARCH_BAILINGMOE3:+ // VL files carry mrope sections; text-only files keep NORM rope+ return model->hparams.use_mrope() ? LLAMA_ROPE_TYPE_MROPE : LLAMA_ROPE_TYPE_NORM;// HY_V4 rotates consecutive pairs, matching the reference implementationcase LLM_ARCH_HY_V4:return LLAMA_ROPE_TYPE_NORM;diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cppindex e208c7d5aa..907b25c67c 100644--- a/src/models/bailingmoe3.cpp+++ b/src/models/bailingmoe3.cpp@@ -15,6 +15,7 @@ void llama_model_bailingmoe3::load_arch_hparams(llama_model_loader & ml) {hparams.kda_safe_gate = true;}ml.get_key(LLM_KV_KDA_GATE_LOWER_BOUND, hparams.kda_gate_lower_bound);+ ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false);ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);@@ -233,6 +234,10 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graphconst int64_t d_conv = hparams.ssm_d_conv;const int64_t n_seqs = ubatch.n_seqs;const int64_t n_seq_tokens = ubatch.n_seq_tokens;++ const bool use_mrope = hparams.use_mrope();+ int sections[4];+ std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);const int64_t qk_head_dim = hparams.n_embd_head_k_mla();const int64_t v_head_dim = hparams.n_embd_head_v_mla();const int64_t qk_rope_head_dim = hparams.n_rot();@@ -326,10 +331,17 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graphggml_row_size(kv_all->type, kv_lora_rank + qk_rope_head_dim),ggml_row_size(kv_all->type, kv_lora_rank));- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,- ext_factor, attn_factor, beta_fast, beta_slow);- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,- ext_factor, attn_factor, beta_fast, beta_slow);+ if (use_mrope) {+ q_pe = ggml_rope_multi(ctx0, q_pe, inp_pos, nullptr, n_rot, sections, rope_type,+ n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);+ k_pe = ggml_rope_multi(ctx0, k_pe, inp_pos, nullptr, n_rot, sections, rope_type,+ n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);+ } else {+ q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow);+ k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow);+ }kv = build_norm(kv, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);@@ -482,10 +494,21 @@ llama_model_bailingmoe3::graph_mtp::graph_mtp(const llama_model & model, const lggml_row_size(kv_all->type, kv_lora_rank + qk_rope_head_dim),ggml_row_size(kv_all->type, kv_lora_rank));- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,- ext_factor, attn_factor, beta_fast, beta_slow);- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,- ext_factor, attn_factor, beta_fast, beta_slow);+ const bool use_mrope = hparams.use_mrope();+ int sections[4];+ std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);++ if (use_mrope) {+ q_pe = ggml_rope_multi(ctx0, q_pe, inp_pos, nullptr, n_rot, sections, rope_type,+ n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);+ k_pe = ggml_rope_multi(ctx0, k_pe, inp_pos, nullptr, n_rot, sections, rope_type,+ n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);+ } else {+ q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow);+ k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow);+ }kv = build_norm(kv, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cppindex 27d00b0dfc..5a8a196c1f 100644--- a/tests/test-llama-archs.cpp+++ b/tests/test-llama-archs.cpp@@ -333,7 +333,13 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {ms.add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, uint32_t(4));ms.add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, uint32_t(1));- ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector<uint32_t>({n_embd_head/4, n_embd_head/4, n_embd_head/4, n_embd_head/4}));+ // mrope sections count rope pairs; Ling 3.0 VL files carry [t, h, w] sections+ // summing to n_rot / 2 (n_rot is 64 in this fixture)+ if (arch == LLM_ARCH_BAILINGMOE3) {+ ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector<uint32_t>({8, 12, 12, 0}));+ } else {+ ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector<uint32_t>({n_embd_head/4, n_embd_head/4, n_embd_head/4, n_embd_head/4}));+ }if (arch == LLM_ARCH_HY_V4) {ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txtindex 907468e87e..91db32c402 100644--- a/tools/mtmd/CMakeLists.txt+++ b/tools/mtmd/CMakeLists.txt@@ -55,6 +55,7 @@ add_library(mtmdmodels/qwen2vl.cppmodels/minimax-m3.cppmodels/qwen3vl.cpp+ models/ling3vl.cppmodels/mimovl.cppmodels/qwen3a.cppmodels/mimo-audio.cppdiff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.hindex 72148a4d9a..d18c76bf8f 100644--- a/tools/mtmd/clip-impl.h+++ b/tools/mtmd/clip-impl.h@@ -450,6 +450,7 @@ enum projector_type {PROJECTOR_TYPE_GLM_EDGE,PROJECTOR_TYPE_QWEN2VL,PROJECTOR_TYPE_QWEN3VL,+ PROJECTOR_TYPE_LING3VL,PROJECTOR_TYPE_STEP3VL,PROJECTOR_TYPE_GEMMA3,PROJECTOR_TYPE_GEMMA3NV,@@ -516,6 +517,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {{ PROJECTOR_TYPE_QWEN2VL, "qwen2vl_merger"},{ PROJECTOR_TYPE_QWEN25VL, "qwen2.5vl_merger"},{ PROJECTOR_TYPE_QWEN3VL, "qwen3vl_merger"},+ { PROJECTOR_TYPE_LING3VL, "ling3vl"},{ PROJECTOR_TYPE_STEP3VL, "step3vl"},{ PROJECTOR_TYPE_GEMMA3, "gemma3"},{ PROJECTOR_TYPE_GEMMA3NV, "gemma3nv"},diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex feceb7ff76..62d146e1d9 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -976,6 +976,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const{builder = std::make_unique<clip_graph_qwen3vl>(ctx, img);} break;+ case PROJECTOR_TYPE_LING3VL:+ {+ builder = std::make_unique<clip_graph_ling3vl>(ctx, img);+ } break;case PROJECTOR_TYPE_EXAONE4_5:{builder = std::make_unique<clip_graph_exaone4_5>(ctx, img);@@ -1661,6 +1665,7 @@ struct clip_model_loader {case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL:case PROJECTOR_TYPE_QWEN3VL:+ case PROJECTOR_TYPE_LING3VL:{hparams.n_merge = 2; // default value for Qwen 2 and 2.5hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;@@ -2488,6 +2493,15 @@ struct clip_model_loader {model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));} break;+ case PROJECTOR_TYPE_LING3VL:+ {+ model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); // merger.norm+ model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B); // merger.norm+ model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); // linear_proj.0+ model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));+ model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); // linear_proj.2+ model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));+ } break;case PROJECTOR_TYPE_MIMOVL:{model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));@@ -4049,6 +4063,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL:case PROJECTOR_TYPE_QWEN3VL:+ case PROJECTOR_TYPE_LING3VL:case PROJECTOR_TYPE_EXAONE4_5:case PROJECTOR_TYPE_MIMOVL:case PROJECTOR_TYPE_GLM4V:@@ -4075,6 +4090,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL:case PROJECTOR_TYPE_QWEN3VL:+ case PROJECTOR_TYPE_LING3VL:case PROJECTOR_TYPE_EXAONE4_5:case PROJECTOR_TYPE_MIMOVL:case PROJECTOR_TYPE_GLM4V:@@ -4155,6 +4171,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL:case PROJECTOR_TYPE_QWEN3VL:+ case PROJECTOR_TYPE_LING3VL:case PROJECTOR_TYPE_EXAONE4_5:case PROJECTOR_TYPE_MIMOVL:case PROJECTOR_TYPE_MINIMAX_M3:@@ -4795,6 +4812,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {} break;case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN3VL:+ case PROJECTOR_TYPE_LING3VL:case PROJECTOR_TYPE_GLM4V:{const int merge_ratio = hparams.n_merge;@@ -5959,6 +5977,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {case PROJECTOR_TYPE_QWEN3VL:// main path + deepstack pathsreturn ctx->model.mm_1_b->ne[0] * (1 + ctx->model.n_deepstack_layers);+ case PROJECTOR_TYPE_LING3VL:+ return ctx->model.mm_1_b->ne[0];case PROJECTOR_TYPE_MIMOVL:return ctx->model.mm_1_w->ne[1];case PROJECTOR_TYPE_STEP3VL:@@ -6052,6 +6072,7 @@ int clip_model_n_temporal_merge(const struct clip_ctx * ctx) {case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL:case PROJECTOR_TYPE_QWEN3VL:+ case PROJECTOR_TYPE_LING3VL:return 2;default:return 1;diff --git a/tools/mtmd/models/ling3vl.cpp b/tools/mtmd/models/ling3vl.cppnew file mode 100644index 0000000000..ef8eb3a839--- /dev/null+++ b/tools/mtmd/models/ling3vl.cpp@@ -0,0 +1,86 @@+#include "models.h"++ggml_cgraph * clip_graph_ling3vl::build() {+ // same vision tower as qwen3vl, but the merger is norm-only (no fc1/fc2) and+ // the projector MLP lives at the top level (mm.0 / mm.2)+ GGML_ASSERT(model.class_embedding == nullptr);+ GGML_ASSERT(model.mm_input_norm_w != nullptr); // merger norm (pre spatial merge)++ const int batch_size = 1;+ const int n_pos = n_patches;++ norm_type norm_t = NORM_TYPE_NORMAL;++ // vision M-RoPE, same layout as qwen3vl: [row, col, row, col] quarters+ int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};++ ggml_tensor * inp = build_inp_with_temporal_merge();++ // spatial merge+ {+ inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); // [w, h, c, b] -> [c, w, h, b]+ inp = ggml_cont_4d(+ ctx0, inp,+ n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);+ inp = ggml_reshape_4d(+ ctx0, inp,+ n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));+ inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);+ inp = ggml_cont_3d(+ ctx0, inp,+ n_embd, n_patches_x * n_patches_y, batch_size);+ }++ // add patch bias+ if (model.patch_bias != nullptr) {+ inp = ggml_add(ctx0, inp, model.patch_bias);+ cb(inp, "patch_bias", -1);+ }++ // calculate absolute position embedding and apply+ ggml_tensor * learned_pos_embd = resize_position_embeddings(GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS);+ learned_pos_embd = ggml_cont_4d(+ ctx0, learned_pos_embd,+ n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);+ learned_pos_embd = ggml_reshape_4d(+ ctx0, learned_pos_embd,+ n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));+ learned_pos_embd = ggml_permute(ctx0, learned_pos_embd, 0, 2, 1, 3);+ learned_pos_embd = ggml_cont_3d(+ ctx0, learned_pos_embd,+ n_embd, n_patches_x * n_patches_y, batch_size);++ const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position+ ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);+ ggml_set_name(positions, "positions");+ ggml_set_input(positions);++ ggml_tensor * inpL = build_vit(+ inp, n_pos, norm_t, hparams.ffn_op, learned_pos_embd,+ [&](ggml_tensor * c, const clip_layer &) {+ return ggml_rope_multi(+ ctx0, c, positions, nullptr,+ d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);+ });++ // multimodal projection (linear_proj MLP over the merged patches)+ ggml_tensor * embeddings = inpL;++ // per-patch merger norm, applied post-blocks before the 2x2 merge+ // (merger.norm, LayerNorm over n_embd)+ embeddings = build_norm(embeddings, model.mm_input_norm_w, model.mm_input_norm_b, norm_t, eps, -1);+ cb(embeddings, "merger_norm", -1);++ embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size);++ embeddings = build_ffn(embeddings,+ model.mm_0_w, model.mm_0_b,+ nullptr, nullptr,+ model.mm_1_w, model.mm_1_b,+ ffn_op_type::FFN_GELU, -1);++ // build the graph+ ggml_build_forward_expand(gf, embeddings);++ return gf;+}diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.hindex 5945c6d92c..adb5ede309 100644--- a/tools/mtmd/models/models.h+++ b/tools/mtmd/models/models.h@@ -50,6 +50,11 @@ struct clip_graph_qwen3vl : clip_graph_qwen2vl {ggml_cgraph * build() override;};+struct clip_graph_ling3vl : clip_graph_qwen3vl {+ clip_graph_ling3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen3vl(ctx, img) {}+ ggml_cgraph * build() override;+};+struct clip_graph_minimax_m3 : clip_graph {clip_graph_minimax_m3(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}ggml_cgraph * build() override;diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cppindex 00ecadcf4d..2efa4d65d3 100644--- a/tools/mtmd/mtmd.cpp+++ b/tools/mtmd/mtmd.cpp@@ -694,6 +694,7 @@ struct mtmd_context {case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL:case PROJECTOR_TYPE_QWEN3VL:+ case PROJECTOR_TYPE_LING3VL:case PROJECTOR_TYPE_MIMOVL:{// <|vision_start|> ... (image embeddings) ... <|vision_end|>