LLM inference in C/C++
C++ ggml
mtmd: add minicpmv46 downsample (#25993)
tc-mbde699957b92f
diff --git a/conversion/minicpm.py b/conversion/minicpm.pyindex e31b26a008..bf3fa81421 100644--- a/conversion/minicpm.py+++ b/conversion/minicpm.py@@ -137,6 +137,15 @@ class MiniCPMV4_6TextModel(Qwen3_5TextModel):class MiniCPMV4_6VisionModel(MmprojModel):def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)+ self.downsample_mode = self.preprocessor_config.get("downsample_mode", "16x")+ if self.downsample_mode not in {"4x", "16x"}:+ raise ValueError(f"Unsupported downsample mode: {self.downsample_mode}")+ if self.downsample_mode == "4x":+ self.model_tensors = {+ name: tensor for name, tensor in self.model_tensors.items()+ if ".vit_merger." not in name+ }+if self.hparams_vision is not None:# In MiniCPM-V 4.6 `vision_config.image_size` (980) describes the SigLIP# positional embedding bucket grid (70 x 70), while the per-slice processing@@ -156,8 +165,8 @@ class MiniCPMV4_6VisionModel(MmprojModel):# (mapped to PROJECTOR_TYPE_MINICPMV4_6).self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINICPMV4_6)- # ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension; used for slice alignment- self.gguf_writer.add_vision_projector_scale_factor(4)+ self.gguf_writer.add_vision_projector_scale_factor(+ 2 if self.downsample_mode == "4x" else 4)# borrow wa_layer_indexes for vit_merger insertion pointinsert_layer_id = int(self.global_config.get(diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex 5f0d00b660..c1870813fb 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -1337,6 +1337,7 @@ struct clip_model_loader {// ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimensionhparams.n_merge = 4;get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);+ GGML_ASSERT(hparams.n_merge == 2 || hparams.n_merge == 4);// borrow wa_layer_indexes for vit_merger insertion pointstd::vector<int> wa_layer_indexes_vec;@@ -2143,24 +2144,29 @@ struct clip_model_loader {} break;case PROJECTOR_TYPE_MINICPMV4_6:{+ const bool merger_required = hparams.n_merge == 4;+ auto get_merger_tensor = [&](const std::string & name, bool required = true) {+ return get_tensor(name, merger_required && required);+ };+// ViT merger: window self-attention- model.vit_merger_ln1_w = get_tensor(string_format(TN_VIT_MERGER_LN1, "weight"));- model.vit_merger_ln1_b = get_tensor(string_format(TN_VIT_MERGER_LN1, "bias"));- model.vit_merger_attn_q_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "weight"));- model.vit_merger_attn_q_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "bias"), false);- model.vit_merger_attn_k_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_K, "weight"));- model.vit_merger_attn_k_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_K, "bias"), false);- model.vit_merger_attn_v_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_V, "weight"));- model.vit_merger_attn_v_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_V, "bias"), false);- model.vit_merger_attn_o_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_O, "weight"));- model.vit_merger_attn_o_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_O, "bias"), false);+ model.vit_merger_ln1_w = get_merger_tensor(string_format(TN_VIT_MERGER_LN1, "weight"));+ model.vit_merger_ln1_b = get_merger_tensor(string_format(TN_VIT_MERGER_LN1, "bias"));+ model.vit_merger_attn_q_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "weight"));+ model.vit_merger_attn_q_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "bias"), false);+ model.vit_merger_attn_k_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_K, "weight"));+ model.vit_merger_attn_k_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_K, "bias"), false);+ model.vit_merger_attn_v_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_V, "weight"));+ model.vit_merger_attn_v_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_V, "bias"), false);+ model.vit_merger_attn_o_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_O, "weight"));+ model.vit_merger_attn_o_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_O, "bias"), false);// ViT merger: MLP downsample- model.vit_merger_ds_ln_w = get_tensor(string_format(TN_VIT_MERGER_DS_LN, "weight"));- model.vit_merger_ds_ln_b = get_tensor(string_format(TN_VIT_MERGER_DS_LN, "bias"));- model.vit_merger_ds_up_w = get_tensor(string_format(TN_VIT_MERGER_DS_UP, "weight"));- model.vit_merger_ds_up_b = get_tensor(string_format(TN_VIT_MERGER_DS_UP, "bias"), false);- model.vit_merger_ds_down_w = get_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "weight"));- model.vit_merger_ds_down_b = get_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "bias"), false);+ model.vit_merger_ds_ln_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_LN, "weight"));+ model.vit_merger_ds_ln_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_LN, "bias"));+ model.vit_merger_ds_up_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_UP, "weight"));+ model.vit_merger_ds_up_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_UP, "bias"), false);+ model.vit_merger_ds_down_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "weight"));+ model.vit_merger_ds_down_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "bias"), false);// Final Merger (DownsampleMLP)model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B, false);@@ -3591,8 +3597,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {} break;case PROJECTOR_TYPE_MINICPMV4_6:{- // ViT merger 4x + final merger 4x = 16x total spatial downsample- n_patches = n_patches / 16;+ n_patches /= params.n_merge * params.n_merge;} break;case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL:@@ -3974,6 +3979,8 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32} break;case PROJECTOR_TYPE_MINICPMV4_6:{+ const bool is_4x = hparams.n_merge == 2;+// SigLIP position buckets (same as resampler path)std::vector<int32_t> positions(pos_h * pos_w);int bucket_coords_h[1024];@@ -3994,40 +4001,6 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32const int half_h = pos_h / 2;const int half_w = pos_w / 2;- // window reorder indices for 2x2 windows- std::vector<int32_t> window_idx(n_pos);- std::vector<int32_t> inv_window_idx(n_pos);- {- int k = 0;- for (int wi = 0; wi < half_h; wi++) {- for (int wj = 0; wj < half_w; wj++) {- window_idx[k++] = (2*wi ) * pos_w + (2*wj );- window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1);- window_idx[k++] = (2*wi + 1) * pos_w + (2*wj );- window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1);- }- }- for (int i = 0; i < n_pos; i++) {- inv_window_idx[window_idx[i]] = i;- }- }- set_input_i32("vit_merger_window_idx", window_idx);- set_input_i32("vit_merger_inv_window_idx", inv_window_idx);-- // block-diagonal attention mask: tokens in the same 4-token- // window attend to each other (mask = 0), all other positions- // are masked out (-inf). matches the window-major reorder above.- std::vector<float> window_mask_data(n_pos * n_pos, std::numeric_limits<float>::lowest());- for (int wi = 0; wi < n_pos / 4; wi++) {- for (int i = 0; i < 4; i++) {- for (int j = 0; j < 4; j++) {- window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f;- }- }- }- set_input_f32("vit_merger_window_mask", window_mask_data);-- // ViT merger 2x2 downsample indicesauto make_ds_idx = [](int off_r, int off_c, int ds_h, int ds_w, int stride_w) {std::vector<int32_t> idx(ds_h * ds_w);for (int i = 0; i < ds_h; i++) {@@ -4037,22 +4010,58 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32}return idx;};- auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w);- auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w);- auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w);- auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w);- set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0);- set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1);- set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2);- set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3);-- // final merger 2x2 downsample indices (operates on half_h x half_w grid)- const int qh = half_h / 2;- const int qw = half_w / 2;- auto m_ds_0 = make_ds_idx(0, 0, qh, qw, half_w);- auto m_ds_1 = make_ds_idx(0, 1, qh, qw, half_w);- auto m_ds_2 = make_ds_idx(1, 0, qh, qw, half_w);- auto m_ds_3 = make_ds_idx(1, 1, qh, qw, half_w);++ if (!is_4x) {+ // window reorder indices for 2x2 windows+ std::vector<int32_t> window_idx(n_pos);+ std::vector<int32_t> inv_window_idx(n_pos);+ {+ int k = 0;+ for (int wi = 0; wi < half_h; wi++) {+ for (int wj = 0; wj < half_w; wj++) {+ window_idx[k++] = (2*wi ) * pos_w + (2*wj );+ window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1);+ window_idx[k++] = (2*wi + 1) * pos_w + (2*wj );+ window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1);+ }+ }+ for (int i = 0; i < n_pos; i++) {+ inv_window_idx[window_idx[i]] = i;+ }+ }+ set_input_i32("vit_merger_window_idx", window_idx);+ set_input_i32("vit_merger_inv_window_idx", inv_window_idx);++ // block-diagonal attention mask: tokens in the same 4-token+ // window attend to each other (mask = 0), all other positions+ // are masked out (-inf). matches the window-major reorder above.+ std::vector<float> window_mask_data(n_pos * n_pos, std::numeric_limits<float>::lowest());+ for (int wi = 0; wi < n_pos / 4; wi++) {+ for (int i = 0; i < 4; i++) {+ for (int j = 0; j < 4; j++) {+ window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f;+ }+ }+ }+ set_input_f32("vit_merger_window_mask", window_mask_data);++ // ViT merger 2x2 downsample indices+ auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w);+ auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w);+ auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w);+ auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w);+ set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0);+ set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1);+ set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2);+ set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3);+ }++ const int merger_h = is_4x ? pos_h : half_h;+ const int merger_w = is_4x ? pos_w : half_w;+ auto m_ds_0 = make_ds_idx(0, 0, merger_h / 2, merger_w / 2, merger_w);+ auto m_ds_1 = make_ds_idx(0, 1, merger_h / 2, merger_w / 2, merger_w);+ auto m_ds_2 = make_ds_idx(1, 0, merger_h / 2, merger_w / 2, merger_w);+ auto m_ds_3 = make_ds_idx(1, 1, merger_h / 2, merger_w / 2, merger_w);set_input_i32("merger_ds_idx_0", m_ds_0);set_input_i32("merger_ds_idx_1", m_ds_1);set_input_i32("merger_ds_idx_2", m_ds_2);diff --git a/tools/mtmd/models/minicpmv.cpp b/tools/mtmd/models/minicpmv.cppindex bac087ffdf..3e9c4c2a11 100644--- a/tools/mtmd/models/minicpmv.cpp+++ b/tools/mtmd/models/minicpmv.cpp@@ -114,14 +114,12 @@ ggml_cgraph * clip_graph_minicpmv::build() {}ggml_cgraph * clip_graph_minicpmv4_6::build() {- const int insert_lid = hparams.insert_layer_id;- const int n_pos = n_patches;- const int half_h = n_patches_y / 2;- const int half_w = n_patches_x / 2;- const int n_ds = half_h * half_w; // after ViT merger 2x2 downsample- const int qh = half_h / 2;- const int qw = half_w / 2;- const int n_ds2 = qh * qw; // after final merger 2x2 downsample+ const bool is_4x = hparams.n_merge == 2;+ const int n_pos = n_patches;+ const int half_h = n_patches_y / 2;+ const int half_w = n_patches_x / 2;+ const int n_ds = half_h * half_w;+ const int n_out = is_4x ? n_ds : (half_h / 2) * (half_w / 2);auto add_i32_input = [&](const char * name, int n) {ggml_tensor * t = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n);@@ -134,29 +132,39 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() {ggml_tensor * positions = add_i32_input("positions", n_pos);ggml_tensor * learned_pos_embd = ggml_get_rows(ctx0, model.position_embeddings, positions);- // ViT merger window reorder indices + block-diagonal mask- // (mask layout follows qwen2vl: -inf except for 4x4 blocks on the diagonal,- // so each window-major group of 4 tokens only attends to itself)- ggml_tensor * vit_merger_window_idx = add_i32_input("vit_merger_window_idx", n_pos);- ggml_tensor * vit_merger_inv_window_idx = add_i32_input("vit_merger_inv_window_idx", n_pos);- ggml_tensor * vit_merger_window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);- ggml_set_name(vit_merger_window_mask, "vit_merger_window_mask");- ggml_set_input(vit_merger_window_mask);- if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {- vit_merger_window_mask = ggml_cast(ctx0, vit_merger_window_mask, GGML_TYPE_F16);- }+ ggml_tensor * vit_merger_window_idx = nullptr;+ ggml_tensor * vit_merger_inv_window_idx = nullptr;+ ggml_tensor * vit_merger_window_mask = nullptr;+ ggml_tensor * vit_merger_ds_idx_0 = nullptr;+ ggml_tensor * vit_merger_ds_idx_1 = nullptr;+ ggml_tensor * vit_merger_ds_idx_2 = nullptr;+ ggml_tensor * vit_merger_ds_idx_3 = nullptr;++ if (!is_4x) {+ // ViT merger window reorder indices + block-diagonal mask+ // (mask layout follows qwen2vl: -inf except for 4x4 blocks on the diagonal,+ // so each window-major group of 4 tokens only attends to itself)+ vit_merger_window_idx = add_i32_input("vit_merger_window_idx", n_pos);+ vit_merger_inv_window_idx = add_i32_input("vit_merger_inv_window_idx", n_pos);+ vit_merger_window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);+ ggml_set_name(vit_merger_window_mask, "vit_merger_window_mask");+ ggml_set_input(vit_merger_window_mask);+ if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {+ vit_merger_window_mask = ggml_cast(ctx0, vit_merger_window_mask, GGML_TYPE_F16);+ }- // ViT merger 2x2 downsample gather indices- ggml_tensor * vit_merger_ds_idx_0 = add_i32_input("vit_merger_ds_idx_0", n_ds);- ggml_tensor * vit_merger_ds_idx_1 = add_i32_input("vit_merger_ds_idx_1", n_ds);- ggml_tensor * vit_merger_ds_idx_2 = add_i32_input("vit_merger_ds_idx_2", n_ds);- ggml_tensor * vit_merger_ds_idx_3 = add_i32_input("vit_merger_ds_idx_3", n_ds);+ // ViT merger 2x2 downsample gather indices+ vit_merger_ds_idx_0 = add_i32_input("vit_merger_ds_idx_0", n_ds);+ vit_merger_ds_idx_1 = add_i32_input("vit_merger_ds_idx_1", n_ds);+ vit_merger_ds_idx_2 = add_i32_input("vit_merger_ds_idx_2", n_ds);+ vit_merger_ds_idx_3 = add_i32_input("vit_merger_ds_idx_3", n_ds);+ }// final merger 2x2 downsample gather indices- ggml_tensor * merger_ds_idx_0 = add_i32_input("merger_ds_idx_0", n_ds2);- ggml_tensor * merger_ds_idx_1 = add_i32_input("merger_ds_idx_1", n_ds2);- ggml_tensor * merger_ds_idx_2 = add_i32_input("merger_ds_idx_2", n_ds2);- ggml_tensor * merger_ds_idx_3 = add_i32_input("merger_ds_idx_3", n_ds2);+ ggml_tensor * merger_ds_idx_0 = add_i32_input("merger_ds_idx_0", n_out);+ ggml_tensor * merger_ds_idx_1 = add_i32_input("merger_ds_idx_1", n_out);+ ggml_tensor * merger_ds_idx_2 = add_i32_input("merger_ds_idx_2", n_out);+ ggml_tensor * merger_ds_idx_3 = add_i32_input("merger_ds_idx_3", n_out);// patch embedding + positional embeddingggml_tensor * inp = build_inp();@@ -169,150 +177,10 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() {cb(inpL, "pre_ln", -1);}- // ViT layers 0..insert_layer_id (inclusive)- // Mirrors the separate-qkv path of clip_graph::build_vit so the two manually- // unrolled segments around the ViT merger read like build_vit() expansions.- for (int il = 0; il <= insert_lid; il++) {- auto & layer = model.layers[il];- ggml_tensor * cur = inpL;-- cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);- cb(cur, "layer_inp_normed", il);-- {- ggml_tensor * Qcur = build_mm(layer.q_w, cur);- if (layer.q_b) {- Qcur = ggml_add(ctx0, Qcur, layer.q_b);- }- ggml_tensor * Kcur = build_mm(layer.k_w, cur);- if (layer.k_b) {- Kcur = ggml_add(ctx0, Kcur, layer.k_b);- }- ggml_tensor * Vcur = build_mm(layer.v_w, cur);- if (layer.v_b) {- Vcur = ggml_add(ctx0, Vcur, layer.v_b);- }-- Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);- Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);- Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);- cb(Qcur, "Qcur", il);- cb(Kcur, "Kcur", il);- cb(Vcur, "Vcur", il);-- cur = build_attn(layer.o_w, layer.o_b, Qcur, Kcur, Vcur, nullptr, kq_scale, il);- cb(cur, "attn_out", il);- }-- if (layer.ls_1_w) {- cur = ggml_mul(ctx0, cur, layer.ls_1_w);- cb(cur, "attn_out_scaled", il);- }- cur = ggml_add(ctx0, cur, inpL);- inpL = cur;- cb(cur, "ffn_inp", il);-- cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);- cb(cur, "ffn_inp_normed", il);-- cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, layer.ff_gate_w, layer.ff_gate_b,- layer.ff_down_w, layer.ff_down_b, hparams.ffn_op, il);- cb(cur, "ffn_out", il);-- if (layer.ls_2_w) {- cur = ggml_mul(ctx0, cur, layer.ls_2_w);- cb(cur, "ffn_out_scaled", il);- }- cur = ggml_add(ctx0, inpL, cur);- cb(cur, "layer_out", il);-- inpL = cur;- }-- // ViT merger: window self-attention- // Tokens are reordered to window-major (4 tokens per window are contiguous),- // and a block-diagonal mask restricts attention to within each window. This- // mirrors the qwen2vl windowed-attention pattern so build_attn() can pick the- // flash-attention path when available.- {- ggml_tensor * residual = inpL;- ggml_tensor * cur = build_norm(inpL,- model.vit_merger_ln1_w, model.vit_merger_ln1_b,- NORM_TYPE_NORMAL, eps, -1);- cb(cur, "vit_merger_attn_inp_normed", -1);-- cur = ggml_get_rows(ctx0, cur, vit_merger_window_idx);- cb(cur, "vit_merger_window_reorder", -1);-- ggml_tensor * Qcur = build_mm(model.vit_merger_attn_q_w, cur);- if (model.vit_merger_attn_q_b) {- Qcur = ggml_add(ctx0, Qcur, model.vit_merger_attn_q_b);- }- ggml_tensor * Kcur = build_mm(model.vit_merger_attn_k_w, cur);- if (model.vit_merger_attn_k_b) {- Kcur = ggml_add(ctx0, Kcur, model.vit_merger_attn_k_b);- }- ggml_tensor * Vcur = build_mm(model.vit_merger_attn_v_w, cur);- if (model.vit_merger_attn_v_b) {- Vcur = ggml_add(ctx0, Vcur, model.vit_merger_attn_v_b);- }-- Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);- Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);- Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);- cb(Qcur, "vit_merger_Qcur", -1);- cb(Kcur, "vit_merger_Kcur", -1);- cb(Vcur, "vit_merger_Vcur", -1);-- cur = build_attn(model.vit_merger_attn_o_w, model.vit_merger_attn_o_b,- Qcur, Kcur, Vcur, vit_merger_window_mask, kq_scale, -1);- cb(cur, "vit_merger_attn_out", -1);-- cur = ggml_get_rows(ctx0, cur, vit_merger_inv_window_idx);- inpL = ggml_add(ctx0, cur, residual);- cb(inpL, "vit_merger_attn_residual", -1);- }-- // ViT merger: 2x2 spatial downsample + MLP (4 tokens -> 1)- {- ggml_tensor * p0 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_0);- ggml_tensor * p1 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_1);- ggml_tensor * p2 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_2);- ggml_tensor * p3 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_3);-- ggml_tensor * mean_res = ggml_add(ctx0, p0, p1);- mean_res = ggml_add(ctx0, mean_res, p2);- mean_res = ggml_add(ctx0, mean_res, p3);- mean_res = ggml_scale(ctx0, mean_res, 0.25f);- cb(mean_res, "vit_merger_ds_mean_res", -1);-- ggml_tensor * cat = ggml_concat(ctx0, p0, p1, 0);- cat = ggml_concat(ctx0, cat, p2, 0);- cat = ggml_concat(ctx0, cat, p3, 0);-- ggml_tensor * cur = build_norm(cat,- model.vit_merger_ds_ln_w, model.vit_merger_ds_ln_b,- NORM_TYPE_NORMAL, eps, -1);- cb(cur, "vit_merger_ds_normed", -1);-- // ViTWindowAttentionMerger downsample MLP uses gelu_pytorch_tanh (FFN_GELU)- cur = build_ffn(cur,- model.vit_merger_ds_up_w, model.vit_merger_ds_up_b,- nullptr, nullptr,- model.vit_merger_ds_down_w, model.vit_merger_ds_down_b,- FFN_GELU, -1);- cb(cur, "vit_merger_ds_mlp_out", -1);-- inpL = ggml_add(ctx0, cur, mean_res);- cb(inpL, "vit_merger_ds_out", -1);- }-- // ViT layers (insert_layer_id+1)..n_layer-1, operating on the downsampled tokens- {- const int64_t n_pos_ds = n_ds;- for (int il = insert_lid + 1; il < n_layer; il++) {+ auto build_vit_layers = [&](ggml_tensor * input, int il_begin, int il_end, int64_t n_pos_layer) {+ for (int il = il_begin; il < il_end; il++) {auto & layer = model.layers[il];- ggml_tensor * cur = inpL;+ ggml_tensor * cur = input;cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);cb(cur, "layer_inp_normed", il);@@ -331,9 +199,9 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() {Vcur = ggml_add(ctx0, Vcur, layer.v_b);}- Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos_ds);- Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos_ds);- Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos_ds);+ Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos_layer);+ Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos_layer);+ Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos_layer);cb(Qcur, "Qcur", il);cb(Kcur, "Kcur", il);cb(Vcur, "Vcur", il);@@ -346,8 +214,8 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() {cur = ggml_mul(ctx0, cur, layer.ls_1_w);cb(cur, "attn_out_scaled", il);}- cur = ggml_add(ctx0, cur, inpL);- inpL = cur;+ cur = ggml_add(ctx0, cur, input);+ input = cur;cb(cur, "ffn_inp", il);cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);@@ -361,11 +229,98 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() {cur = ggml_mul(ctx0, cur, layer.ls_2_w);cb(cur, "ffn_out_scaled", il);}- cur = ggml_add(ctx0, inpL, cur);- cb(cur, "layer_out", il);+ input = ggml_add(ctx0, input, cur);+ cb(input, "layer_out", il);+ }+ return input;+ };++ if (!is_4x) {+ const int insert_lid = hparams.insert_layer_id;++ inpL = build_vit_layers(inpL, 0, insert_lid + 1, n_pos);++ // ViT merger: window self-attention+ // Tokens are reordered to window-major (4 tokens per window are contiguous),+ // and a block-diagonal mask restricts attention to within each window. This+ // mirrors the qwen2vl windowed-attention pattern so build_attn() can pick the+ // flash-attention path when available.+ {+ ggml_tensor * residual = inpL;+ ggml_tensor * cur = build_norm(inpL,+ model.vit_merger_ln1_w, model.vit_merger_ln1_b,+ NORM_TYPE_NORMAL, eps, -1);+ cb(cur, "vit_merger_attn_inp_normed", -1);++ cur = ggml_get_rows(ctx0, cur, vit_merger_window_idx);+ cb(cur, "vit_merger_window_reorder", -1);++ ggml_tensor * Qcur = build_mm(model.vit_merger_attn_q_w, cur);+ if (model.vit_merger_attn_q_b) {+ Qcur = ggml_add(ctx0, Qcur, model.vit_merger_attn_q_b);+ }+ ggml_tensor * Kcur = build_mm(model.vit_merger_attn_k_w, cur);+ if (model.vit_merger_attn_k_b) {+ Kcur = ggml_add(ctx0, Kcur, model.vit_merger_attn_k_b);+ }+ ggml_tensor * Vcur = build_mm(model.vit_merger_attn_v_w, cur);+ if (model.vit_merger_attn_v_b) {+ Vcur = ggml_add(ctx0, Vcur, model.vit_merger_attn_v_b);+ }- inpL = cur;+ Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);+ Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);+ Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);+ cb(Qcur, "vit_merger_Qcur", -1);+ cb(Kcur, "vit_merger_Kcur", -1);+ cb(Vcur, "vit_merger_Vcur", -1);++ cur = build_attn(model.vit_merger_attn_o_w, model.vit_merger_attn_o_b,+ Qcur, Kcur, Vcur, vit_merger_window_mask, kq_scale, -1);+ cb(cur, "vit_merger_attn_out", -1);++ cur = ggml_get_rows(ctx0, cur, vit_merger_inv_window_idx);+ inpL = ggml_add(ctx0, cur, residual);+ cb(inpL, "vit_merger_attn_residual", -1);}++ // ViT merger: 2x2 spatial downsample + MLP (4 tokens -> 1)+ {+ ggml_tensor * p0 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_0);+ ggml_tensor * p1 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_1);+ ggml_tensor * p2 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_2);+ ggml_tensor * p3 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_3);++ ggml_tensor * mean_res = ggml_add(ctx0, p0, p1);+ mean_res = ggml_add(ctx0, mean_res, p2);+ mean_res = ggml_add(ctx0, mean_res, p3);+ mean_res = ggml_scale(ctx0, mean_res, 0.25f);+ cb(mean_res, "vit_merger_ds_mean_res", -1);++ ggml_tensor * cat = ggml_concat(ctx0, p0, p1, 0);+ cat = ggml_concat(ctx0, cat, p2, 0);+ cat = ggml_concat(ctx0, cat, p3, 0);++ ggml_tensor * cur = build_norm(cat,+ model.vit_merger_ds_ln_w, model.vit_merger_ds_ln_b,+ NORM_TYPE_NORMAL, eps, -1);+ cb(cur, "vit_merger_ds_normed", -1);++ // ViTWindowAttentionMerger downsample MLP uses gelu_pytorch_tanh (FFN_GELU)+ cur = build_ffn(cur,+ model.vit_merger_ds_up_w, model.vit_merger_ds_up_b,+ nullptr, nullptr,+ model.vit_merger_ds_down_w, model.vit_merger_ds_down_b,+ FFN_GELU, -1);+ cb(cur, "vit_merger_ds_mlp_out", -1);++ inpL = ggml_add(ctx0, cur, mean_res);+ cb(inpL, "vit_merger_ds_out", -1);+ }++ inpL = build_vit_layers(inpL, insert_lid + 1, n_layer, n_ds);+ } else {+ inpL = build_vit_layers(inpL, 0, n_layer, n_pos);}if (model.post_ln_w) {diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cppindex 72d35fce69..10cfe52f56 100644--- a/tools/mtmd/mtmd-image.cpp+++ b/tools/mtmd/mtmd-image.cpp@@ -972,6 +972,26 @@ mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const clreturn output;}+//+// mtmd_image_preprocessor_minicpmv+//++mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_minicpmv::get_slice_instructions(const clip_image_size & original_size) {+ if (hparams.n_merge == 2) {+ const int slice_size = hparams.image_size;+ const float ratio = (float)original_size.width * original_size.height / (slice_size * slice_size);+ if (ratio <= 1.0f) {+ mtmd_image_preprocessor_llava_uhd::slice_instructions inst;+ const int patch_size = hparams.patch_size * hparams.n_merge;+ inst.overview_size = get_best_resize(original_size, slice_size, patch_size, true);+ inst.refined_size = clip_image_size{0, 0};+ inst.grid_size = clip_image_size{0, 0};+ return inst;+ }+ }+ return mtmd_image_preprocessor_llava_uhd::get_slice_instructions(original_size);+}+//// mtmd_image_preprocessor_lfm2//diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.hindex 115cba51e8..ecb203f767 100644--- a/tools/mtmd/mtmd-image.h+++ b/tools/mtmd/mtmd-image.h@@ -74,7 +74,6 @@ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor {std::vector<slice_coordinates> slices;};- // LFM2 override this function to implement its custom slicing logicvirtual slice_instructions get_slice_instructions(const clip_image_size & original_size);struct slice_output {@@ -83,9 +82,10 @@ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor {};slice_output slice_image(const clip_image_u8 & img, const slice_instructions & inst);-private:+protected:clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false);+private:clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max);/**@@ -129,6 +129,12 @@ struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor {mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;};+// custom llava-uhd slicing logic for MiniCPM-V+struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd {+ using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;+ slice_instructions get_slice_instructions(const clip_image_size & original_size) override;+};+// custom llava-uhd slicing logic for LFM2// ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.pystruct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cppindex 93ca8cbcf8..d3899f5c85 100644--- a/tools/mtmd/mtmd.cpp+++ b/tools/mtmd/mtmd.cpp@@ -451,7 +451,7 @@ struct mtmd_context {tok_row_end = {lookup_token("\n")};tok_row_end_trail = false; // no trailing end-of-row tokenov_img_first = true;- image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);+ image_preproc = std::make_unique<mtmd_image_preprocessor_minicpmv>(ctx_v);} break;case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL: