LLM inference in C/C++
C++ ggml
mtmd : add MiniCPM-OCR 1.0 support (#30979)
tc-mba08ab75ae74c
diff --git a/conversion/__init__.py b/conversion/__init__.pyindex 8921f53e44..5f252b490d 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -196,6 +196,7 @@ TEXT_MODEL_MAP: dict[str, str] = {"MiniCPMForCausalLM": "minicpm","MiniCPMV4_6ForConditionalGeneration": "minicpm","MiniCPMV4_7ForConditionalGeneration": "minicpm",+ "MiniCPMOcrForConditionalGeneration": "minicpm","MiniMaxText01ForCausalLM": "minimax","MiniMaxM1ForCausalLM": "minimax","MiniMaxM2ForCausalLM": "minimax",@@ -361,6 +362,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {"MiniMaxM3SparseForConditionalGeneration": "minimax","MiniCPMV4_6ForConditionalGeneration": "minicpm","MiniCPMV4_7ForConditionalGeneration": "minicpm",+ "MiniCPMOcrForConditionalGeneration": "minicpm","Mistral3ForConditionalGeneration": "llava","Mistral3Model": "llava","NemotronH_Omni_Reasoning_V3": "nemotron",diff --git a/conversion/minicpm.py b/conversion/minicpm.pyindex 937ac033a9..d8fceb0c08 100644--- a/conversion/minicpm.py+++ b/conversion/minicpm.py@@ -272,3 +272,18 @@ class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel):yield self.format_tensor_name(gguf.MODEL_TENSOR.V_TOK_EMBD_SEP, suffix=""), data_torch[ids]returnyield from super().modify_tensors(data_torch, name, bid)+++# MiniCPM-OCR 1.0: the 4.6 stack with model_type minicpm_ocr.+[email protected]("MiniCPMOcrForConditionalGeneration")[email protected]("openbmb/MiniCPM-OCR-1.0")+class MiniCPMOCR1_0TextModel(MiniCPMV4_6TextModel):+ model_arch = gguf.MODEL_ARCH.QWEN35+ no_mtp = True++[email protected]("MiniCPMOcrForConditionalGeneration")[email protected]("openbmb/MiniCPM-OCR-1.0")+class MiniCPMOCR1_0VisionModel(MiniCPMV4_6VisionModel):+ projector_type = gguf.VisionProjectorType.MINICPM_OCR_1_0diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 0b95580bd2..b8cfa3a85e 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -6452,6 +6452,7 @@ class VisionProjectorType:MINIMAXM3 = "minimax_m3"MINICPMV4_6 = "minicpmv4_6"MINICPMV4_7 = "minicpmv4_7"+ MINICPM_OCR_1_0 = "minicpm_ocr_1_0"GRANITE_SPEECH = "granite_speech" # audioMIMOVL = "mimovl"MIMO_AUDIO = "mimo_audio"diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.hindex 6ca4652808..5583a7c574 100644--- a/tools/mtmd/clip-impl.h+++ b/tools/mtmd/clip-impl.h@@ -537,6 +537,7 @@ enum projector_type {PROJECTOR_TYPE_EXAONE4_5,PROJECTOR_TYPE_MINICPMV4_6,PROJECTOR_TYPE_MINICPMV4_7,+ PROJECTOR_TYPE_MINICPM_OCR_1_0,PROJECTOR_TYPE_GRANITE_SPEECH,PROJECTOR_TYPE_MIMOVL,PROJECTOR_TYPE_MINIMAX_M3,@@ -611,6 +612,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {{ PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"},{ PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"},{ PROJECTOR_TYPE_MINICPMV4_7, "minicpmv4_7"},+ { PROJECTOR_TYPE_MINICPM_OCR_1_0, "minicpm_ocr_1_0"},{ PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"},{ PROJECTOR_TYPE_MIMOVL, "mimovl"},{ PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"},diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex 869e3fff82..5737400df2 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -1031,6 +1031,7 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const} break;case PROJECTOR_TYPE_MINICPMV4_6:case PROJECTOR_TYPE_MINICPMV4_7:+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:{builder = std::make_unique<clip_graph_minicpmv4_6>(ctx, img);} break;@@ -1496,6 +1497,7 @@ struct clip_model_loader {} break;case PROJECTOR_TYPE_MINICPMV4_6:case PROJECTOR_TYPE_MINICPMV4_7:+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:{// MiniCPM-V 4.6/4.7 unified merger projector// ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension@@ -2527,6 +2529,7 @@ struct clip_model_loader {|| model.proj_type == PROJECTOR_TYPE_MINICPMV|| model.proj_type == PROJECTOR_TYPE_MINICPMV4_6|| model.proj_type == PROJECTOR_TYPE_MINICPMV4_7+ || model.proj_type == PROJECTOR_TYPE_MINICPM_OCR_1_0) && layer.ff_up_w && layer.ff_down_w && layer.ff_down_w->ne[0] == hparams.n_embd;if (is_ffn_swapped) {// swap up and down weights@@ -2630,6 +2633,7 @@ struct clip_model_loader {} break;case PROJECTOR_TYPE_MINICPMV4_6:case PROJECTOR_TYPE_MINICPMV4_7:+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:{const bool merger_required = hparams.n_merge == 4;auto get_merger_tensor = [&](const std::string & name, bool required = true) {@@ -4348,6 +4352,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {case PROJECTOR_TYPE_STEP3VL:case PROJECTOR_TYPE_MINICPMV4_6:case PROJECTOR_TYPE_MINICPMV4_7:+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:return img->nx() / (params.patch_size * params.n_merge);case PROJECTOR_TYPE_DEEPSEEKOCR:case PROJECTOR_TYPE_DEEPSEEKOCR2:@@ -4379,6 +4384,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {case PROJECTOR_TYPE_STEP3VL:case PROJECTOR_TYPE_MINICPMV4_6:case PROJECTOR_TYPE_MINICPMV4_7:+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:return img->ny() / (params.patch_size * params.n_merge);default:break;@@ -4445,6 +4451,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {} break;case PROJECTOR_TYPE_MINICPMV4_6:case PROJECTOR_TYPE_MINICPMV4_7:+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:{n_patches /= params.n_merge * params.n_merge;} break;@@ -5169,6 +5176,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {} break;case PROJECTOR_TYPE_MINICPMV4_6:case PROJECTOR_TYPE_MINICPMV4_7:+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:{const bool is_4x = hparams.n_merge == 2;@@ -6437,6 +6445,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {return ctx->model.mm_model_proj->ne[0];case PROJECTOR_TYPE_MINICPMV4_6:case PROJECTOR_TYPE_MINICPMV4_7:+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:return ctx->model.mm_ffn_down_w->ne[1];case PROJECTOR_TYPE_GLM_EDGE:return ctx->model.mm_model_mlp_3_w->ne[1];diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.hindex 3aea23ced6..bc90a76c62 100644--- a/tools/mtmd/mtmd-image.h+++ b/tools/mtmd/mtmd-image.h@@ -190,6 +190,17 @@ protected:}};+// MiniCPM-OCR 1.0: find_best_resize aligns to patch_size * n_merge (28 in 4x).+struct mtmd_image_preprocessor_minicpm_ocr_1_0 : mtmd_image_preprocessor_minicpmv {+ using mtmd_image_preprocessor_minicpmv::mtmd_image_preprocessor_minicpmv;++protected:+ int get_slice_align() const override {+ const int merge = hparams.n_merge > 0 ? hparams.n_merge : 2;+ return hparams.patch_size * merge;+ }+};+// custom llava-uhd slicing logic for LFM2// tiling params (min/max tiles, tile size, pixels tolerance) come from hparams// ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.pydiff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cppindex 9460f15fe2..9a1663015f 100644--- a/tools/mtmd/mtmd.cpp+++ b/tools/mtmd/mtmd.cpp@@ -722,6 +722,7 @@ struct mtmd_context {} break;case PROJECTOR_TYPE_MINICPMV4_6:case PROJECTOR_TYPE_MINICPMV4_7:+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:{slice_tmpl = MTMD_SLICE_TMPL_MINICPMV_2_6;tok_ov_img_start = {lookup_token("<image>")};@@ -732,7 +733,11 @@ struct mtmd_context {tok_row_end_trail = false; // no trailing end-of-row tokenov_img_first = true;use_image_id = true;- image_preproc = std::make_unique<mtmd_image_preprocessor_minicpmv>(ctx_v);+ if (clip_get_projector_type(ctx_v) == PROJECTOR_TYPE_MINICPM_OCR_1_0) {+ image_preproc = std::make_unique<mtmd_image_preprocessor_minicpm_ocr_1_0>(ctx_v);+ } else {+ image_preproc = std::make_unique<mtmd_image_preprocessor_minicpmv>(ctx_v);+ }} break;case PROJECTOR_TYPE_QWEN2VL:case PROJECTOR_TYPE_QWEN25VL: