Skip to content

LLM inference in C/C++

C++ ggml

mtmd : add MiniCPM-OCR 1.0 support (#30979)

tc-mba08ab75ae74c
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 8921f53e44..5f252b490d 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -196,6 +196,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"MiniCPMForCausalLM": "minicpm",
"MiniCPMV4_6ForConditionalGeneration": "minicpm",
"MiniCPMV4_7ForConditionalGeneration": "minicpm",
+ "MiniCPMOcrForConditionalGeneration": "minicpm",
"MiniMaxText01ForCausalLM": "minimax",
"MiniMaxM1ForCausalLM": "minimax",
"MiniMaxM2ForCausalLM": "minimax",
@@ -361,6 +362,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"MiniMaxM3SparseForConditionalGeneration": "minimax",
"MiniCPMV4_6ForConditionalGeneration": "minicpm",
"MiniCPMV4_7ForConditionalGeneration": "minicpm",
+ "MiniCPMOcrForConditionalGeneration": "minicpm",
"Mistral3ForConditionalGeneration": "llava",
"Mistral3Model": "llava",
"NemotronH_Omni_Reasoning_V3": "nemotron",
diff --git a/conversion/minicpm.py b/conversion/minicpm.py
index 937ac033a9..d8fceb0c08 100644
--- a/conversion/minicpm.py
+++ b/conversion/minicpm.py
@@ -272,3 +272,18 @@ class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel):
yield self.format_tensor_name(gguf.MODEL_TENSOR.V_TOK_EMBD_SEP, suffix=""), data_torch[ids]
return
yield from super().modify_tensors(data_torch, name, bid)
+
+
+# MiniCPM-OCR 1.0: the 4.6 stack with model_type minicpm_ocr.
+
[email protected]("MiniCPMOcrForConditionalGeneration")
[email protected]("openbmb/MiniCPM-OCR-1.0")
+class MiniCPMOCR1_0TextModel(MiniCPMV4_6TextModel):
+ model_arch = gguf.MODEL_ARCH.QWEN35
+ no_mtp = True
+
+
[email protected]("MiniCPMOcrForConditionalGeneration")
[email protected]("openbmb/MiniCPM-OCR-1.0")
+class MiniCPMOCR1_0VisionModel(MiniCPMV4_6VisionModel):
+ projector_type = gguf.VisionProjectorType.MINICPM_OCR_1_0
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 0b95580bd2..b8cfa3a85e 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -6452,6 +6452,7 @@ class VisionProjectorType:
MINIMAXM3 = "minimax_m3"
MINICPMV4_6 = "minicpmv4_6"
MINICPMV4_7 = "minicpmv4_7"
+ MINICPM_OCR_1_0 = "minicpm_ocr_1_0"
GRANITE_SPEECH = "granite_speech" # audio
MIMOVL = "mimovl"
MIMO_AUDIO = "mimo_audio"
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 6ca4652808..5583a7c574 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -537,6 +537,7 @@ enum projector_type {
PROJECTOR_TYPE_EXAONE4_5,
PROJECTOR_TYPE_MINICPMV4_6,
PROJECTOR_TYPE_MINICPMV4_7,
+ PROJECTOR_TYPE_MINICPM_OCR_1_0,
PROJECTOR_TYPE_GRANITE_SPEECH,
PROJECTOR_TYPE_MIMOVL,
PROJECTOR_TYPE_MINIMAX_M3,
@@ -611,6 +612,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"},
{ PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"},
{ PROJECTOR_TYPE_MINICPMV4_7, "minicpmv4_7"},
+ { PROJECTOR_TYPE_MINICPM_OCR_1_0, "minicpm_ocr_1_0"},
{ PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"},
{ PROJECTOR_TYPE_MIMOVL, "mimovl"},
{ PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"},
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 869e3fff82..5737400df2 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -1031,6 +1031,7 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
builder = std::make_unique<clip_graph_minicpmv4_6>(ctx, img);
} break;
@@ -1496,6 +1497,7 @@ struct clip_model_loader {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
// MiniCPM-V 4.6/4.7 unified merger projector
// ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension
@@ -2527,6 +2529,7 @@ struct clip_model_loader {
|| model.proj_type == PROJECTOR_TYPE_MINICPMV
|| model.proj_type == PROJECTOR_TYPE_MINICPMV4_6
|| model.proj_type == PROJECTOR_TYPE_MINICPMV4_7
+ || model.proj_type == PROJECTOR_TYPE_MINICPM_OCR_1_0
) && layer.ff_up_w && layer.ff_down_w && layer.ff_down_w->ne[0] == hparams.n_embd;
if (is_ffn_swapped) {
// swap up and down weights
@@ -2630,6 +2633,7 @@ struct clip_model_loader {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
const bool merger_required = hparams.n_merge == 4;
auto get_merger_tensor = [&](const std::string & name, bool required = true) {
@@ -4348,6 +4352,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_STEP3VL:
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
return img->nx() / (params.patch_size * params.n_merge);
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
@@ -4379,6 +4384,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_STEP3VL:
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
return img->ny() / (params.patch_size * params.n_merge);
default:
break;
@@ -4445,6 +4451,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
n_patches /= params.n_merge * params.n_merge;
} break;
@@ -5169,6 +5176,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
const bool is_4x = hparams.n_merge == 2;
@@ -6437,6 +6445,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_model_proj->ne[0];
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
return ctx->model.mm_ffn_down_w->ne[1];
case PROJECTOR_TYPE_GLM_EDGE:
return ctx->model.mm_model_mlp_3_w->ne[1];
diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h
index 3aea23ced6..bc90a76c62 100644
--- a/tools/mtmd/mtmd-image.h
+++ b/tools/mtmd/mtmd-image.h
@@ -190,6 +190,17 @@ protected:
}
};
+// MiniCPM-OCR 1.0: find_best_resize aligns to patch_size * n_merge (28 in 4x).
+struct mtmd_image_preprocessor_minicpm_ocr_1_0 : mtmd_image_preprocessor_minicpmv {
+ using mtmd_image_preprocessor_minicpmv::mtmd_image_preprocessor_minicpmv;
+
+protected:
+ int get_slice_align() const override {
+ const int merge = hparams.n_merge > 0 ? hparams.n_merge : 2;
+ return hparams.patch_size * merge;
+ }
+};
+
// custom llava-uhd slicing logic for LFM2
// tiling params (min/max tiles, tile size, pixels tolerance) come from hparams
// ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 9460f15fe2..9a1663015f 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -722,6 +722,7 @@ struct mtmd_context {
} break;
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
+ case PROJECTOR_TYPE_MINICPM_OCR_1_0:
{
slice_tmpl = MTMD_SLICE_TMPL_MINICPMV_2_6;
tok_ov_img_start = {lookup_token("<image>")};
@@ -732,7 +733,11 @@ struct mtmd_context {
tok_row_end_trail = false; // no trailing end-of-row token
ov_img_first = true;
use_image_id = true;
- image_preproc = std::make_unique<mtmd_image_preprocessor_minicpmv>(ctx_v);
+ if (clip_get_projector_type(ctx_v) == PROJECTOR_TYPE_MINICPM_OCR_1_0) {
+ image_preproc = std::make_unique<mtmd_image_preprocessor_minicpm_ocr_1_0>(ctx_v);
+ } else {
+ image_preproc = std::make_unique<mtmd_image_preprocessor_minicpmv>(ctx_v);
+ }
} break;
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL: