Skip to content

LLM inference in C/C++

C++ ggml

mtmd: add GLM-5.2-Vision (#26126)

Eric Hartfordaf285020e909
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 0b08e6e570..c5ecc68cfd 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -269,6 +269,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"Gemma4UnifiedForConditionalGeneration": "gemma",
"Glm4vForConditionalGeneration": "qwen3vl",
"Glm4vMoeForConditionalGeneration": "qwen3vl",
+ "Glm5vForConditionalGeneration": "kimivl",
"GlmOcrForConditionalGeneration": "qwen3vl",
"GlmasrModel": "ultravox",
"Granite4VisionForConditionalGeneration": "granite",
diff --git a/conversion/kimivl.py b/conversion/kimivl.py
index 63b8a079b7..5ff3c39ca9 100644
--- a/conversion/kimivl.py
+++ b/conversion/kimivl.py
@@ -152,3 +152,19 @@ class KimiK25Model(MmprojModel):
name = name.replace(".proj.2.", ".proj.linear_2.")
yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("Glm5vForConditionalGeneration")
+class Glm5vModel(KimiK25Model):
+ """GLM-5.2-Vision MoonViT3d encoder and projector
+
+ Uses the same vision encoder and projector as Kimi-K2.5, so it reuses the
+ kimik25 projector type. The image begin/end tokens differ, but they are
+ resolved at runtime from the text model vocab.
+ """
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name.startswith("mm_projector.linear_"):
+ name = name.replace("mm_projector.linear_", "mm_projector.proj.linear_", 1)
+
+ yield from super().modify_tensors(data_torch, name, bid)
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index e10ccf186b..5915b4cba9 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -555,9 +555,17 @@ struct mtmd_context {
} break;
case PROJECTOR_TYPE_KIMIK25:
{
- // <|media_begin|> ... (image embeddings) ... <|media_end|>
- img_beg = "<|media_begin|>";
- img_end = "<|media_end|>";
+ // GLM-5.2-V reuses the Kimi-K2.5 vision encoder and projector, but marks
+ // images with its own tokens, so decide based on the text model vocab
+ if (lookup_token("<|begin_of_image|>") != LLAMA_TOKEN_NULL) {
+ // <|begin_of_image|> ... (image embeddings) ... <|end_of_image|>
+ img_beg = "<|begin_of_image|>";
+ img_end = "<|end_of_image|>";
+ } else {
+ // <|media_begin|> ... (image embeddings) ... <|media_end|>
+ img_beg = "<|media_begin|>";
+ img_end = "<|media_end|>";
+ }
image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
} break;
case PROJECTOR_TYPE_LIGHTONOCR: