LLM inference in C/C++
C++ ggml
mtmd: add GLM-5.2-Vision (#26126)
Eric Hartfordaf285020e909
diff --git a/conversion/__init__.py b/conversion/__init__.pyindex 0b08e6e570..c5ecc68cfd 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -269,6 +269,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {"Gemma4UnifiedForConditionalGeneration": "gemma","Glm4vForConditionalGeneration": "qwen3vl","Glm4vMoeForConditionalGeneration": "qwen3vl",+ "Glm5vForConditionalGeneration": "kimivl","GlmOcrForConditionalGeneration": "qwen3vl","GlmasrModel": "ultravox","Granite4VisionForConditionalGeneration": "granite",diff --git a/conversion/kimivl.py b/conversion/kimivl.pyindex 63b8a079b7..5ff3c39ca9 100644--- a/conversion/kimivl.py+++ b/conversion/kimivl.py@@ -152,3 +152,19 @@ class KimiK25Model(MmprojModel):name = name.replace(".proj.2.", ".proj.linear_2.")yield from super().modify_tensors(data_torch, name, bid)++[email protected]("Glm5vForConditionalGeneration")+class Glm5vModel(KimiK25Model):+ """GLM-5.2-Vision MoonViT3d encoder and projector++ Uses the same vision encoder and projector as Kimi-K2.5, so it reuses the+ kimik25 projector type. The image begin/end tokens differ, but they are+ resolved at runtime from the text model vocab.+ """++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ if name.startswith("mm_projector.linear_"):+ name = name.replace("mm_projector.linear_", "mm_projector.proj.linear_", 1)++ yield from super().modify_tensors(data_torch, name, bid)diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cppindex e10ccf186b..5915b4cba9 100644--- a/tools/mtmd/mtmd.cpp+++ b/tools/mtmd/mtmd.cpp@@ -555,9 +555,17 @@ struct mtmd_context {} break;case PROJECTOR_TYPE_KIMIK25:{- // <|media_begin|> ... (image embeddings) ... <|media_end|>- img_beg = "<|media_begin|>";- img_end = "<|media_end|>";+ // GLM-5.2-V reuses the Kimi-K2.5 vision encoder and projector, but marks+ // images with its own tokens, so decide based on the text model vocab+ if (lookup_token("<|begin_of_image|>") != LLAMA_TOKEN_NULL) {+ // <|begin_of_image|> ... (image embeddings) ... <|end_of_image|>+ img_beg = "<|begin_of_image|>";+ img_end = "<|end_of_image|>";+ } else {+ // <|media_begin|> ... (image embeddings) ... <|media_end|>+ img_beg = "<|media_begin|>";+ img_end = "<|media_end|>";+ }image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);} break;case PROJECTOR_TYPE_LIGHTONOCR: