Skip to content

LLM inference in C/C++

C++ ggml

mtmd: Add Vision Support for Minimax-M3 (#25113)

timkhronos3d1c3a8975f9
diff --git a/conversion/__init__.py b/conversion/__init__.py
index c5ecc68cfd..b2bb7e5161 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -288,6 +288,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"LlavaForConditionalGeneration": "llava",
"MERaLiON2ForConditionalGeneration": "ultravox",
"MiMoV2ForCausalLM": "mimo",
+ "MiniMaxM3SparseForConditionalGeneration": "minimax",
"MiniCPMV4_6ForConditionalGeneration": "minicpm",
"Mistral3ForConditionalGeneration": "llava",
"NemotronH_Nano_VL_V2": "nemotron",
diff --git a/conversion/minimax.py b/conversion/minimax.py
index e82e393a38..c2175cc932 100644
--- a/conversion/minimax.py
+++ b/conversion/minimax.py
@@ -7,7 +7,7 @@ import torch
if TYPE_CHECKING:
from torch import Tensor
-from .base import ModelBase, TextModel, gguf
+from .base import ModelBase, TextModel, MmprojModel, gguf
@ModelBase.register("MiniMaxM2ForCausalLM")
@@ -92,3 +92,78 @@ class MiniMaxM3Model(MiniMaxM2Model):
data_torch = data_torch + 1.0
yield from super().modify_tensors(data_torch, name, bid)
+
+
[email protected]("MiniMaxM3SparseForConditionalGeneration", "MiniMaxM3VLForConditionalGeneration")
+class MiniMaxM3VisionModel(MmprojModel):
+ @classmethod
+ def filter_tensors(cls, item):
+ name, gen = item
+ # keep only the vision-side tensors; text / mtp / sparse-index are dropped
+ if not name.startswith(("vision_tower.", "multi_modal_projector.", "patch_merge_mlp.")):
+ return None
+ return super().filter_tensors((name, gen))
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ assert self.hparams_vision is not None
+
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINIMAXM3)
+ self.gguf_writer.add_vision_use_gelu(True)
+
+ # the ViT carries its own LayerNorm eps (text tower uses a different one)
+ self.gguf_writer.add_vision_attention_layernorm_eps(
+ self.hparams_vision.get("layer_norm_eps", 1e-5)
+ )
+
+ comp = self.hparams_vision.get("img_token_compression_config", {})
+ merge_size = comp.get("spatial_merge_size", 2)
+ self.gguf_writer.add_vision_spatial_merge_size(int(merge_size))
+
+ def modify_tensors(self, data_torch, name, bid):
+ assert self.hparams_vision is not None
+
+ # Conv3d patch embed -> Conv2d slices
+ if name == "vision_tower.vision_model.embeddings.patch_embedding.weight":
+ if data_torch.ndim != 5:
+ raise ValueError(f"unexpected patch_embedding rank {data_torch.ndim} for {name}")
+ kt = data_torch.shape[2]
+ base = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH]
+ for t in range(kt):
+ suffix = ".weight" if t == 0 else f".weight.{t}"
+ yield (base + suffix, data_torch[:, :, t, ...])
+ return
+
+ # Permute ViT q/k. HF [Ta Ha Wa | Tb Hb Wb | pad] reorder to [Ta Tb | Ha Hb | Wa Wb | pad].
+ for new_name, tensor in super().modify_tensors(data_torch, name, bid):
+ if ".attn_q." in new_name or ".attn_k." in new_name:
+ tensor = self._permute_vit_qk(tensor, new_name)
+ yield new_name, tensor
+
+ def _permute_vit_qk(self, t: "Tensor", new_name: str) -> "Tensor":
+ assert self.hparams_vision is not None
+ n_head = self.hparams_vision["num_attention_heads"]
+ d_head = t.shape[0] // n_head
+ axis_dim = 2 * ((2 * (d_head // 2) // 3) // 2)
+ ah = axis_dim // 2
+ half = 3 * ah
+ perm = []
+ perm += list(range(0, ah))
+ perm += list(range(half, half + ah))
+ perm += list(range(ah, 2 * ah))
+ perm += list(range(half + ah, half + 2 * ah))
+ perm += list(range(2 * ah, 3 * ah))
+ perm += list(range(half + 2 * ah, half + 3 * ah))
+ perm += list(range(2 * half, d_head))
+
+ assert axis_dim % 2 == 0
+ assert 3 * axis_dim <= d_head
+ assert len(perm) == d_head
+ assert sorted(perm) == list(range(d_head)), "perm is not a bijection of d_head"
+ assert t.shape[0] == n_head * d_head, f"{new_name}: {t.shape[0]} != {n_head}*{d_head}"
+ assert d_head == 80
+
+ idx = torch.tensor(perm, dtype=torch.long)
+ if t.ndim == 2:
+ return t.reshape(n_head, d_head, t.shape[1])[:, idx, :].reshape(t.shape)
+ return t.reshape(n_head, d_head)[:, idx].reshape(t.shape)
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 66d50cca26..2071e3eaa8 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -857,6 +857,8 @@ class MODEL_TENSOR(IntEnum):
V_MM_UP = auto() # cogvlm
V_MM_DOWN = auto() # cogvlm
V_MM_GATE = auto() # cogvlm
+ V_MM_MERGER_FC1 = auto() # minimax-m3 (patch-merge MLP)
+ V_MM_MERGER_FC2 = auto() # minimax-m3 (patch-merge MLP)
V_TOK_BOI = auto() # cogvlm
V_TOK_EOI = auto() # cogvlm
V_TOK_IMG_BEGIN = auto() # hunyuanvl
@@ -1441,6 +1443,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.V_MM_UP: "mm.up",
MODEL_TENSOR.V_MM_DOWN: "mm.down",
MODEL_TENSOR.V_MM_GATE: "mm.gate",
+ MODEL_TENSOR.V_MM_MERGER_FC1: "mm.merger.fc1",
+ MODEL_TENSOR.V_MM_MERGER_FC2: "mm.merger.fc2",
MODEL_TENSOR.V_TOK_BOI: "v.boi",
MODEL_TENSOR.V_TOK_EOI: "v.eoi",
MODEL_TENSOR.V_MM_PRE_NORM: "mm.pre_norm",
@@ -1637,6 +1641,8 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.V_RESMPL_QUERY,
MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK,
MODEL_TENSOR.V_MM_PATCH_MERGER,
+ MODEL_TENSOR.V_MM_MERGER_FC1,
+ MODEL_TENSOR.V_MM_MERGER_FC2,
MODEL_TENSOR.V_DS_NORM,
MODEL_TENSOR.V_DS_FC1,
MODEL_TENSOR.V_DS_FC2,
@@ -4771,6 +4777,7 @@ class VisionProjectorType:
YOUTUVL = "youtuvl"
NEMOTRON_V2_VL = "nemotron_v2_vl"
HUNYUANVL = "hunyuanvl"
+ MINIMAXM3 = "minimax_m3"
MINICPMV4_6 = "minicpmv4_6"
GRANITE_SPEECH = "granite_speech" # audio
MIMOVL = "mimovl"
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index 59623accfd..62d7a827e3 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -1838,6 +1838,14 @@ class TensorNameMap:
"visual.downsample", # glm4v
),
+ MODEL_TENSOR.V_MM_MERGER_FC1: (
+ "patch_merge_mlp.linear_1", # minimax-m3
+ ),
+
+ MODEL_TENSOR.V_MM_MERGER_FC2: (
+ "patch_merge_mlp.linear_2", # minimax-m3
+ ),
+
MODEL_TENSOR.V_DS_NORM: (
"model.visual.deepstack_merger_list.{bid}.norm", # deepstack in qwen3vl
),
diff --git a/skills/code-review/SKILL.md b/skills/code-review/SKILL.md
index 84075fea32..ba76c48115 100644
--- a/skills/code-review/SKILL.md
+++ b/skills/code-review/SKILL.md
@@ -110,6 +110,15 @@ Public API changes carry a higher bar than internal ones (`CONTRIBUTING.md`). Re
- Security: don't trust client-supplied headers (e.g. `X-Forwarded-For`) or add footguns; things like IP allowlisting belong at a reverse proxy unless there's a trusted-proxy design.
- Wire new behavior into the existing request/response and checkpoint paths correctly; watch for resource leaks across requests.
+## Multimodal (`tools/mtmd/`)
+
+- Tensor names must be prefixed by `v.`, `a.`, `mm.` or `a.mm.` (legacy naming doesn't follow this convention - this is expected, but new code should follow it).
+- Do not use explicit sin/cos for RoPE; use `ggml_rope_ext` instead, see `HOWTO-add-model.md`. If it can't express the needed behavior, that's a design discussion, not a PR.
+- New GGML ops must not be introduced in the same PR, you must push it as a separate PR.
+- In most cases, `build_vit` should be enough to build the transformer graph for vision models. Do not add a loop to build the transformer graph manually, unless you have a very good reason to do so. If you do, please explain why in the PR description.
+- If you need a dedicated preprocessor, there is a high chance that it can be a derived class from one of the existing preprocessors. Check carefully before adding a new preprocessor class.
+- If the model need a new public API in `mtmd.h`, open a discussion first.
+
## General (always)
Enforce the `AGENTS.md` / `CONTRIBUTING.md` coding and naming guidelines on every changed line - this is a distinct pass from checking that the code works, and matters just as much for review speed:
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index d0329ca567..fd7ddceb0b 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -47,6 +47,7 @@ add_library(mtmd
models/paddleocr.cpp
models/pixtral.cpp
models/qwen2vl.cpp
+ models/minimax-m3.cpp
models/qwen3vl.cpp
models/mimovl.cpp
models/qwen3a.cpp
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 5b413681f0..42374311ce 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -131,6 +131,8 @@
#define TN_MM_SOFT_EMB_N "mm.soft_emb_norm.weight" // gemma3
#define TN_MM_PROJECTOR "mm.model.fc.%s" // idefics3, deepseekocr
#define TN_MM_PATCH_MERGER "mm.patch_merger.%s" // mistral small 3.1, glm4v
+#define TN_MM_MERGER_FC1 "mm.merger.fc1.%s" // minimax-m3 patch-merge MLP
+#define TN_MM_MERGER_FC2 "mm.merger.fc2.%s"
#define TN_TOK_IMG_BREAK "v.token_embd.img_break" // pixtral
#define TN_TOK_GLM_BOI "adapter.boi" // glm-edge (these embeddings are not in text model)
#define TN_TOK_GLM_EOI "adapter.eoi" // glm-edge (these embeddings are not in text model)
@@ -370,6 +372,7 @@ enum projector_type {
PROJECTOR_TYPE_MINICPMV4_6,
PROJECTOR_TYPE_GRANITE_SPEECH,
PROJECTOR_TYPE_MIMOVL,
+ PROJECTOR_TYPE_MINIMAX_M3,
PROJECTOR_TYPE_GRANITE4_VISION,
PROJECTOR_TYPE_UNKNOWN,
};
@@ -424,6 +427,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"},
{ PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"},
{ PROJECTOR_TYPE_MIMOVL, "mimovl"},
+ { PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"},
{ PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"},
};
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h
index 6d4336c401..850957d7de 100644
--- a/tools/mtmd/clip-model.h
+++ b/tools/mtmd/clip-model.h
@@ -397,6 +397,10 @@ struct clip_model {
ggml_tensor * mm_0_b = nullptr;
ggml_tensor * mm_2_w = nullptr;
ggml_tensor * mm_2_b = nullptr;
+ ggml_tensor * mm_merger_fc1_w = nullptr; // minimax-m3
+ ggml_tensor * mm_merger_fc1_b = nullptr;
+ ggml_tensor * mm_merger_fc2_w = nullptr;
+ ggml_tensor * mm_merger_fc2_b = nullptr;
ggml_tensor * image_newline = nullptr;
ggml_tensor * view_seperator = nullptr;
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index b886650649..e0e2107a0b 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -915,6 +915,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_mimovl>(ctx, img);
} break;
+ case PROJECTOR_TYPE_MINIMAX_M3:
+ {
+ builder = std::make_unique<clip_graph_minimax_m3>(ctx, img);
+ } break;
case PROJECTOR_TYPE_STEP3VL:
{
builder = std::make_unique<clip_graph_step3vl>(ctx, img);
@@ -1469,6 +1473,17 @@ struct clip_model_loader {
LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__);
}
} break;
+ case PROJECTOR_TYPE_MINIMAX_M3:
+ {
+ hparams.n_merge = 2; // spatial_merge_size
+ hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW;
+ hparams.image_resize_pad = PAD_NONE;
+ get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
+ hparams.rope_theta = 10000.0f; // vision_config.rope_theta
+ // MiniMax-M3: max_pixels 451584 (=672^2) -> 576 merged tokens (image_seq_length)
+ hparams.set_limit_image_tokens(8, 576);
+ hparams.set_warmup_n_tokens(16*16);
+ } break;
case PROJECTOR_TYPE_MIMOVL:
{
hparams.n_merge = 2; // spatial_merge_size
@@ -2089,6 +2104,19 @@ struct clip_model_loader {
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false);
} break;
+ case PROJECTOR_TYPE_MINIMAX_M3:
+ {
+ // per-patch MLP: mm.1 -> gelu -> mm.2
+ model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
+ model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
+ model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
+ model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
+ // 2x2 merge MLP: mm.merge.fc1 -> gelu -> mm.merge.fc2
+ model.mm_merger_fc1_w = get_tensor(string_format(TN_MM_MERGER_FC1, "weight"));
+ model.mm_merger_fc1_b = get_tensor(string_format(TN_MM_MERGER_FC1, "bias"));
+ model.mm_merger_fc2_w = get_tensor(string_format(TN_MM_MERGER_FC2, "weight"));
+ model.mm_merger_fc2_b = get_tensor(string_format(TN_MM_MERGER_FC2, "bias"));
+ } break;
case PROJECTOR_TYPE_STEP3VL:
{
model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
@@ -3360,6 +3388,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_MIMOVL:
+ case PROJECTOR_TYPE_MINIMAX_M3:
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_YOUTUVL:
{
@@ -3866,6 +3895,24 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
set_input_i32("positions", positions);
} break;
+ case PROJECTOR_TYPE_MINIMAX_M3:
+ {
+ const int n_merge = hparams.n_merge;
+ const int gh = image_size_height / patch_size;
+ const int gw = image_size_width / patch_size;
+ std::vector<int32_t> pos_h, pos_w;
+ pos_h.reserve(gh * gw);
+ pos_w.reserve(gh * gw);
+ for (int bh = 0; bh < gh / n_merge; bh++)
+ for (int bw = 0; bw < gw / n_merge; bw++)
+ for (int mh = 0; mh < n_merge; mh++)
+ for (int mw = 0; mw < n_merge; mw++) {
+ pos_h.push_back(bh * n_merge + mh);
+ pos_w.push_back(bw * n_merge + mw);
+ }
+ set_input_i32("minimax_pos_h", pos_h);
+ set_input_i32("minimax_pos_w", pos_w);
+ } break;
case PROJECTOR_TYPE_DOTS_OCR:
{
const int pw = image_size_width / patch_size;
@@ -4569,6 +4616,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_ffn_down_w->ne[1];
case PROJECTOR_TYPE_GLM_EDGE:
return ctx->model.mm_model_mlp_3_w->ne[1];
+ case PROJECTOR_TYPE_MINIMAX_M3:
+ return ctx->model.mm_merger_fc2_b->ne[0];
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_EXAONE4_5:
diff --git a/tools/mtmd/models/minimax-m3.cpp b/tools/mtmd/models/minimax-m3.cpp
new file mode 100644
index 0000000000..447621754e
--- /dev/null
+++ b/tools/mtmd/models/minimax-m3.cpp
@@ -0,0 +1,84 @@
+#include "models.h"
+
+ggml_tensor * clip_graph_minimax_m3::apply_rope(
+ ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) {
+ const int64_t Hn = x->ne[1];
+ const int64_t P = x->ne[2];
+ const size_t es = ggml_element_size(x);
+ const int dh = (int) x->ne[0];
+ const int axd = 2 * ((2 * (dh / 2) / 3) / 2);
+
+ GGML_ASSERT(x->nb[0] == es);
+ GGML_ASSERT(3 * axd <= dh);
+
+ const float th = hparams.rope_theta;
+
+ // layout of x is [t, h, w, pad]
+ // t is unrotated, h and w are rotated, pad is unrotated
+ // note: everything from n_dims onward untouched, so w and pad are rotated in one call.
+ auto sl = [&](int off, int n) {
+ return ggml_cont(ctx0, ggml_view_3d(ctx0, x, n, Hn, P, x->nb[1], x->nb[2], (size_t) off * es));
+ };
+ ggml_tensor * t = sl(0, axd);
+ ggml_tensor * h = sl(axd, axd);
+ ggml_tensor * w = sl(2 * axd, dh - 2 * axd); // w + pad
+
+ h = ggml_rope_ext(ctx0, h, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+ w = ggml_rope_ext(ctx0, w, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+ return ggml_concat(ctx0, ggml_concat(ctx0, t, h, 0), w, 0);
+}
+
+ggml_cgraph * clip_graph_minimax_m3::build() {
+ GGML_ASSERT(model.patch_bias == nullptr);
+ GGML_ASSERT(model.class_embedding == nullptr);
+ GGML_ASSERT(model.patch_embeddings_0 && model.patch_embeddings_1);
+ GGML_ASSERT(model.mm_1_w && model.mm_2_w);
+ GGML_ASSERT(model.mm_merger_fc1_w && model.mm_merger_fc2_w);
+
+ const int batch_size = 1;
+ const int n_pos = n_patches;
+ const int merge = hparams.n_merge;
+
+ // patch embedding
+ ggml_tensor * inp_raw = build_inp_raw();
+ ggml_tensor * inp = ggml_add(ctx0,
+ ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1),
+ ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
+
+ // spatial merge
+ {
+ inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);
+ inp = ggml_cont_4d(ctx0, inp, n_embd * merge, n_patches_x / merge, n_patches_y, batch_size);
+ inp = ggml_reshape_4d(ctx0, inp, n_embd * merge, n_patches_x / merge, merge, batch_size * (n_patches_y / merge));
+ inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
+ inp = ggml_cont_3d(ctx0, inp, n_embd, n_patches_x * n_patches_y, batch_size);
+ }
+
+ // t (time axis) is always 0 for now, so we leave it unrotated
+ ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
+ ggml_set_name(pos_h, "minimax_pos_h"); ggml_set_input(pos_h);
+ ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
+ ggml_set_name(pos_w, "minimax_pos_w"); ggml_set_input(pos_w);
+
+ ggml_tensor * inpL = build_vit(
+ inp, n_pos, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr,
+ [&](ggml_tensor * c, const clip_layer &) {
+ return apply_rope(c, pos_h, pos_w);
+ });
+
+ // projector
+ ggml_tensor * emb = inpL;
+ emb = build_ffn(emb, model.mm_1_w, model.mm_1_b,
+ nullptr, nullptr,
+ model.mm_2_w, model.mm_2_b, FFN_GELU_ERF, -1);
+
+ const int64_t proj = emb->ne[0];
+ emb = ggml_reshape_2d(ctx0, emb, proj * merge * merge, n_pos / (merge * merge));
+
+ emb = build_ffn(emb, model.mm_merger_fc1_w, model.mm_merger_fc1_b,
+ nullptr, nullptr,
+ model.mm_merger_fc2_w, model.mm_merger_fc2_b, FFN_GELU_ERF, -1);
+
+ ggml_build_forward_expand(gf, emb);
+ return gf;
+}
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 5f1493fa60..2d7555da41 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -40,6 +40,12 @@ struct clip_graph_qwen3vl : clip_graph_qwen2vl {
ggml_cgraph * build() override;
};
+struct clip_graph_minimax_m3 : clip_graph {
+ clip_graph_minimax_m3(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+ ggml_cgraph * build() override;
+ ggml_tensor * apply_rope(ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w);
+};
+
struct clip_graph_mimovl : clip_graph {
clip_graph_mimovl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 5915b4cba9..bb49b211ef 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -463,6 +463,13 @@ struct mtmd_context {
img_end = "<|vision_end|>";
image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
} break;
+ case PROJECTOR_TYPE_MINIMAX_M3:
+ {
+ // ]<]start of image[>[ ... (image embeddings) ... ]<]end of image[>[
+ img_beg = "]<]start of image[>[";
+ img_end = "]<]end of image[>[";
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ } break;
case PROJECTOR_TYPE_YOUTUVL:
{
// <|vision_start|> ... (image embeddings) ... <|vision_end|>