Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(minimax_m3): serve image input on MiniMax-M3 (#480)

Xiaoze Fan68a81ffddaea
diff --git a/docs/models.md b/docs/models.md
index d2c811ba..16bedb41 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -17,6 +17,7 @@ for them; other checkpoints of the same architectures work too.
| gpt-oss | [openai/gpt-oss-120b](https://huggingface.co/openai/gpt-oss-120b), [openai/gpt-oss-20b](https://huggingface.co/openai/gpt-oss-20b) |
| Gemma-4 | [google/gemma-4-26B-A4B-it](https://huggingface.co/google/gemma-4-26B-A4B-it), [nvidia/Gemma-4-26B-A4B-NVFP4](https://huggingface.co/nvidia/Gemma-4-26B-A4B-NVFP4), [google/gemma-4-12B-it](https://huggingface.co/google/gemma-4-12B-it), [nvidia/Gemma-4-31B-IT-NVFP4](https://huggingface.co/nvidia/Gemma-4-31B-IT-NVFP4) .. |
| MiniMax-M2.5 | [nvidia/MiniMax-M2.5-NVFP4](https://huggingface.co/nvidia/MiniMax-M2.5-NVFP4) |
+| MiniMax-M3 | [nvidia/MiniMax-M3-NVFP4](https://huggingface.co/nvidia/MiniMax-M3-NVFP4) |
| Muse-Glimmer | [meta-models/Muse-Glimmer-30B](https://huggingface.co/meta-models/Muse-Glimmer-30B), [RedHatAI/Muse-Glimmer-30B-NVFP4](https://huggingface.co/RedHatAI/Muse-Glimmer-30B-NVFP4) |
### Image input
@@ -31,6 +32,7 @@ These families accept image input by default; pass `--text-model-only` to skip t
| Gemma-4 12B (`gemma4_unified`: linear patch embedder, resident under either placement) | same budgets, one 48x48 super-patch per soft token | same as the tower releases | same |
| GLM-5.3-Flash (`glm5_next`: ViT tower, streamed under `--mm-encoder-weights host`) | one token per 28x28 pixels of the resized image, dynamic resolution on a canvas zero-padded to a 28-multiple | token counts, passed through as the processor's `min_image_tokens` / `max_image_tokens`; checkpoint defaults 16 to 8000 tokens | `{"max_image_tokens": 2048}` |
| Muse-Glimmer-30B (windowed ViT tower, streamed under `--mm-encoder-weights host`) | one token per 28x28 pixels of the resized image, aspect ratio kept under a token cap; checkpoint default 4096 tokens | the maximum is the cap (`max_image_tokens`); the minimum has no effect | `{"max_image_tokens": 1024}` |
+| MiniMax-M3 (`minimax_m3`: CLIP-style ViT tower, streamed under `--mm-encoder-weights host`) | one token per 28x28 pixels of the resized image, dynamic resolution | pixel areas in `size.shortest_edge` / `longest_edge`; checkpoint defaults 4 to 576 tokens | `{"size": {"longest_edge": 1048576}}` |
## MoE strategies
diff --git a/python/freetoken/attention/m3_sparse.py b/python/freetoken/attention/m3_sparse.py
index 19515f90..82a98a5a 100644
--- a/python/freetoken/attention/m3_sparse.py
+++ b/python/freetoken/attention/m3_sparse.py
@@ -100,7 +100,7 @@ def _pick_inner_backend(block_size: int) -> str:
from freetoken.engine.engine import _resolve_auto_attention_backend
- name = _resolve_auto_attention_backend(frozenset({AttnType.FULL}), False)
+ name = _resolve_auto_attention_backend(frozenset({AttnType.FULL}))
if not _page_ok(name):
# trtllm (the sm_100 first pick) pins 16/32/64-token pages; walk the rest
# of the SAME tree (same arch gates, same requirement probes) with the
diff --git a/python/freetoken/mm/processors/minimax_m3.py b/python/freetoken/mm/processors/minimax_m3.py
new file mode 100644
index 00000000..660d72c2
--- /dev/null
+++ b/python/freetoken/mm/processors/minimax_m3.py
@@ -0,0 +1,92 @@
+"""MiniMax-M3 image processor: patch-grid ViT with a 2x2 merger, a pixel-area budget and 1-D rope."""
+
+from __future__ import annotations
+
+import struct
+from typing import Any
+
+import torch
+
+from freetoken.message import MMItem
+from freetoken.mm import mm_pad_value
+from freetoken.mm.config import MultimodalConfig
+from freetoken.mm.processor import MMProcessor, PromptReplacement, content_hash
+
+# the config names only the image token; the reference processor reads these two from the tokenizer's fixed added tokens
+IMAGE_START_ID = 200029 # ]<]start of image[>[
+IMAGE_END_ID = 200030 # ]<]end of image[>[
+
+
+def _compression(vc: Any, key: str) -> int:
+ """The native vision config carries the merge sizes flat; the checkpoint's own config nests them under img_token_compression_config."""
+ value = getattr(vc, key, None)
+ return int(vc.img_token_compression_config[key] if value is None else value)
+
+
+class MiniMaxM3MMProcessor(MMProcessor):
+ def __init__(self, hf_config: Any, model_path: str, mm: MultimodalConfig) -> None:
+ super().__init__(model_path, mm)
+ vc = hf_config.vision_config
+ # the native config maps image_token_id onto the checkpoint's image_token_index
+ self.image_token_id = getattr(hf_config, "image_token_id", getattr(hf_config, "image_token_index", None))
+ # the template renders one image token per image; the wrappers come from the replacement
+ self.placeholder = [self.image_token_id]
+ self.merge = _compression(vc, "spatial_merge_size")
+ self.pixels_per_token = (vc.patch_size * self.merge) ** 2
+ self.patch_dim = vc.num_channels * _compression(vc, "temporal_patch_size") * vc.patch_size**2
+
+ def get_mm_processor_kwargs(self, mm: MultimodalConfig) -> dict[str, Any]:
+ kwargs: dict[str, Any] = {"return_tensors": "pt"}
+ if mm.image_min_tokens is not None or mm.image_max_tokens is not None:
+ # the budget travels as pixel areas in size.shortest_edge / longest_edge
+ size = dict(self._image_processor().size)
+ if mm.image_min_tokens is not None:
+ size["shortest_edge"] = mm.image_min_tokens * self.pixels_per_token
+ if mm.image_max_tokens is not None:
+ size["longest_edge"] = mm.image_max_tokens * self.pixels_per_token
+ kwargs["size"] = size
+ return {**kwargs, **mm.processor_kwargs}
+
+ def process(self, images: list[Any]) -> list[MMItem]:
+ processor = self._image_processor()
+ kwargs = self.get_mm_processor_kwargs(self.mm)
+ items: list[MMItem] = []
+ for pil in images:
+ out = processor(images=pil, **kwargs)
+ pixel_values = out["pixel_values"].to(torch.bfloat16)
+ grid = out["image_grid_thw"][0].tolist()
+ t, h, w = int(grid[0]), int(grid[1]), int(grid[2])
+ if t != 1:
+ raise ValueError("video/temporal input is not supported")
+ # hash the bf16 wire buffer: half the bytes, and the exact tensor the cache and radix key agree on
+ item_hash = content_hash(pixel_values, struct.pack("<3i", t, h, w))
+ items.append(
+ MMItem(
+ modality="image",
+ hash=item_hash,
+ pad_value=mm_pad_value(item_hash),
+ offsets=[],
+ feature=pixel_values,
+ model_specific_data={"grid_thw": [t, h, w]},
+ )
+ )
+ return items
+
+ def prompt_replacement(self, item: MMItem) -> PromptReplacement:
+ t, h, w = item.grid_thw
+ full = [IMAGE_START_ID] + [self.image_token_id] * ((t * h * w) // (self.merge * self.merge)) + [IMAGE_END_ID]
+ return PromptReplacement.select_token_id(full, self.image_token_id)
+
+ def dummy_items(self, dtype: torch.dtype, device: torch.device) -> list[MMItem]:
+ merge = self.merge
+ return [MMItem(
+ modality="image",
+ hash=0,
+ pad_value=0,
+ offsets=[[0, 1]],
+ feature=torch.zeros(merge * merge, self.patch_dim, dtype=dtype, device=device),
+ model_specific_data={"grid_thw": [1, merge, merge]},
+ )]
+
+
+__all__ = ["IMAGE_END_ID", "IMAGE_START_ID", "MiniMaxM3MMProcessor"]
diff --git a/python/freetoken/models/minimax_m3/__init__.py b/python/freetoken/models/minimax_m3/__init__.py
index cfcebb6f..ca66b680 100644
--- a/python/freetoken/models/minimax_m3/__init__.py
+++ b/python/freetoken/models/minimax_m3/__init__.py
@@ -1,5 +1,6 @@
-from .config import parse_config
-from .model import MiniMaxM3ForCausalLM
+from .config import VisionConfig, parse_config, parse_vision_config
+from .model import MiniMaxM3ForCausalLM, MiniMaxM3ForConditionalGeneration
+from .vision import MiniMaxM3VisionModel
from .weight import (
nvfp4_expert_spec,
iter_weights,
@@ -8,6 +9,10 @@ from .weight import (
__all__ = [
"nvfp4_expert_spec",
"MiniMaxM3ForCausalLM",
+ "MiniMaxM3ForConditionalGeneration",
+ "MiniMaxM3VisionModel",
+ "VisionConfig",
"parse_config",
+ "parse_vision_config",
"iter_weights",
]
diff --git a/python/freetoken/models/minimax_m3/config.py b/python/freetoken/models/minimax_m3/config.py
index 7df81d79..a54d48e9 100644
--- a/python/freetoken/models/minimax_m3/config.py
+++ b/python/freetoken/models/minimax_m3/config.py
@@ -1,10 +1,9 @@
"""Engine-facing config for MiniMax-M3 (``minimax_m3``).
The checkpoint is a multimodal wrapper (``model_type=minimax_m3_vl``): the text tower
-lives in ``text_config`` and the weights carry a ``language_model.`` prefix. FreeToken
-serves the text tower; the ViT vision stack (``vision_tower.`` / projector) is skipped
-at load like the other VL checkpoints' (``VISION_KEY_PREFIXES``) -- multimodal input
-is future work, so ``ModelConfig.vision_config`` stays None here.
+lives in ``text_config`` and the weights carry a ``language_model.`` prefix. Its vision
+section becomes ``ModelConfig.vision_config`` for the tower in vision.py; an engine that
+builds no vision encoder hands parse_config a config without that section.
Attention is GQA with block-sparse selection on the trailing layers: parse_config
declares ONE full-attention group over all layers carrying ``mla=False`` plus the
@@ -27,6 +26,7 @@ their swigluoai activation restricts the NVFP4 GEMM backend to the Triton kernel
from __future__ import annotations
import os
+from dataclasses import dataclass
from typing import Any
from freetoken.models.config import (
@@ -41,6 +41,54 @@ def _text_config(hf_config: Any) -> Any:
return getattr(hf_config, "text_config", None) or hf_config
+@dataclass(frozen=True)
+class VisionConfig:
+ hidden_size: int
+ num_layers: int
+ num_heads: int
+ intermediate_size: int
+ num_channels: int
+ patch_size: int
+ temporal_patch_size: int
+ spatial_merge_size: int
+ layer_norm_eps: float
+ rope_theta: float
+ projector_hidden_size: int
+ text_hidden_size: int
+
+
+def _compression(vc: Any, key: str) -> int:
+ """The native vision config carries the merge sizes flat; the checkpoint's own config nests them under img_token_compression_config."""
+ value = getattr(vc, key, None)
+ return int(vc.img_token_compression_config[key] if value is None else value)
+
+
+def parse_vision_config(hf_config: Any) -> VisionConfig | None:
+ """None when the config carries no vision section, which is how a text-only engine asks for no tower."""
+ vc = getattr(hf_config, "vision_config", None)
+ if vc is None:
+ return None
+ if vc.hidden_act != "gelu":
+ raise NotImplementedError(f"minimax_m3 vision tower activation {vc.hidden_act!r}; only gelu is implemented")
+ # the native config moves rope_theta into rope_parameters; the checkpoint's own config keeps it flat
+ rope_params = getattr(vc, "rope_parameters", None) or {}
+ text_hidden_size = _text_config(hf_config).hidden_size
+ return VisionConfig(
+ hidden_size=vc.hidden_size,
+ num_layers=vc.num_hidden_layers,
+ num_heads=vc.num_attention_heads,
+ intermediate_size=vc.intermediate_size,
+ num_channels=vc.num_channels,
+ patch_size=vc.patch_size,
+ temporal_patch_size=_compression(vc, "temporal_patch_size"),
+ spatial_merge_size=_compression(vc, "spatial_merge_size"),
+ layer_norm_eps=vc.layer_norm_eps,
+ rope_theta=float(rope_params["rope_theta"] if "rope_theta" in rope_params else vc.rope_theta),
+ projector_hidden_size=getattr(hf_config, "projector_hidden_size", None) or text_hidden_size,
+ text_hidden_size=text_hidden_size,
+ )
+
+
# House rule: an env switch that changes WHAT IS SERVED must leave a server-log
# trace. parse_config runs a few times per process (engine + weight loader), so
# each resolved-mode line is logged once per distinct value.
@@ -179,7 +227,10 @@ def parse_config(hf_config: Any) -> ModelConfig:
dense_quant="mxfp8" if dense_mxfp8 else "none",
lm_head_quant="none",
m3_args=args,
+ vision_config=parse_vision_config(hf_config),
+ # the native config maps image_token_id onto the checkpoint's image_token_index
+ image_token_id=getattr(hf_config, "image_token_id", getattr(hf_config, "image_token_index", None)),
)
-__all__ = ["parse_config"]
+__all__ = ["VisionConfig", "parse_config", "parse_vision_config"]
diff --git a/python/freetoken/models/minimax_m3/model.py b/python/freetoken/models/minimax_m3/model.py
index e8b6e5d0..d78fc10a 100644
--- a/python/freetoken/models/minimax_m3/model.py
+++ b/python/freetoken/models/minimax_m3/model.py
@@ -11,14 +11,16 @@ from freetoken.layers import (
ParallelLMHead,
VocabParallelEmbedding,
)
-from freetoken.models.blocks import BaseLLMModel
+from freetoken.models.blocks import BaseLLMModel, embed_input_ids
from freetoken.utils import nvtx_annotate
from .attention import MiniMaxM3Attention
from .mlp import MiniMaxM3MLP
from .moe import MiniMaxM3SparseMoeBlock
+from .vision import MiniMaxM3VisionModel
if TYPE_CHECKING:
+ from freetoken.message import MMItem
from freetoken.models.config import ModelConfig
@@ -79,7 +81,7 @@ class MiniMaxM3Model(BaseOP):
self.norm = GemmaPlusOneRMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
- x = self.embed_tokens.forward(input_ids)
+ x = embed_input_ids(self.embed_tokens, input_ids, get_global_ctx().batch)
residual: torch.Tensor | None = None
for layer in self.layers.op_list:
x, residual = layer.forward(x, residual)
@@ -105,4 +107,17 @@ class MiniMaxM3ForCausalLM(BaseLLMModel):
return self.lm_head.forward(output)
-__all__ = ["MiniMaxM3ForCausalLM"]
+class MiniMaxM3ForConditionalGeneration(MiniMaxM3ForCausalLM):
+ def __init__(self, config: ModelConfig):
+ super().__init__(config)
+ if config.is_multimodal:
+ self.vision_tower = MiniMaxM3VisionModel(config.vision_config)
+
+ def place_encoder_weights(self, mode: str) -> None:
+ self.vision_tower.place_weights(mode)
+
+ def encode(self, item: MMItem) -> torch.Tensor:
+ return self.vision_tower.forward(item.feature, item.grid_thw)
+
+
+__all__ = ["MiniMaxM3ForCausalLM", "MiniMaxM3ForConditionalGeneration"]
diff --git a/python/freetoken/models/minimax_m3/vision.py b/python/freetoken/models/minimax_m3/vision.py
new file mode 100644
index 00000000..97074ec8
--- /dev/null
+++ b/python/freetoken/models/minimax_m3/vision.py
@@ -0,0 +1,161 @@
+"""MiniMax-M3 vision tower: Conv3d patch embedding, pre-LayerNorm CLIP blocks with a 3-axis rope, a GELU projector and a 2x2 patch-merge MLP into the text width."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, List
+
+import torch
+import torch.nn.functional as F
+from freetoken.layers import BaseOP, LayerNorm, LinearReplicated, OPList
+from freetoken.models.weight_stream import BlockWeightStreamer
+
+if TYPE_CHECKING:
+ from .config import VisionConfig
+
+
+class _Conv3dWeight(BaseOP):
+ def __init__(self, out_ch: int, shape: tuple[int, ...]):
+ self.weight = torch.empty(out_ch, *shape)
+
+ forward = None
+
+
+class MiniMaxM3VisionEmbeddings(BaseOP):
+ """Kept as a real Conv3d (kernel == stride): an equivalent unfolded matmul rounds differently."""
+
+ def __init__(self, vc: VisionConfig):
+ self._shape = (vc.num_channels, vc.temporal_patch_size, vc.patch_size, vc.patch_size)
+ self.patch_embedding = _Conv3dWeight(vc.hidden_size, self._shape)
+
+ def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
+ x = pixel_values.view(-1, *self._shape).to(self.patch_embedding.weight.dtype)
+ return F.conv3d(x, self.patch_embedding.weight, stride=self._shape[1:]).view(x.shape[0], -1)
+
+
+def _rope_table(grid_thw: List[int], merge: int, head_dim: int, theta: float, device: torch.device) -> tuple[torch.Tensor, torch.Tensor]:
+ """Per-token [cos | sin] rows over the rotated dims: the even part of the head split into three equal t/h/w bands, the remainder passes through."""
+ t, h, w = grid_thw
+ axis_dim = 2 * (((2 * (head_dim // 2)) // 3) // 2)
+ inv_freq = 1.0 / (theta ** (torch.arange(0, axis_dim, 2, dtype=torch.float32, device=device) / axis_dim))
+ hpos, wpos = torch.meshgrid(torch.arange(h, device=device), torch.arange(w, device=device), indexing="ij")
+ # block-major over the merge blocks, the order the processor flattens patches in
+ block = (h // merge, merge, w // merge, merge)
+ hpos = hpos.reshape(block).transpose(1, 2).flatten().repeat(t)
+ wpos = wpos.reshape(block).transpose(1, 2).flatten().repeat(t)
+ tpos = torch.arange(t, device=device).repeat_interleave(h * w)
+ coords = torch.stack([tpos, hpos, wpos], dim=-1).float()
+ freqs = (coords.unsqueeze(-1) * inv_freq).flatten(1)
+ cache = torch.cat((freqs.cos(), freqs.sin()), dim=-1).contiguous()
+ return cache, torch.arange(coords.shape[0], dtype=torch.int32, device=device)
+
+
+class MiniMaxM3VisionAttention(BaseOP):
+ def __init__(self, vc: VisionConfig):
+ self.num_heads = vc.num_heads
+ self.head_dim = vc.hidden_size // vc.num_heads
+ self.qkv = LinearReplicated(vc.hidden_size, 3 * vc.hidden_size, has_bias=True)
+ self.out_proj = LinearReplicated(vc.hidden_size, vc.hidden_size, has_bias=True)
+
+ def forward(self, x: torch.Tensor, cache: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel.triton.rope import apply_rope_with_cos_sin_cache_inplace
+
+ S, H, D = x.shape[0], self.num_heads, self.head_dim
+ q, k, v = self.qkv.forward(x).view(S, 3, H * D).unbind(1)
+ apply_rope_with_cos_sin_cache_inplace(positions, q, k, D, cache, is_neox=True)
+ # the fused SDPA kernels want a batch dim; 3-D inputs fall back to the O(S^2) math path
+ q, k, v = (t.view(S, H, D).transpose(0, 1).unsqueeze(0) for t in (q, k, v))
+ o = F.scaled_dot_product_attention(q, k, v)
+ return self.out_proj.forward(o[0].transpose(0, 1).reshape(S, H * D))
+
+
+class MiniMaxM3VisionMLP(BaseOP):
+ def __init__(self, vc: VisionConfig):
+ self.fc1 = LinearReplicated(vc.hidden_size, vc.intermediate_size, has_bias=True)
+ self.fc2 = LinearReplicated(vc.intermediate_size, vc.hidden_size, has_bias=True)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ return self.fc2.forward(F.gelu(self.fc1.forward(x)))
+
+
+class MiniMaxM3VisionEncoderLayer(BaseOP):
+ def __init__(self, vc: VisionConfig):
+ self.self_attn = MiniMaxM3VisionAttention(vc)
+ self.layer_norm1 = LayerNorm(vc.hidden_size, vc.layer_norm_eps)
+ self.mlp = MiniMaxM3VisionMLP(vc)
+ self.layer_norm2 = LayerNorm(vc.hidden_size, vc.layer_norm_eps)
+
+ def forward(self, x: torch.Tensor, cache: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
+ x = x + self.self_attn.forward(self.layer_norm1.forward(x), cache, positions)
+ return x + self.mlp.forward(self.layer_norm2.forward(x))
+
+
+class MiniMaxM3VisionEncoder(BaseOP):
+ def __init__(self, vc: VisionConfig):
+ self.layers = OPList([MiniMaxM3VisionEncoderLayer(vc) for _ in range(vc.num_layers)])
+
+ forward = None
+
+
+class MiniMaxM3VisionTransformer(BaseOP):
+ """Patches of one image -> [t * h * w, hidden]; the checkpoint has no post layernorm."""
+
+ def __init__(self, vc: VisionConfig):
+ self.embeddings = MiniMaxM3VisionEmbeddings(vc)
+ self.pre_layrnorm = LayerNorm(vc.hidden_size, vc.layer_norm_eps)
+ self.encoder = MiniMaxM3VisionEncoder(vc)
+ self._vc = vc
+ self._streamer: BlockWeightStreamer | None = None
+
+ def place_weights(self, mode: str) -> None:
+ """gpu: every tensor resident; host: encoder layer tensors in pinned banks streamed two layers at a time."""
+ if mode == "host" and self._streamer is None:
+ self._streamer = BlockWeightStreamer(self.encoder.layers.op_list, self.embeddings.patch_embedding.weight.device)
+ elif mode == "gpu" and self._streamer is not None:
+ self._streamer.unstream()
+ self._streamer = None
+ elif mode not in ("gpu", "host"):
+ raise ValueError(f"unknown vision weight placement {mode!r}")
+
+ def _layers(self):
+ if self._streamer is None:
+ return enumerate(self.encoder.layers.op_list)
+ return self._streamer.blocks(self.encoder.layers.op_list)
+
+ def forward(self, pixel_values: torch.Tensor, grid_thw: List[int]) -> torch.Tensor:
+ vc = self._vc
+ device = self.embeddings.patch_embedding.weight.device
+ x = self.pre_layrnorm.forward(self.embeddings.forward(pixel_values.to(device)))
+ cache, positions = _rope_table(grid_thw, vc.spatial_merge_size, vc.hidden_size // vc.num_heads, vc.rope_theta, device)
+ for _, layer in self._layers():
+ x = layer.forward(x, cache, positions)
+ return x
+
+
+class MiniMaxM3ProjectorMLP(BaseOP):
+ def __init__(self, in_size: int, mid_size: int, out_size: int):
+ self.linear_1 = LinearReplicated(in_size, mid_size, has_bias=True)
+ self.linear_2 = LinearReplicated(mid_size, out_size, has_bias=True)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ return self.linear_2.forward(F.gelu(self.linear_1.forward(x)))
+
+
+class MiniMaxM3VisionModel(BaseOP):
+ """Pixels of one image -> soft tokens [t * h * w / merge^2, text_hidden]."""
+
+ def __init__(self, vc: VisionConfig):
+ self.vision_model = MiniMaxM3VisionTransformer(vc)
+ self.multi_modal_projector = MiniMaxM3ProjectorMLP(vc.hidden_size, vc.projector_hidden_size, vc.text_hidden_size)
+ self.patch_merge_mlp = MiniMaxM3ProjectorMLP(vc.spatial_merge_size**2 * vc.text_hidden_size, vc.projector_hidden_size, vc.text_hidden_size)
+ self._merge = vc.spatial_merge_size**2
+
+ def place_weights(self, mode: str) -> None:
+ self.vision_model.place_weights(mode)
+
+ @torch.inference_mode()
+ def forward(self, pixel_values: torch.Tensor, grid_thw: List[int]) -> torch.Tensor:
+ x = self.multi_modal_projector.forward(self.vision_model.forward(pixel_values, grid_thw))
+ return self.patch_merge_mlp.forward(x.view(x.shape[0] // self._merge, -1))
+
+
+__all__ = ["MiniMaxM3VisionModel"]
diff --git a/python/freetoken/models/minimax_m3/weight.py b/python/freetoken/models/minimax_m3/weight.py
index 3aea2347..b3c928e2 100644
--- a/python/freetoken/models/minimax_m3/weight.py
+++ b/python/freetoken/models/minimax_m3/weight.py
@@ -3,7 +3,8 @@
The checkpoint is the multimodal wrapper layout: every text-tower tensor carries a
``language_model.`` prefix (``language_model.model.layers.N...``,
``language_model.lm_head.weight``); the ViT stack (``vision_tower.`` /
-``multi_modal_projector.`` / ``patch_merge_mlp.``) is never read (text-only serving).
+``multi_modal_projector.`` / ``patch_merge_mlp.``) is read by ``_iter_vision`` only when
+the engine builds the vision encoder (``include_vision``).
Resident (non routed-expert) dense projections are MXFP8 in the checkpoint
(fp8-e4m3 ``weight`` + uint8 e8m0 block-32 ``weight_scale_inv``): the fp8 weight and
@@ -135,12 +136,31 @@ def _emit_fused(
)
+_VISION_ATTN = "vision_tower.vision_model.encoder.layers.{}.self_attn"
+
+
+def _iter_vision(reader: _ShardReader, weight_map: dict, num_layers: int) -> Iterator[tuple[str, torch.Tensor]]:
+ """The tower under its checkpoint names with q/k/v fused, the projector and patch-merge MLP under the tower prefix; all bf16 (the patch embedding is stored fp32)."""
+ for name in weight_map:
+ if name.startswith(("multi_modal_projector.", "patch_merge_mlp.")):
+ yield "vision_tower." + name, reader.get(name).to(torch.bfloat16)
+ elif name.startswith("vision_tower.") and ".self_attn." not in name:
+ yield name, reader.get(name).to(torch.bfloat16)
+ for layer in range(num_layers):
+ attn = _VISION_ATTN.format(layer)
+ for kind in ("weight", "bias"):
+ parts = [reader.get(f"{attn}.{proj}.{kind}") for proj in ("q_proj", "k_proj", "v_proj")]
+ yield f"{attn}.qkv.{kind}", torch.cat(parts, dim=0).to(torch.bfloat16)
+ yield f"{attn}.out_proj.{kind}", reader.get(f"{attn}.out_proj.{kind}").to(torch.bfloat16)
+
+
def iter_weights(
model_path: str,
device: torch.device,
*,
include_moe_experts: bool,
include_non_moe: bool,
+ include_vision: bool = True,
) -> Iterator[tuple[str, torch.Tensor]]:
assert not include_moe_experts, (
"MiniMax-M3 stores routed experts as NVFP4 and only supports the offload MoE "
@@ -234,6 +254,8 @@ def iter_weights(
yield "model.embed_tokens.weight", reader.get("language_model.model.embed_tokens.weight")
yield "model.norm.weight", reader.get("language_model.model.norm.weight")
yield "lm_head.weight", reader.get("language_model.lm_head.weight")
+ if include_vision and config.vision_config is not None:
+ yield from _iter_vision(reader, weight_map, config.vision_config.num_layers)
finally:
reader.close()
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index 1d7ac3e5..2a4ced10 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -82,6 +82,8 @@ _MUSE_GLIMMER_ENCODERS = (EncoderSpec("vision", "vision_config", ("image",)),)
_MINIMAX_M3_PACKED = _DENSE_PACKED + (
("index_qk_proj", ("index_q_proj", "index_k_proj")),
) + _EXPERTS_W123_PACKED
+_MINIMAX_M3_PROCESSOR = "freetoken.mm.processors.minimax_m3:MiniMaxM3MMProcessor"
+_MINIMAX_M3_ENCODERS = (EncoderSpec("vision", "vision_config", ("image",)),)
_MODEL_REGISTRY: dict[str, ModelSpec] = {
"LlamaForCausalLM": ModelSpec(
@@ -127,14 +129,17 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
packed_modules_mapping=_DENSE_PACKED + _EXPERTS_W123_PACKED,
),
# MiniMax-M3 (model_type minimax_m3_vl): multimodal wrapper config (text tower in
- # text_config, weights under language_model.); served text-only. GQA + block-sparse
- # attention (lightning indexer, top-k 128-token blocks) on the trailing layers,
- # sigmoid/bias-routed NVFP4 experts + MXFP8 shared expert, swigluoai activation.
+ # text_config, weights under language_model.); the CLIP-style tower under vision_tower.
+ # serves image input. GQA + block-sparse attention (lightning indexer, top-k 128-token
+ # blocks) on the trailing layers, sigmoid/bias-routed NVFP4 experts + MXFP8 shared
+ # expert, swigluoai activation.
"MiniMaxM3SparseForConditionalGeneration": ModelSpec(
"freetoken.models.minimax_m3",
- "MiniMaxM3ForCausalLM",
+ "MiniMaxM3ForConditionalGeneration",
checkpoint_roots=(("model", "language_model.model"), ("lm_head", "language_model.lm_head")),
packed_modules_mapping=_MINIMAX_M3_PACKED,
+ mm_processor=_MINIMAX_M3_PROCESSOR,
+ encoders=_MINIMAX_M3_ENCODERS,
),
# Text-only sibling (the text_config's own architectures entry).
"MiniMaxM3SparseForCausalLM": ModelSpec(
diff --git a/tests/README.md b/tests/README.md
index 1f90f6a3..26dba633 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -54,6 +54,7 @@ checkpoint is set:
| `FREETOKEN_GEMMA4_UNIFIED_MODEL` | `models/test_gemma4_vision.py` — local gemma-4-12B `gemma4_unified` checkpoint (embedder parity) |
| `FREETOKEN_GLM53_MODEL` | `tokenizer/test_mm_tokenize.py` — local GLM-5.3-Flash checkpoint (image-token expansion) |
| `FREETOKEN_MUSE_MODEL` | `models/test_muse_glimmer_vision.py`, `tokenizer/test_mm_tokenize.py` — local Muse-Glimmer-30B checkpoint (tower parity, image_start/image_end expansion) |
+| `FREETOKEN_MINIMAX_M3_MODEL` | `tokenizer/test_mm_tokenize.py` — local MiniMax-M3 checkpoint (start/end expansion, token budget) |
| `FREETOKEN_TEST_MOE_CACHE_SIZE` | `e2e/test_aime.py` — >0 switches to the offload MoE backend with this cache size |
| `FREETOKEN_TEST_MEM_RATIO` | `e2e/test_aime.py` — offload-mode memory_ratio (default `0.9`) |
| `FREETOKEN_REBUILD_TEST_MODEL` | `e2e/test_cache_rebuild.py` — a SMALL local model dir; boots a real server (falls back to `FREETOKEN_TEST_MODEL`) |
diff --git a/tests/mm/test_processor.py b/tests/mm/test_processor.py
index c3049256..a9a50e3c 100644
--- a/tests/mm/test_processor.py
+++ b/tests/mm/test_processor.py
@@ -15,6 +15,7 @@ from freetoken.mm.processor import MMProcessor, PromptReplacement, image_positio
from freetoken.mm.processors.gemma4 import Gemma4MMProcessor, Gemma4UnifiedMMProcessor
from freetoken.mm.processors.glm5_next import Glm5NextMMProcessor
from freetoken.mm.processors.muse_glimmer import MuseGlimmerMMProcessor
+from freetoken.mm.processors.minimax_m3 import IMAGE_END_ID, IMAGE_START_ID, MiniMaxM3MMProcessor
from freetoken.mm.processors.qwen_vl import QwenVLMMProcessor
PLACEHOLDER = 7
@@ -350,6 +351,45 @@ def test_muse_wraps_the_pads_in_image_start_end_and_takes_only_a_token_maximum()
assert dummy.feature.shape == (4, 1176) and dummy.num_tokens == 1
+def _minimax_config():
+ # the checkpoint's own config shape: image_token_index, merge sizes under img_token_compression_config
+ return SimpleNamespace(
+ architectures=["MiniMaxM3SparseForConditionalGeneration"],
+ image_token_index=200025,
+ vision_config=SimpleNamespace(patch_size=14, num_channels=3, img_token_compression_config={"spatial_merge_size": 2, "temporal_patch_size": 2}),
+ text_config=SimpleNamespace(vocab_size=200064),
+ )
+
+
+def test_minimax_token_budget_becomes_pixel_bounds_and_the_span_gets_start_end_tokens():
+ calls = []
+
+ class _FakeImageProcessor:
+ size = {"shortest_edge": 3136, "longest_edge": 451584}
+
+ def __call__(self, images, **kwargs):
+ calls.append(kwargs)
+ return {"pixel_values": torch.zeros(16, 1176), "image_grid_thw": torch.tensor([[1, 4, 4]])}
+
+ mm = MultimodalConfig(image_min_tokens=8, image_max_tokens=256, processor_kwargs={"do_convert_rgb": False})
+ proc = MiniMaxM3MMProcessor(_minimax_config(), "/nonexistent", mm)
+ proc._image_processor = lambda: _FakeImageProcessor()
+ (item,) = proc.process([object()])
+ # 784 pixels per token: patch 14 x merge 2, squared
+ assert calls == [{"return_tensors": "pt", "size": {"shortest_edge": 8 * 784, "longest_edge": 256 * 784}, "do_convert_rgb": False}]
+ assert item.grid_thw == [1, 4, 4] and item.feature.dtype == torch.bfloat16
+ repl = proc.prompt_replacement(item)
+ assert proc.placeholder == [200025] and repl.full == [IMAGE_START_ID] + [200025] * 4 + [IMAGE_END_ID] and repl.embed_spans() == [[1, 5]]
+ assert proc.positions(10, [item]) is None
+ proc = MiniMaxM3MMProcessor(_minimax_config(), "/nonexistent", MultimodalConfig())
+ proc._image_processor = lambda: _FakeImageProcessor()
+ proc.process([object()])
+ assert calls[-1] == {"return_tensors": "pt"}
+ (dummy,) = proc.dummy_items(torch.bfloat16, torch.device("cpu"))
+ dummy.validate()
+ assert dummy.feature.shape == (4, 1176) and dummy.num_tokens == 1
+
+
def test_registry_resolves_by_architecture(monkeypatch):
import freetoken.utils
from freetoken.mm.processor import get_mm_processor
@@ -360,6 +400,7 @@ def test_registry_resolves_by_architecture(monkeypatch):
"/gemma-unified": _gemma_unified_config(),
"/glm": _glm_config(),
"/muse": _muse_config(),
+ "/minimax": _minimax_config(),
"/text-only": _hf_config(vision=False),
"/unknown-vlm": _hf_config(arch="SomeOtherForConditionalGeneration"),
}
@@ -370,6 +411,7 @@ def test_registry_resolves_by_architecture(monkeypatch):
assert isinstance(get_mm_processor("/gemma-unified"), Gemma4UnifiedMMProcessor)
assert isinstance(get_mm_processor("/glm"), Glm5NextMMProcessor)
assert isinstance(get_mm_processor("/muse"), MuseGlimmerMMProcessor)
+ assert isinstance(get_mm_processor("/minimax"), MiniMaxM3MMProcessor)
assert get_mm_processor("/text-only") is None
assert get_mm_processor("/unknown-vlm") is None
assert get_mm_processor("/missing") is None # a config that fails to load means no vision
diff --git a/tests/models/test_minimax_m3.py b/tests/models/test_minimax_m3.py
index 1062261e..dc0ab90d 100644
--- a/tests/models/test_minimax_m3.py
+++ b/tests/models/test_minimax_m3.py
@@ -14,7 +14,7 @@ import torch
from freetoken.layers.quantization import MoEConfig
from freetoken.attention.base import AttnType
-from freetoken.models.minimax_m3.config import parse_config
+from freetoken.models.minimax_m3.config import VisionConfig, parse_config
class _Cfg:
@@ -213,16 +213,51 @@ def test_sparse_ablation_env(monkeypatch):
assert resolve_pool_class(cfg) is MHAKVCache
+def _vision_section(native: bool) -> _Cfg:
+ section = {
+ "hidden_size": 1280, "num_hidden_layers": 32, "num_attention_heads": 16, "intermediate_size": 5120,
+ "num_channels": 3, "patch_size": 14, "hidden_act": "gelu", "layer_norm_eps": 1e-5,
+ }
+ if native:
+ section.update(temporal_patch_size=2, spatial_merge_size=2, rope_parameters={"rope_theta": 10000.0, "rope_type": "axial"})
+ else:
+ section.update(rope_theta=10000.0, img_token_compression_config={"spatial_merge_size": 2, "temporal_patch_size": 2})
+ return _Cfg(section)
+
+
+def test_parse_config_vision_section_in_both_config_shapes(monkeypatch):
+ monkeypatch.delenv("FREETOKEN_M3_MAX_LAYERS", raising=False)
+ expected = VisionConfig(
+ hidden_size=1280, num_layers=32, num_heads=16, intermediate_size=5120, num_channels=3, patch_size=14, temporal_patch_size=2,
+ spatial_merge_size=2, layer_norm_eps=1e-5, rope_theta=10000.0, projector_hidden_size=6144, text_hidden_size=6144,
+ )
+ raw = _hf_config()
+ raw.vision_config, raw.image_token_index, raw.projector_hidden_size = _vision_section(native=False), 200025, 6144
+ native = _hf_config_native()
+ native.vision_config, native.image_token_id, native.projector_hidden_size = _vision_section(native=True), 200025, 6144
+ for cfg in (raw, native):
+ parsed = parse_config(cfg)
+ assert parsed.is_multimodal and parsed.vision_config == expected and parsed.image_token_id == 200025
+ # a text-only engine hands over a config without the section
+ assert not parse_config(_hf_config()).is_multimodal
+
+
def test_registry_resolves_both_architectures():
from freetoken.models.register import get_model_spec
for arch in ("MiniMaxM3SparseForConditionalGeneration", "MiniMaxM3SparseForCausalLM"):
spec = get_model_spec(arch)
assert spec.module == "freetoken.models.minimax_m3"
+ multimodal = get_model_spec("MiniMaxM3SparseForConditionalGeneration")
+ assert multimodal.model_cls == "MiniMaxM3ForConditionalGeneration"
+ assert multimodal.mm_processor == "freetoken.mm.processors.minimax_m3:MiniMaxM3MMProcessor"
+ assert [(e.kind, e.config_key, e.modalities) for e in multimodal.encoders] == [("vision", "vision_config", ("image",))]
+ assert get_model_spec("MiniMaxM3SparseForCausalLM").encoders == ()
-def test_auto_backend_resolution():
+def test_auto_backend_resolution(monkeypatch):
from freetoken.attention import attention_backend_info
+ from freetoken.attention.m3_sparse import _pick_inner_backend
from freetoken.engine.engine import _required_attn_types, _resolve_auto_attention_backend
cfg = parse_config(_hf_config())
@@ -230,6 +265,9 @@ def test_auto_backend_resolution():
assert required == frozenset({AttnType.BSA})
assert _resolve_auto_attention_backend(required) == "m3_sparse"
assert attention_backend_info("m3_sparse").page_sizes == (128,)
+ # the dense leading layers take a FULL backend from the same resolver, filtered to 128-token pages
+ monkeypatch.delenv("FREETOKEN_M3_INNER_BACKEND", raising=False)
+ assert _pick_inner_backend(128) in ("fa,fi", "fi", "triton")
def test_nvfp4_experts_restricted_to_triton_for_swigluoai(monkeypatch):
diff --git a/tests/models/test_minimax_m3_vision.py b/tests/models/test_minimax_m3_vision.py
new file mode 100644
index 00000000..a2885b1a
--- /dev/null
+++ b/tests/models/test_minimax_m3_vision.py
@@ -0,0 +1,65 @@
+"""MiniMax-M3 vision tower: shapes and host streaming on a tiny random tower."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+from freetoken.models.minimax_m3 import MiniMaxM3VisionModel, VisionConfig
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
+
+
[email protected](autouse=True)
+def _single_rank():
+ from freetoken.distributed import set_tp_info, try_get_tp_info
+
+ if try_get_tp_info() is None:
+ set_tp_info(rank=0, size=1)
+
+
+def _tiny_vc():
+ return VisionConfig(
+ hidden_size=64, num_layers=2, num_heads=4, intermediate_size=128, num_channels=3, patch_size=4, temporal_patch_size=2,
+ spatial_merge_size=2, layer_norm_eps=1e-5, rope_theta=10000.0, projector_hidden_size=96, text_hidden_size=48,
+ )
+
+
+def _build(vc):
+ torch.set_default_dtype(torch.bfloat16)
+ try:
+ with torch.device("cuda"):
+ tower = MiniMaxM3VisionModel(vc)
+ finally:
+ torch.set_default_dtype(torch.float32)
+ for p in tower.state_dict().values():
+ p.normal_(0, 0.02)
+ return tower
+
+
+def test_tiny_tower_merges_four_patches_per_token():
+ tower = _build(_tiny_vc())
+ feature = torch.randn(64, 3 * 2 * 4 * 4, dtype=torch.bfloat16, device="cuda")
+ out = tower.forward(feature, [1, 8, 8])
+ assert out.shape == (16, 48) and out.dtype == torch.bfloat16
+ keys = tower.state_dict()
+ assert keys["vision_model.embeddings.patch_embedding.weight"].shape == (64, 3, 2, 4, 4)
+ assert keys["vision_model.encoder.layers.0.self_attn.qkv.bias"].shape == (192,)
+ assert keys["patch_merge_mlp.linear_1.weight"].shape == (96, 4 * 48)
+ assert not any(k.startswith("vision_model.post_layernorm") for k in keys)
+
+
+def test_host_streamed_weights_compute_the_same_output():
+ tower = _build(_tiny_vc())
+ feature = torch.randn(64, 3 * 2 * 4 * 4, dtype=torch.bfloat16, device="cuda")
+ resident = tower.forward(feature, [1, 8, 8])
+ keys = tower.state_dict()
+ tower.place_weights("host")
+ assert tower.state_dict().keys() == keys.keys()
+ assert tower.state_dict()["vision_model.encoder.layers.0.self_attn.qkv.weight"].device.type == "cpu"
+ assert tower.state_dict()["vision_model.pre_layrnorm.weight"].device.type == "cuda"
+ for _ in range(2): # a second forward reuses the staging buffers behind the release events
+ assert torch.equal(tower.forward(feature, [1, 8, 8]), resident)
+ tower.place_weights("gpu")
+ assert tower.state_dict()["vision_model.encoder.layers.0.self_attn.qkv.weight"].device.type == "cuda"
+ assert torch.equal(tower.forward(feature, [1, 8, 8]), resident)
diff --git a/tests/tokenizer/test_mm_tokenize.py b/tests/tokenizer/test_mm_tokenize.py
index 05f883bd..8f2e0211 100644
--- a/tests/tokenizer/test_mm_tokenize.py
+++ b/tests/tokenizer/test_mm_tokenize.py
@@ -1,4 +1,4 @@
-"""Image tokenization against real Qwen VL, Gemma-4, GLM-5.3 and Muse-Glimmer checkpoints (skipped when absent)."""
+"""Image tokenization against real Qwen VL, Gemma-4, GLM-5.3, Muse-Glimmer and MiniMax-M3 checkpoints (skipped when absent)."""
from __future__ import annotations
@@ -19,12 +19,13 @@ MODELS = [
GEMMA = os.environ.get("FREETOKEN_GEMMA4_MODEL", "")
GLM = os.environ.get("FREETOKEN_GLM53_MODEL", "")
MUSE = os.environ.get("FREETOKEN_MUSE_MODEL", "")
+MINIMAX = os.environ.get("FREETOKEN_MINIMAX_M3_MODEL", "")
pytestmark = [
pytest.mark.needs_weights,
pytest.mark.skipif(
- not MODELS and not any(os.path.exists(os.path.join(p, "config.json")) for p in (GEMMA, GLM, MUSE)),
- reason="no FREETOKEN_QWEN36_MODEL / FREETOKEN_QWEN3VL_MODEL / FREETOKEN_GEMMA4_MODEL / FREETOKEN_GLM53_MODEL / FREETOKEN_MUSE_MODEL checkpoint",
+ not MODELS and not any(os.path.exists(os.path.join(p, "config.json")) for p in (GEMMA, GLM, MUSE, MINIMAX)),
+ reason="no FREETOKEN_QWEN36_MODEL / FREETOKEN_QWEN3VL_MODEL / FREETOKEN_GEMMA4_MODEL / FREETOKEN_GLM53_MODEL / FREETOKEN_MUSE_MODEL / FREETOKEN_MINIMAX_M3_MODEL checkpoint",
),
]
@@ -189,3 +190,27 @@ def test_muse_wraps_the_pad_span_in_image_start_end_and_clamps_to_a_token_maximu
override = TokenizeManager(tokenizer, get_mm_processor(MUSE, MultimodalConfig(processor_kwargs={"max_image_tokens": 256})))
small = override.tokenize([_msg([_png(640, 400)])])[0].mm_items[0]
assert small.grid_thw == [1, 24, 40] and small.num_tokens == 240
+
+
[email protected](not os.path.exists(os.path.join(MINIMAX, "config.json")), reason="FREETOKEN_MINIMAX_M3_MODEL not set")
+def test_minimax_wraps_the_pad_span_in_start_end_tokens():
+ from freetoken.mm.config import MultimodalConfig
+ from freetoken.mm.processor import get_mm_processor
+ from freetoken.mm.processors.minimax_m3 import IMAGE_END_ID, IMAGE_START_ID
+ from freetoken.tokenizer.tokenize import TokenizeManager
+ from freetoken.utils.hf import load_tokenizer
+
+ tokenizer = load_tokenizer(MINIMAX)
+ # the processor hardcodes the wrapper ids the checkpoint config does not name
+ assert tokenizer.convert_tokens_to_ids("]<]start of image[>[") == IMAGE_START_ID
+ assert tokenizer.convert_tokens_to_ids("]<]end of image[>[") == IMAGE_END_ID
+ r = TokenizeManager(tokenizer, get_mm_processor(MINIMAX)).tokenize([_msg([_png(640, 400)])])[0]
+ item = r.mm_items[0]
+ ((start, end),) = item.offsets
+ ids = r.input_ids
+ _, h, w = item.grid_thw
+ assert ids[start - 1].item() == IMAGE_START_ID and ids[end].item() == IMAGE_END_ID
+ assert end - start == h * w // 4 == 322 and r.mrope_positions is None
+ assert bool((ids[start:end] == item.pad_value).all())
+ budgeted = TokenizeManager(tokenizer, get_mm_processor(MINIMAX, MultimodalConfig(image_max_tokens=64))).tokenize([_msg([_png(640, 400)])])[0]
+ assert budgeted.mm_items[0].num_tokens <= 64