FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(minimax_m3): serve image input on MiniMax-M3 (#480)
Xiaoze Fan68a81ffddaea
diff --git a/docs/models.md b/docs/models.mdindex d2c811ba..16bedb41 100644--- a/docs/models.md+++ b/docs/models.md@@ -17,6 +17,7 @@ for them; other checkpoints of the same architectures work too.| gpt-oss | [openai/gpt-oss-120b](https://huggingface.co/openai/gpt-oss-120b), [openai/gpt-oss-20b](https://huggingface.co/openai/gpt-oss-20b) || Gemma-4 | [google/gemma-4-26B-A4B-it](https://huggingface.co/google/gemma-4-26B-A4B-it), [nvidia/Gemma-4-26B-A4B-NVFP4](https://huggingface.co/nvidia/Gemma-4-26B-A4B-NVFP4), [google/gemma-4-12B-it](https://huggingface.co/google/gemma-4-12B-it), [nvidia/Gemma-4-31B-IT-NVFP4](https://huggingface.co/nvidia/Gemma-4-31B-IT-NVFP4) .. || MiniMax-M2.5 | [nvidia/MiniMax-M2.5-NVFP4](https://huggingface.co/nvidia/MiniMax-M2.5-NVFP4) |+| MiniMax-M3 | [nvidia/MiniMax-M3-NVFP4](https://huggingface.co/nvidia/MiniMax-M3-NVFP4) || Muse-Glimmer | [meta-models/Muse-Glimmer-30B](https://huggingface.co/meta-models/Muse-Glimmer-30B), [RedHatAI/Muse-Glimmer-30B-NVFP4](https://huggingface.co/RedHatAI/Muse-Glimmer-30B-NVFP4) |### Image input@@ -31,6 +32,7 @@ These families accept image input by default; pass `--text-model-only` to skip t| Gemma-4 12B (`gemma4_unified`: linear patch embedder, resident under either placement) | same budgets, one 48x48 super-patch per soft token | same as the tower releases | same || GLM-5.3-Flash (`glm5_next`: ViT tower, streamed under `--mm-encoder-weights host`) | one token per 28x28 pixels of the resized image, dynamic resolution on a canvas zero-padded to a 28-multiple | token counts, passed through as the processor's `min_image_tokens` / `max_image_tokens`; checkpoint defaults 16 to 8000 tokens | `{"max_image_tokens": 2048}` || Muse-Glimmer-30B (windowed ViT tower, streamed under `--mm-encoder-weights host`) | one token per 28x28 pixels of the resized image, aspect ratio kept under a token cap; checkpoint default 4096 tokens | the maximum is the cap (`max_image_tokens`); the minimum has no effect | `{"max_image_tokens": 1024}` |+| MiniMax-M3 (`minimax_m3`: CLIP-style ViT tower, streamed under `--mm-encoder-weights host`) | one token per 28x28 pixels of the resized image, dynamic resolution | pixel areas in `size.shortest_edge` / `longest_edge`; checkpoint defaults 4 to 576 tokens | `{"size": {"longest_edge": 1048576}}` |## MoE strategiesdiff --git a/python/freetoken/attention/m3_sparse.py b/python/freetoken/attention/m3_sparse.pyindex 19515f90..82a98a5a 100644--- a/python/freetoken/attention/m3_sparse.py+++ b/python/freetoken/attention/m3_sparse.py@@ -100,7 +100,7 @@ def _pick_inner_backend(block_size: int) -> str:from freetoken.engine.engine import _resolve_auto_attention_backend- name = _resolve_auto_attention_backend(frozenset({AttnType.FULL}), False)+ name = _resolve_auto_attention_backend(frozenset({AttnType.FULL}))if not _page_ok(name):# trtllm (the sm_100 first pick) pins 16/32/64-token pages; walk the rest# of the SAME tree (same arch gates, same requirement probes) with thediff --git a/python/freetoken/mm/processors/minimax_m3.py b/python/freetoken/mm/processors/minimax_m3.pynew file mode 100644index 00000000..660d72c2--- /dev/null+++ b/python/freetoken/mm/processors/minimax_m3.py@@ -0,0 +1,92 @@+"""MiniMax-M3 image processor: patch-grid ViT with a 2x2 merger, a pixel-area budget and 1-D rope."""++from __future__ import annotations++import struct+from typing import Any++import torch++from freetoken.message import MMItem+from freetoken.mm import mm_pad_value+from freetoken.mm.config import MultimodalConfig+from freetoken.mm.processor import MMProcessor, PromptReplacement, content_hash++# the config names only the image token; the reference processor reads these two from the tokenizer's fixed added tokens+IMAGE_START_ID = 200029 # ]<]start of image[>[+IMAGE_END_ID = 200030 # ]<]end of image[>[+++def _compression(vc: Any, key: str) -> int:+ """The native vision config carries the merge sizes flat; the checkpoint's own config nests them under img_token_compression_config."""+ value = getattr(vc, key, None)+ return int(vc.img_token_compression_config[key] if value is None else value)+++class MiniMaxM3MMProcessor(MMProcessor):+ def __init__(self, hf_config: Any, model_path: str, mm: MultimodalConfig) -> None:+ super().__init__(model_path, mm)+ vc = hf_config.vision_config+ # the native config maps image_token_id onto the checkpoint's image_token_index+ self.image_token_id = getattr(hf_config, "image_token_id", getattr(hf_config, "image_token_index", None))+ # the template renders one image token per image; the wrappers come from the replacement+ self.placeholder = [self.image_token_id]+ self.merge = _compression(vc, "spatial_merge_size")+ self.pixels_per_token = (vc.patch_size * self.merge) ** 2+ self.patch_dim = vc.num_channels * _compression(vc, "temporal_patch_size") * vc.patch_size**2++ def get_mm_processor_kwargs(self, mm: MultimodalConfig) -> dict[str, Any]:+ kwargs: dict[str, Any] = {"return_tensors": "pt"}+ if mm.image_min_tokens is not None or mm.image_max_tokens is not None:+ # the budget travels as pixel areas in size.shortest_edge / longest_edge+ size = dict(self._image_processor().size)+ if mm.image_min_tokens is not None:+ size["shortest_edge"] = mm.image_min_tokens * self.pixels_per_token+ if mm.image_max_tokens is not None:+ size["longest_edge"] = mm.image_max_tokens * self.pixels_per_token+ kwargs["size"] = size+ return {**kwargs, **mm.processor_kwargs}++ def process(self, images: list[Any]) -> list[MMItem]:+ processor = self._image_processor()+ kwargs = self.get_mm_processor_kwargs(self.mm)+ items: list[MMItem] = []+ for pil in images:+ out = processor(images=pil, **kwargs)+ pixel_values = out["pixel_values"].to(torch.bfloat16)+ grid = out["image_grid_thw"][0].tolist()+ t, h, w = int(grid[0]), int(grid[1]), int(grid[2])+ if t != 1:+ raise ValueError("video/temporal input is not supported")+ # hash the bf16 wire buffer: half the bytes, and the exact tensor the cache and radix key agree on+ item_hash = content_hash(pixel_values, struct.pack("<3i", t, h, w))+ items.append(+ MMItem(+ modality="image",+ hash=item_hash,+ pad_value=mm_pad_value(item_hash),+ offsets=[],+ feature=pixel_values,+ model_specific_data={"grid_thw": [t, h, w]},+ )+ )+ return items++ def prompt_replacement(self, item: MMItem) -> PromptReplacement:+ t, h, w = item.grid_thw+ full = [IMAGE_START_ID] + [self.image_token_id] * ((t * h * w) // (self.merge * self.merge)) + [IMAGE_END_ID]+ return PromptReplacement.select_token_id(full, self.image_token_id)++ def dummy_items(self, dtype: torch.dtype, device: torch.device) -> list[MMItem]:+ merge = self.merge+ return [MMItem(+ modality="image",+ hash=0,+ pad_value=0,+ offsets=[[0, 1]],+ feature=torch.zeros(merge * merge, self.patch_dim, dtype=dtype, device=device),+ model_specific_data={"grid_thw": [1, merge, merge]},+ )]+++__all__ = ["IMAGE_END_ID", "IMAGE_START_ID", "MiniMaxM3MMProcessor"]diff --git a/python/freetoken/models/minimax_m3/__init__.py b/python/freetoken/models/minimax_m3/__init__.pyindex cfcebb6f..ca66b680 100644--- a/python/freetoken/models/minimax_m3/__init__.py+++ b/python/freetoken/models/minimax_m3/__init__.py@@ -1,5 +1,6 @@-from .config import parse_config-from .model import MiniMaxM3ForCausalLM+from .config import VisionConfig, parse_config, parse_vision_config+from .model import MiniMaxM3ForCausalLM, MiniMaxM3ForConditionalGeneration+from .vision import MiniMaxM3VisionModelfrom .weight import (nvfp4_expert_spec,iter_weights,@@ -8,6 +9,10 @@ from .weight import (__all__ = ["nvfp4_expert_spec","MiniMaxM3ForCausalLM",+ "MiniMaxM3ForConditionalGeneration",+ "MiniMaxM3VisionModel",+ "VisionConfig","parse_config",+ "parse_vision_config","iter_weights",]diff --git a/python/freetoken/models/minimax_m3/config.py b/python/freetoken/models/minimax_m3/config.pyindex 7df81d79..a54d48e9 100644--- a/python/freetoken/models/minimax_m3/config.py+++ b/python/freetoken/models/minimax_m3/config.py@@ -1,10 +1,9 @@"""Engine-facing config for MiniMax-M3 (``minimax_m3``).The checkpoint is a multimodal wrapper (``model_type=minimax_m3_vl``): the text tower-lives in ``text_config`` and the weights carry a ``language_model.`` prefix. FreeToken-serves the text tower; the ViT vision stack (``vision_tower.`` / projector) is skipped-at load like the other VL checkpoints' (``VISION_KEY_PREFIXES``) -- multimodal input-is future work, so ``ModelConfig.vision_config`` stays None here.+lives in ``text_config`` and the weights carry a ``language_model.`` prefix. Its vision+section becomes ``ModelConfig.vision_config`` for the tower in vision.py; an engine that+builds no vision encoder hands parse_config a config without that section.Attention is GQA with block-sparse selection on the trailing layers: parse_configdeclares ONE full-attention group over all layers carrying ``mla=False`` plus the@@ -27,6 +26,7 @@ their swigluoai activation restricts the NVFP4 GEMM backend to the Triton kernelfrom __future__ import annotationsimport os+from dataclasses import dataclassfrom typing import Anyfrom freetoken.models.config import (@@ -41,6 +41,54 @@ def _text_config(hf_config: Any) -> Any:return getattr(hf_config, "text_config", None) or hf_config+@dataclass(frozen=True)+class VisionConfig:+ hidden_size: int+ num_layers: int+ num_heads: int+ intermediate_size: int+ num_channels: int+ patch_size: int+ temporal_patch_size: int+ spatial_merge_size: int+ layer_norm_eps: float+ rope_theta: float+ projector_hidden_size: int+ text_hidden_size: int+++def _compression(vc: Any, key: str) -> int:+ """The native vision config carries the merge sizes flat; the checkpoint's own config nests them under img_token_compression_config."""+ value = getattr(vc, key, None)+ return int(vc.img_token_compression_config[key] if value is None else value)+++def parse_vision_config(hf_config: Any) -> VisionConfig | None:+ """None when the config carries no vision section, which is how a text-only engine asks for no tower."""+ vc = getattr(hf_config, "vision_config", None)+ if vc is None:+ return None+ if vc.hidden_act != "gelu":+ raise NotImplementedError(f"minimax_m3 vision tower activation {vc.hidden_act!r}; only gelu is implemented")+ # the native config moves rope_theta into rope_parameters; the checkpoint's own config keeps it flat+ rope_params = getattr(vc, "rope_parameters", None) or {}+ text_hidden_size = _text_config(hf_config).hidden_size+ return VisionConfig(+ hidden_size=vc.hidden_size,+ num_layers=vc.num_hidden_layers,+ num_heads=vc.num_attention_heads,+ intermediate_size=vc.intermediate_size,+ num_channels=vc.num_channels,+ patch_size=vc.patch_size,+ temporal_patch_size=_compression(vc, "temporal_patch_size"),+ spatial_merge_size=_compression(vc, "spatial_merge_size"),+ layer_norm_eps=vc.layer_norm_eps,+ rope_theta=float(rope_params["rope_theta"] if "rope_theta" in rope_params else vc.rope_theta),+ projector_hidden_size=getattr(hf_config, "projector_hidden_size", None) or text_hidden_size,+ text_hidden_size=text_hidden_size,+ )++# House rule: an env switch that changes WHAT IS SERVED must leave a server-log# trace. parse_config runs a few times per process (engine + weight loader), so# each resolved-mode line is logged once per distinct value.@@ -179,7 +227,10 @@ def parse_config(hf_config: Any) -> ModelConfig:dense_quant="mxfp8" if dense_mxfp8 else "none",lm_head_quant="none",m3_args=args,+ vision_config=parse_vision_config(hf_config),+ # the native config maps image_token_id onto the checkpoint's image_token_index+ image_token_id=getattr(hf_config, "image_token_id", getattr(hf_config, "image_token_index", None)),)-__all__ = ["parse_config"]+__all__ = ["VisionConfig", "parse_config", "parse_vision_config"]diff --git a/python/freetoken/models/minimax_m3/model.py b/python/freetoken/models/minimax_m3/model.pyindex e8b6e5d0..d78fc10a 100644--- a/python/freetoken/models/minimax_m3/model.py+++ b/python/freetoken/models/minimax_m3/model.py@@ -11,14 +11,16 @@ from freetoken.layers import (ParallelLMHead,VocabParallelEmbedding,)-from freetoken.models.blocks import BaseLLMModel+from freetoken.models.blocks import BaseLLMModel, embed_input_idsfrom freetoken.utils import nvtx_annotatefrom .attention import MiniMaxM3Attentionfrom .mlp import MiniMaxM3MLPfrom .moe import MiniMaxM3SparseMoeBlock+from .vision import MiniMaxM3VisionModelif TYPE_CHECKING:+ from freetoken.message import MMItemfrom freetoken.models.config import ModelConfig@@ -79,7 +81,7 @@ class MiniMaxM3Model(BaseOP):self.norm = GemmaPlusOneRMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)def forward(self, input_ids: torch.Tensor) -> torch.Tensor:- x = self.embed_tokens.forward(input_ids)+ x = embed_input_ids(self.embed_tokens, input_ids, get_global_ctx().batch)residual: torch.Tensor | None = Nonefor layer in self.layers.op_list:x, residual = layer.forward(x, residual)@@ -105,4 +107,17 @@ class MiniMaxM3ForCausalLM(BaseLLMModel):return self.lm_head.forward(output)-__all__ = ["MiniMaxM3ForCausalLM"]+class MiniMaxM3ForConditionalGeneration(MiniMaxM3ForCausalLM):+ def __init__(self, config: ModelConfig):+ super().__init__(config)+ if config.is_multimodal:+ self.vision_tower = MiniMaxM3VisionModel(config.vision_config)++ def place_encoder_weights(self, mode: str) -> None:+ self.vision_tower.place_weights(mode)++ def encode(self, item: MMItem) -> torch.Tensor:+ return self.vision_tower.forward(item.feature, item.grid_thw)+++__all__ = ["MiniMaxM3ForCausalLM", "MiniMaxM3ForConditionalGeneration"]diff --git a/python/freetoken/models/minimax_m3/vision.py b/python/freetoken/models/minimax_m3/vision.pynew file mode 100644index 00000000..97074ec8--- /dev/null+++ b/python/freetoken/models/minimax_m3/vision.py@@ -0,0 +1,161 @@+"""MiniMax-M3 vision tower: Conv3d patch embedding, pre-LayerNorm CLIP blocks with a 3-axis rope, a GELU projector and a 2x2 patch-merge MLP into the text width."""++from __future__ import annotations++from typing import TYPE_CHECKING, List++import torch+import torch.nn.functional as F+from freetoken.layers import BaseOP, LayerNorm, LinearReplicated, OPList+from freetoken.models.weight_stream import BlockWeightStreamer++if TYPE_CHECKING:+ from .config import VisionConfig+++class _Conv3dWeight(BaseOP):+ def __init__(self, out_ch: int, shape: tuple[int, ...]):+ self.weight = torch.empty(out_ch, *shape)++ forward = None+++class MiniMaxM3VisionEmbeddings(BaseOP):+ """Kept as a real Conv3d (kernel == stride): an equivalent unfolded matmul rounds differently."""++ def __init__(self, vc: VisionConfig):+ self._shape = (vc.num_channels, vc.temporal_patch_size, vc.patch_size, vc.patch_size)+ self.patch_embedding = _Conv3dWeight(vc.hidden_size, self._shape)++ def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:+ x = pixel_values.view(-1, *self._shape).to(self.patch_embedding.weight.dtype)+ return F.conv3d(x, self.patch_embedding.weight, stride=self._shape[1:]).view(x.shape[0], -1)+++def _rope_table(grid_thw: List[int], merge: int, head_dim: int, theta: float, device: torch.device) -> tuple[torch.Tensor, torch.Tensor]:+ """Per-token [cos | sin] rows over the rotated dims: the even part of the head split into three equal t/h/w bands, the remainder passes through."""+ t, h, w = grid_thw+ axis_dim = 2 * (((2 * (head_dim // 2)) // 3) // 2)+ inv_freq = 1.0 / (theta ** (torch.arange(0, axis_dim, 2, dtype=torch.float32, device=device) / axis_dim))+ hpos, wpos = torch.meshgrid(torch.arange(h, device=device), torch.arange(w, device=device), indexing="ij")+ # block-major over the merge blocks, the order the processor flattens patches in+ block = (h // merge, merge, w // merge, merge)+ hpos = hpos.reshape(block).transpose(1, 2).flatten().repeat(t)+ wpos = wpos.reshape(block).transpose(1, 2).flatten().repeat(t)+ tpos = torch.arange(t, device=device).repeat_interleave(h * w)+ coords = torch.stack([tpos, hpos, wpos], dim=-1).float()+ freqs = (coords.unsqueeze(-1) * inv_freq).flatten(1)+ cache = torch.cat((freqs.cos(), freqs.sin()), dim=-1).contiguous()+ return cache, torch.arange(coords.shape[0], dtype=torch.int32, device=device)+++class MiniMaxM3VisionAttention(BaseOP):+ def __init__(self, vc: VisionConfig):+ self.num_heads = vc.num_heads+ self.head_dim = vc.hidden_size // vc.num_heads+ self.qkv = LinearReplicated(vc.hidden_size, 3 * vc.hidden_size, has_bias=True)+ self.out_proj = LinearReplicated(vc.hidden_size, vc.hidden_size, has_bias=True)++ def forward(self, x: torch.Tensor, cache: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel.triton.rope import apply_rope_with_cos_sin_cache_inplace++ S, H, D = x.shape[0], self.num_heads, self.head_dim+ q, k, v = self.qkv.forward(x).view(S, 3, H * D).unbind(1)+ apply_rope_with_cos_sin_cache_inplace(positions, q, k, D, cache, is_neox=True)+ # the fused SDPA kernels want a batch dim; 3-D inputs fall back to the O(S^2) math path+ q, k, v = (t.view(S, H, D).transpose(0, 1).unsqueeze(0) for t in (q, k, v))+ o = F.scaled_dot_product_attention(q, k, v)+ return self.out_proj.forward(o[0].transpose(0, 1).reshape(S, H * D))+++class MiniMaxM3VisionMLP(BaseOP):+ def __init__(self, vc: VisionConfig):+ self.fc1 = LinearReplicated(vc.hidden_size, vc.intermediate_size, has_bias=True)+ self.fc2 = LinearReplicated(vc.intermediate_size, vc.hidden_size, has_bias=True)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ return self.fc2.forward(F.gelu(self.fc1.forward(x)))+++class MiniMaxM3VisionEncoderLayer(BaseOP):+ def __init__(self, vc: VisionConfig):+ self.self_attn = MiniMaxM3VisionAttention(vc)+ self.layer_norm1 = LayerNorm(vc.hidden_size, vc.layer_norm_eps)+ self.mlp = MiniMaxM3VisionMLP(vc)+ self.layer_norm2 = LayerNorm(vc.hidden_size, vc.layer_norm_eps)++ def forward(self, x: torch.Tensor, cache: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:+ x = x + self.self_attn.forward(self.layer_norm1.forward(x), cache, positions)+ return x + self.mlp.forward(self.layer_norm2.forward(x))+++class MiniMaxM3VisionEncoder(BaseOP):+ def __init__(self, vc: VisionConfig):+ self.layers = OPList([MiniMaxM3VisionEncoderLayer(vc) for _ in range(vc.num_layers)])++ forward = None+++class MiniMaxM3VisionTransformer(BaseOP):+ """Patches of one image -> [t * h * w, hidden]; the checkpoint has no post layernorm."""++ def __init__(self, vc: VisionConfig):+ self.embeddings = MiniMaxM3VisionEmbeddings(vc)+ self.pre_layrnorm = LayerNorm(vc.hidden_size, vc.layer_norm_eps)+ self.encoder = MiniMaxM3VisionEncoder(vc)+ self._vc = vc+ self._streamer: BlockWeightStreamer | None = None++ def place_weights(self, mode: str) -> None:+ """gpu: every tensor resident; host: encoder layer tensors in pinned banks streamed two layers at a time."""+ if mode == "host" and self._streamer is None:+ self._streamer = BlockWeightStreamer(self.encoder.layers.op_list, self.embeddings.patch_embedding.weight.device)+ elif mode == "gpu" and self._streamer is not None:+ self._streamer.unstream()+ self._streamer = None+ elif mode not in ("gpu", "host"):+ raise ValueError(f"unknown vision weight placement {mode!r}")++ def _layers(self):+ if self._streamer is None:+ return enumerate(self.encoder.layers.op_list)+ return self._streamer.blocks(self.encoder.layers.op_list)++ def forward(self, pixel_values: torch.Tensor, grid_thw: List[int]) -> torch.Tensor:+ vc = self._vc+ device = self.embeddings.patch_embedding.weight.device+ x = self.pre_layrnorm.forward(self.embeddings.forward(pixel_values.to(device)))+ cache, positions = _rope_table(grid_thw, vc.spatial_merge_size, vc.hidden_size // vc.num_heads, vc.rope_theta, device)+ for _, layer in self._layers():+ x = layer.forward(x, cache, positions)+ return x+++class MiniMaxM3ProjectorMLP(BaseOP):+ def __init__(self, in_size: int, mid_size: int, out_size: int):+ self.linear_1 = LinearReplicated(in_size, mid_size, has_bias=True)+ self.linear_2 = LinearReplicated(mid_size, out_size, has_bias=True)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ return self.linear_2.forward(F.gelu(self.linear_1.forward(x)))+++class MiniMaxM3VisionModel(BaseOP):+ """Pixels of one image -> soft tokens [t * h * w / merge^2, text_hidden]."""++ def __init__(self, vc: VisionConfig):+ self.vision_model = MiniMaxM3VisionTransformer(vc)+ self.multi_modal_projector = MiniMaxM3ProjectorMLP(vc.hidden_size, vc.projector_hidden_size, vc.text_hidden_size)+ self.patch_merge_mlp = MiniMaxM3ProjectorMLP(vc.spatial_merge_size**2 * vc.text_hidden_size, vc.projector_hidden_size, vc.text_hidden_size)+ self._merge = vc.spatial_merge_size**2++ def place_weights(self, mode: str) -> None:+ self.vision_model.place_weights(mode)++ @torch.inference_mode()+ def forward(self, pixel_values: torch.Tensor, grid_thw: List[int]) -> torch.Tensor:+ x = self.multi_modal_projector.forward(self.vision_model.forward(pixel_values, grid_thw))+ return self.patch_merge_mlp.forward(x.view(x.shape[0] // self._merge, -1))+++__all__ = ["MiniMaxM3VisionModel"]diff --git a/python/freetoken/models/minimax_m3/weight.py b/python/freetoken/models/minimax_m3/weight.pyindex 3aea2347..b3c928e2 100644--- a/python/freetoken/models/minimax_m3/weight.py+++ b/python/freetoken/models/minimax_m3/weight.py@@ -3,7 +3,8 @@The checkpoint is the multimodal wrapper layout: every text-tower tensor carries a``language_model.`` prefix (``language_model.model.layers.N...``,``language_model.lm_head.weight``); the ViT stack (``vision_tower.`` /-``multi_modal_projector.`` / ``patch_merge_mlp.``) is never read (text-only serving).+``multi_modal_projector.`` / ``patch_merge_mlp.``) is read by ``_iter_vision`` only when+the engine builds the vision encoder (``include_vision``).Resident (non routed-expert) dense projections are MXFP8 in the checkpoint(fp8-e4m3 ``weight`` + uint8 e8m0 block-32 ``weight_scale_inv``): the fp8 weight and@@ -135,12 +136,31 @@ def _emit_fused()+_VISION_ATTN = "vision_tower.vision_model.encoder.layers.{}.self_attn"+++def _iter_vision(reader: _ShardReader, weight_map: dict, num_layers: int) -> Iterator[tuple[str, torch.Tensor]]:+ """The tower under its checkpoint names with q/k/v fused, the projector and patch-merge MLP under the tower prefix; all bf16 (the patch embedding is stored fp32)."""+ for name in weight_map:+ if name.startswith(("multi_modal_projector.", "patch_merge_mlp.")):+ yield "vision_tower." + name, reader.get(name).to(torch.bfloat16)+ elif name.startswith("vision_tower.") and ".self_attn." not in name:+ yield name, reader.get(name).to(torch.bfloat16)+ for layer in range(num_layers):+ attn = _VISION_ATTN.format(layer)+ for kind in ("weight", "bias"):+ parts = [reader.get(f"{attn}.{proj}.{kind}") for proj in ("q_proj", "k_proj", "v_proj")]+ yield f"{attn}.qkv.{kind}", torch.cat(parts, dim=0).to(torch.bfloat16)+ yield f"{attn}.out_proj.{kind}", reader.get(f"{attn}.out_proj.{kind}").to(torch.bfloat16)++def iter_weights(model_path: str,device: torch.device,*,include_moe_experts: bool,include_non_moe: bool,+ include_vision: bool = True,) -> Iterator[tuple[str, torch.Tensor]]:assert not include_moe_experts, ("MiniMax-M3 stores routed experts as NVFP4 and only supports the offload MoE "@@ -234,6 +254,8 @@ def iter_weights(yield "model.embed_tokens.weight", reader.get("language_model.model.embed_tokens.weight")yield "model.norm.weight", reader.get("language_model.model.norm.weight")yield "lm_head.weight", reader.get("language_model.lm_head.weight")+ if include_vision and config.vision_config is not None:+ yield from _iter_vision(reader, weight_map, config.vision_config.num_layers)finally:reader.close()diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex 1d7ac3e5..2a4ced10 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -82,6 +82,8 @@ _MUSE_GLIMMER_ENCODERS = (EncoderSpec("vision", "vision_config", ("image",)),)_MINIMAX_M3_PACKED = _DENSE_PACKED + (("index_qk_proj", ("index_q_proj", "index_k_proj")),) + _EXPERTS_W123_PACKED+_MINIMAX_M3_PROCESSOR = "freetoken.mm.processors.minimax_m3:MiniMaxM3MMProcessor"+_MINIMAX_M3_ENCODERS = (EncoderSpec("vision", "vision_config", ("image",)),)_MODEL_REGISTRY: dict[str, ModelSpec] = {"LlamaForCausalLM": ModelSpec(@@ -127,14 +129,17 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {packed_modules_mapping=_DENSE_PACKED + _EXPERTS_W123_PACKED,),# MiniMax-M3 (model_type minimax_m3_vl): multimodal wrapper config (text tower in- # text_config, weights under language_model.); served text-only. GQA + block-sparse- # attention (lightning indexer, top-k 128-token blocks) on the trailing layers,- # sigmoid/bias-routed NVFP4 experts + MXFP8 shared expert, swigluoai activation.+ # text_config, weights under language_model.); the CLIP-style tower under vision_tower.+ # serves image input. GQA + block-sparse attention (lightning indexer, top-k 128-token+ # blocks) on the trailing layers, sigmoid/bias-routed NVFP4 experts + MXFP8 shared+ # expert, swigluoai activation."MiniMaxM3SparseForConditionalGeneration": ModelSpec("freetoken.models.minimax_m3",- "MiniMaxM3ForCausalLM",+ "MiniMaxM3ForConditionalGeneration",checkpoint_roots=(("model", "language_model.model"), ("lm_head", "language_model.lm_head")),packed_modules_mapping=_MINIMAX_M3_PACKED,+ mm_processor=_MINIMAX_M3_PROCESSOR,+ encoders=_MINIMAX_M3_ENCODERS,),# Text-only sibling (the text_config's own architectures entry)."MiniMaxM3SparseForCausalLM": ModelSpec(diff --git a/tests/README.md b/tests/README.mdindex 1f90f6a3..26dba633 100644--- a/tests/README.md+++ b/tests/README.md@@ -54,6 +54,7 @@ checkpoint is set:| `FREETOKEN_GEMMA4_UNIFIED_MODEL` | `models/test_gemma4_vision.py` — local gemma-4-12B `gemma4_unified` checkpoint (embedder parity) || `FREETOKEN_GLM53_MODEL` | `tokenizer/test_mm_tokenize.py` — local GLM-5.3-Flash checkpoint (image-token expansion) || `FREETOKEN_MUSE_MODEL` | `models/test_muse_glimmer_vision.py`, `tokenizer/test_mm_tokenize.py` — local Muse-Glimmer-30B checkpoint (tower parity, image_start/image_end expansion) |+| `FREETOKEN_MINIMAX_M3_MODEL` | `tokenizer/test_mm_tokenize.py` — local MiniMax-M3 checkpoint (start/end expansion, token budget) || `FREETOKEN_TEST_MOE_CACHE_SIZE` | `e2e/test_aime.py` — >0 switches to the offload MoE backend with this cache size || `FREETOKEN_TEST_MEM_RATIO` | `e2e/test_aime.py` — offload-mode memory_ratio (default `0.9`) || `FREETOKEN_REBUILD_TEST_MODEL` | `e2e/test_cache_rebuild.py` — a SMALL local model dir; boots a real server (falls back to `FREETOKEN_TEST_MODEL`) |diff --git a/tests/mm/test_processor.py b/tests/mm/test_processor.pyindex c3049256..a9a50e3c 100644--- a/tests/mm/test_processor.py+++ b/tests/mm/test_processor.py@@ -15,6 +15,7 @@ from freetoken.mm.processor import MMProcessor, PromptReplacement, image_positiofrom freetoken.mm.processors.gemma4 import Gemma4MMProcessor, Gemma4UnifiedMMProcessorfrom freetoken.mm.processors.glm5_next import Glm5NextMMProcessorfrom freetoken.mm.processors.muse_glimmer import MuseGlimmerMMProcessor+from freetoken.mm.processors.minimax_m3 import IMAGE_END_ID, IMAGE_START_ID, MiniMaxM3MMProcessorfrom freetoken.mm.processors.qwen_vl import QwenVLMMProcessorPLACEHOLDER = 7@@ -350,6 +351,45 @@ def test_muse_wraps_the_pads_in_image_start_end_and_takes_only_a_token_maximum()assert dummy.feature.shape == (4, 1176) and dummy.num_tokens == 1+def _minimax_config():+ # the checkpoint's own config shape: image_token_index, merge sizes under img_token_compression_config+ return SimpleNamespace(+ architectures=["MiniMaxM3SparseForConditionalGeneration"],+ image_token_index=200025,+ vision_config=SimpleNamespace(patch_size=14, num_channels=3, img_token_compression_config={"spatial_merge_size": 2, "temporal_patch_size": 2}),+ text_config=SimpleNamespace(vocab_size=200064),+ )+++def test_minimax_token_budget_becomes_pixel_bounds_and_the_span_gets_start_end_tokens():+ calls = []++ class _FakeImageProcessor:+ size = {"shortest_edge": 3136, "longest_edge": 451584}++ def __call__(self, images, **kwargs):+ calls.append(kwargs)+ return {"pixel_values": torch.zeros(16, 1176), "image_grid_thw": torch.tensor([[1, 4, 4]])}++ mm = MultimodalConfig(image_min_tokens=8, image_max_tokens=256, processor_kwargs={"do_convert_rgb": False})+ proc = MiniMaxM3MMProcessor(_minimax_config(), "/nonexistent", mm)+ proc._image_processor = lambda: _FakeImageProcessor()+ (item,) = proc.process([object()])+ # 784 pixels per token: patch 14 x merge 2, squared+ assert calls == [{"return_tensors": "pt", "size": {"shortest_edge": 8 * 784, "longest_edge": 256 * 784}, "do_convert_rgb": False}]+ assert item.grid_thw == [1, 4, 4] and item.feature.dtype == torch.bfloat16+ repl = proc.prompt_replacement(item)+ assert proc.placeholder == [200025] and repl.full == [IMAGE_START_ID] + [200025] * 4 + [IMAGE_END_ID] and repl.embed_spans() == [[1, 5]]+ assert proc.positions(10, [item]) is None+ proc = MiniMaxM3MMProcessor(_minimax_config(), "/nonexistent", MultimodalConfig())+ proc._image_processor = lambda: _FakeImageProcessor()+ proc.process([object()])+ assert calls[-1] == {"return_tensors": "pt"}+ (dummy,) = proc.dummy_items(torch.bfloat16, torch.device("cpu"))+ dummy.validate()+ assert dummy.feature.shape == (4, 1176) and dummy.num_tokens == 1++def test_registry_resolves_by_architecture(monkeypatch):import freetoken.utilsfrom freetoken.mm.processor import get_mm_processor@@ -360,6 +400,7 @@ def test_registry_resolves_by_architecture(monkeypatch):"/gemma-unified": _gemma_unified_config(),"/glm": _glm_config(),"/muse": _muse_config(),+ "/minimax": _minimax_config(),"/text-only": _hf_config(vision=False),"/unknown-vlm": _hf_config(arch="SomeOtherForConditionalGeneration"),}@@ -370,6 +411,7 @@ def test_registry_resolves_by_architecture(monkeypatch):assert isinstance(get_mm_processor("/gemma-unified"), Gemma4UnifiedMMProcessor)assert isinstance(get_mm_processor("/glm"), Glm5NextMMProcessor)assert isinstance(get_mm_processor("/muse"), MuseGlimmerMMProcessor)+ assert isinstance(get_mm_processor("/minimax"), MiniMaxM3MMProcessor)assert get_mm_processor("/text-only") is Noneassert get_mm_processor("/unknown-vlm") is Noneassert get_mm_processor("/missing") is None # a config that fails to load means no visiondiff --git a/tests/models/test_minimax_m3.py b/tests/models/test_minimax_m3.pyindex 1062261e..dc0ab90d 100644--- a/tests/models/test_minimax_m3.py+++ b/tests/models/test_minimax_m3.py@@ -14,7 +14,7 @@ import torchfrom freetoken.layers.quantization import MoEConfigfrom freetoken.attention.base import AttnType-from freetoken.models.minimax_m3.config import parse_config+from freetoken.models.minimax_m3.config import VisionConfig, parse_configclass _Cfg:@@ -213,16 +213,51 @@ def test_sparse_ablation_env(monkeypatch):assert resolve_pool_class(cfg) is MHAKVCache+def _vision_section(native: bool) -> _Cfg:+ section = {+ "hidden_size": 1280, "num_hidden_layers": 32, "num_attention_heads": 16, "intermediate_size": 5120,+ "num_channels": 3, "patch_size": 14, "hidden_act": "gelu", "layer_norm_eps": 1e-5,+ }+ if native:+ section.update(temporal_patch_size=2, spatial_merge_size=2, rope_parameters={"rope_theta": 10000.0, "rope_type": "axial"})+ else:+ section.update(rope_theta=10000.0, img_token_compression_config={"spatial_merge_size": 2, "temporal_patch_size": 2})+ return _Cfg(section)+++def test_parse_config_vision_section_in_both_config_shapes(monkeypatch):+ monkeypatch.delenv("FREETOKEN_M3_MAX_LAYERS", raising=False)+ expected = VisionConfig(+ hidden_size=1280, num_layers=32, num_heads=16, intermediate_size=5120, num_channels=3, patch_size=14, temporal_patch_size=2,+ spatial_merge_size=2, layer_norm_eps=1e-5, rope_theta=10000.0, projector_hidden_size=6144, text_hidden_size=6144,+ )+ raw = _hf_config()+ raw.vision_config, raw.image_token_index, raw.projector_hidden_size = _vision_section(native=False), 200025, 6144+ native = _hf_config_native()+ native.vision_config, native.image_token_id, native.projector_hidden_size = _vision_section(native=True), 200025, 6144+ for cfg in (raw, native):+ parsed = parse_config(cfg)+ assert parsed.is_multimodal and parsed.vision_config == expected and parsed.image_token_id == 200025+ # a text-only engine hands over a config without the section+ assert not parse_config(_hf_config()).is_multimodal++def test_registry_resolves_both_architectures():from freetoken.models.register import get_model_specfor arch in ("MiniMaxM3SparseForConditionalGeneration", "MiniMaxM3SparseForCausalLM"):spec = get_model_spec(arch)assert spec.module == "freetoken.models.minimax_m3"+ multimodal = get_model_spec("MiniMaxM3SparseForConditionalGeneration")+ assert multimodal.model_cls == "MiniMaxM3ForConditionalGeneration"+ assert multimodal.mm_processor == "freetoken.mm.processors.minimax_m3:MiniMaxM3MMProcessor"+ assert [(e.kind, e.config_key, e.modalities) for e in multimodal.encoders] == [("vision", "vision_config", ("image",))]+ assert get_model_spec("MiniMaxM3SparseForCausalLM").encoders == ()-def test_auto_backend_resolution():+def test_auto_backend_resolution(monkeypatch):from freetoken.attention import attention_backend_info+ from freetoken.attention.m3_sparse import _pick_inner_backendfrom freetoken.engine.engine import _required_attn_types, _resolve_auto_attention_backendcfg = parse_config(_hf_config())@@ -230,6 +265,9 @@ def test_auto_backend_resolution():assert required == frozenset({AttnType.BSA})assert _resolve_auto_attention_backend(required) == "m3_sparse"assert attention_backend_info("m3_sparse").page_sizes == (128,)+ # the dense leading layers take a FULL backend from the same resolver, filtered to 128-token pages+ monkeypatch.delenv("FREETOKEN_M3_INNER_BACKEND", raising=False)+ assert _pick_inner_backend(128) in ("fa,fi", "fi", "triton")def test_nvfp4_experts_restricted_to_triton_for_swigluoai(monkeypatch):diff --git a/tests/models/test_minimax_m3_vision.py b/tests/models/test_minimax_m3_vision.pynew file mode 100644index 00000000..a2885b1a--- /dev/null+++ b/tests/models/test_minimax_m3_vision.py@@ -0,0 +1,65 @@+"""MiniMax-M3 vision tower: shapes and host streaming on a tiny random tower."""++from __future__ import annotations++import pytest+import torch++from freetoken.models.minimax_m3 import MiniMaxM3VisionModel, VisionConfig++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")++[email protected](autouse=True)+def _single_rank():+ from freetoken.distributed import set_tp_info, try_get_tp_info++ if try_get_tp_info() is None:+ set_tp_info(rank=0, size=1)+++def _tiny_vc():+ return VisionConfig(+ hidden_size=64, num_layers=2, num_heads=4, intermediate_size=128, num_channels=3, patch_size=4, temporal_patch_size=2,+ spatial_merge_size=2, layer_norm_eps=1e-5, rope_theta=10000.0, projector_hidden_size=96, text_hidden_size=48,+ )+++def _build(vc):+ torch.set_default_dtype(torch.bfloat16)+ try:+ with torch.device("cuda"):+ tower = MiniMaxM3VisionModel(vc)+ finally:+ torch.set_default_dtype(torch.float32)+ for p in tower.state_dict().values():+ p.normal_(0, 0.02)+ return tower+++def test_tiny_tower_merges_four_patches_per_token():+ tower = _build(_tiny_vc())+ feature = torch.randn(64, 3 * 2 * 4 * 4, dtype=torch.bfloat16, device="cuda")+ out = tower.forward(feature, [1, 8, 8])+ assert out.shape == (16, 48) and out.dtype == torch.bfloat16+ keys = tower.state_dict()+ assert keys["vision_model.embeddings.patch_embedding.weight"].shape == (64, 3, 2, 4, 4)+ assert keys["vision_model.encoder.layers.0.self_attn.qkv.bias"].shape == (192,)+ assert keys["patch_merge_mlp.linear_1.weight"].shape == (96, 4 * 48)+ assert not any(k.startswith("vision_model.post_layernorm") for k in keys)+++def test_host_streamed_weights_compute_the_same_output():+ tower = _build(_tiny_vc())+ feature = torch.randn(64, 3 * 2 * 4 * 4, dtype=torch.bfloat16, device="cuda")+ resident = tower.forward(feature, [1, 8, 8])+ keys = tower.state_dict()+ tower.place_weights("host")+ assert tower.state_dict().keys() == keys.keys()+ assert tower.state_dict()["vision_model.encoder.layers.0.self_attn.qkv.weight"].device.type == "cpu"+ assert tower.state_dict()["vision_model.pre_layrnorm.weight"].device.type == "cuda"+ for _ in range(2): # a second forward reuses the staging buffers behind the release events+ assert torch.equal(tower.forward(feature, [1, 8, 8]), resident)+ tower.place_weights("gpu")+ assert tower.state_dict()["vision_model.encoder.layers.0.self_attn.qkv.weight"].device.type == "cuda"+ assert torch.equal(tower.forward(feature, [1, 8, 8]), resident)diff --git a/tests/tokenizer/test_mm_tokenize.py b/tests/tokenizer/test_mm_tokenize.pyindex 05f883bd..8f2e0211 100644--- a/tests/tokenizer/test_mm_tokenize.py+++ b/tests/tokenizer/test_mm_tokenize.py@@ -1,4 +1,4 @@-"""Image tokenization against real Qwen VL, Gemma-4, GLM-5.3 and Muse-Glimmer checkpoints (skipped when absent)."""+"""Image tokenization against real Qwen VL, Gemma-4, GLM-5.3, Muse-Glimmer and MiniMax-M3 checkpoints (skipped when absent)."""from __future__ import annotations@@ -19,12 +19,13 @@ MODELS = [GEMMA = os.environ.get("FREETOKEN_GEMMA4_MODEL", "")GLM = os.environ.get("FREETOKEN_GLM53_MODEL", "")MUSE = os.environ.get("FREETOKEN_MUSE_MODEL", "")+MINIMAX = os.environ.get("FREETOKEN_MINIMAX_M3_MODEL", "")pytestmark = [pytest.mark.needs_weights,pytest.mark.skipif(- not MODELS and not any(os.path.exists(os.path.join(p, "config.json")) for p in (GEMMA, GLM, MUSE)),- reason="no FREETOKEN_QWEN36_MODEL / FREETOKEN_QWEN3VL_MODEL / FREETOKEN_GEMMA4_MODEL / FREETOKEN_GLM53_MODEL / FREETOKEN_MUSE_MODEL checkpoint",+ not MODELS and not any(os.path.exists(os.path.join(p, "config.json")) for p in (GEMMA, GLM, MUSE, MINIMAX)),+ reason="no FREETOKEN_QWEN36_MODEL / FREETOKEN_QWEN3VL_MODEL / FREETOKEN_GEMMA4_MODEL / FREETOKEN_GLM53_MODEL / FREETOKEN_MUSE_MODEL / FREETOKEN_MINIMAX_M3_MODEL checkpoint",),]@@ -189,3 +190,27 @@ def test_muse_wraps_the_pad_span_in_image_start_end_and_clamps_to_a_token_maximuoverride = TokenizeManager(tokenizer, get_mm_processor(MUSE, MultimodalConfig(processor_kwargs={"max_image_tokens": 256})))small = override.tokenize([_msg([_png(640, 400)])])[0].mm_items[0]assert small.grid_thw == [1, 24, 40] and small.num_tokens == 240++[email protected](not os.path.exists(os.path.join(MINIMAX, "config.json")), reason="FREETOKEN_MINIMAX_M3_MODEL not set")+def test_minimax_wraps_the_pad_span_in_start_end_tokens():+ from freetoken.mm.config import MultimodalConfig+ from freetoken.mm.processor import get_mm_processor+ from freetoken.mm.processors.minimax_m3 import IMAGE_END_ID, IMAGE_START_ID+ from freetoken.tokenizer.tokenize import TokenizeManager+ from freetoken.utils.hf import load_tokenizer++ tokenizer = load_tokenizer(MINIMAX)+ # the processor hardcodes the wrapper ids the checkpoint config does not name+ assert tokenizer.convert_tokens_to_ids("]<]start of image[>[") == IMAGE_START_ID+ assert tokenizer.convert_tokens_to_ids("]<]end of image[>[") == IMAGE_END_ID+ r = TokenizeManager(tokenizer, get_mm_processor(MINIMAX)).tokenize([_msg([_png(640, 400)])])[0]+ item = r.mm_items[0]+ ((start, end),) = item.offsets+ ids = r.input_ids+ _, h, w = item.grid_thw+ assert ids[start - 1].item() == IMAGE_START_ID and ids[end].item() == IMAGE_END_ID+ assert end - start == h * w // 4 == 322 and r.mrope_positions is None+ assert bool((ids[start:end] == item.pad_value).all())+ budgeted = TokenizeManager(tokenizer, get_mm_processor(MINIMAX, MultimodalConfig(image_max_tokens=64))).tokenize([_msg([_png(640, 400)])])[0]+ assert budgeted.mm_items[0].num_tokens <= 64