FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Engine-facing config for MiniMax-M3 (``minimax_m3``).
The checkpoint is a multimodal wrapper (``model_type=minimax_m3_vl``): the text tower
lives in ``text_config`` and the weights carry a ``language_model.`` prefix. Its vision
section becomes ``ModelConfig.vision_config`` for the tower in vision.py; an engine that
builds no vision encoder hands parse_config a config without that section.
Attention is GQA with block-sparse selection on the trailing layers: parse_config
declares ONE full-attention group over all layers carrying ``mla=False`` plus the
index dims, which resolves to ``AttnType.BSA`` -- the pool factory builds
``BSAKVCache`` (paged GQA K/V + the index-key slab), the KV cost model budgets the
slab from the same spec, and the ``m3_sparse`` backend serves both the dense leading
layers and the sparse layers. ``FREETOKEN_M3_SPARSE=0`` zeroes the index dims (dense
ablation: plain MHAKVCache, every layer attends its whole history through a generic
FULL backend). ``page_size`` is pinned to the 128-token sparse block by the backend's
``page_sizes`` declaration, so one KV page == one sparse block.
The resident projections are served in the precision the checkpoint stores (MXFP8 for the
official export); ``attn_quant`` / ``dense_quant`` record what was detected for the weight
loader. The routed experts are NVFP4
(same ModelOpt layout as MiniMax-M2 / GLM) and always live in the offload cache;
their swigluoai activation restricts the NVFP4 GEMM backend to the Triton kernels
(the NVFP4 MoE kernel objects). lm_head / embeddings are BF16 in the checkpoint.
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Any
from freetoken.models.config import (
FullAttentionGroupConfig,
ModelConfig,
RotaryConfig,
)
from .args import load_args
# the config names only the image token; the reference processor reads these two from the tokenizer's fixed added tokens
IMAGE_START_ID = 200029 # ]<]start of image[>[
IMAGE_END_ID = 200030 # ]<]end of image[>[
def _text_config(hf_config: Any) -> Any:
return getattr(hf_config, "text_config", None) or hf_config
@dataclass(frozen=True)
class VisionConfig:
hidden_size: int
num_layers: int
num_heads: int
intermediate_size: int
num_channels: int
patch_size: int
temporal_patch_size: int
spatial_merge_size: int
layer_norm_eps: float
rope_theta: float
projector_hidden_size: int
text_hidden_size: int
def _compression(vc: Any, key: str) -> int:
"""The native vision config carries the merge sizes flat; the checkpoint's own config nests them under img_token_compression_config."""
value = getattr(vc, key, None)
return int(vc.img_token_compression_config[key] if value is None else value)
def parse_vision_config(hf_config: Any) -> VisionConfig | None:
"""None when the config carries no vision section, which is how a text-only engine asks for no tower."""
vc = getattr(hf_config, "vision_config", None)
if vc is None:
return None
if vc.hidden_act != "gelu":
raise NotImplementedError(f"minimax_m3 vision tower activation {vc.hidden_act!r}; only gelu is implemented")
# the native config moves rope_theta into rope_parameters; the checkpoint's own config keeps it flat
rope_params = getattr(vc, "rope_parameters", None) or {}
text_hidden_size = _text_config(hf_config).hidden_size
return VisionConfig(
hidden_size=vc.hidden_size,
num_layers=vc.num_hidden_layers,
num_heads=vc.num_attention_heads,
intermediate_size=vc.intermediate_size,
num_channels=vc.num_channels,
patch_size=vc.patch_size,
temporal_patch_size=_compression(vc, "temporal_patch_size"),
spatial_merge_size=_compression(vc, "spatial_merge_size"),
layer_norm_eps=vc.layer_norm_eps,
rope_theta=float(rope_params["rope_theta"] if "rope_theta" in rope_params else vc.rope_theta),
projector_hidden_size=getattr(hf_config, "projector_hidden_size", None) or text_hidden_size,
text_hidden_size=text_hidden_size,
)
# House rule: an env switch that changes WHAT IS SERVED must leave a server-log
# trace. parse_config runs a few times per process (engine + weight loader), so
# each resolved-mode line is logged once per distinct value.
_LOGGED_MODES: set = set()
def _log_mode_once(key: str, message: str) -> None:
if key in _LOGGED_MODES:
return
_LOGGED_MODES.add(key)
from freetoken.utils import init_logger
init_logger(__name__).info(message)
def parse_config(hf_config: Any) -> ModelConfig:
text = _text_config(hf_config)
num_layers = text.num_hidden_layers
# Dev/testing only: cap the layer count (e.g. FREETOKEN_M3_MAX_LAYERS=5 -> 3 dense
# + 2 MoE layers) so the forward path / KV / offload cache can be exercised without
# pinning the full ~230 GB of experts. Unset in normal use (GLM-5.2 precedent).
_cap = os.environ.get("FREETOKEN_M3_MAX_LAYERS")
if _cap and int(_cap) < num_layers: # a cap >= the real count changes nothing
num_layers = int(_cap)
_log_mode_once(
f"cap={num_layers}",
f"FREETOKEN_M3_MAX_LAYERS: serving a TRUNCATED model "
f"({num_layers}/{text.num_hidden_layers} layers) -- dev/testing only, "
"outputs are garbage; unset the env for real serving.",
)
sparse_enabled = os.getenv("FREETOKEN_M3_SPARSE", "1") != "0"
if not sparse_enabled:
_log_mode_once(
"sparse=0",
"FREETOKEN_M3_SPARSE=0: serving the DENSE-attention ablation (plain "
"MHA pool, no indexer; different math and outputs from the shipped "
"block-sparse model).",
)
args = load_args(text, num_layers, sparse_enabled=sparse_enabled)
# the quantized release stores the dense projections MXFP8; the reader keeps them native and the layers take their scheme from the QuantConfig
dense_mxfp8 = getattr(hf_config, "quantization_config", None) is not None
# Leading dense-FFN layers: M3's moe_layer_freq is a contiguous 0-prefix; the
# offload cache and the generic num_moe_layers arithmetic rely on that shape.
first_k_dense = num_layers - len(args.moe_layer_ids)
assert args.moe_layer_ids == tuple(range(first_k_dense, num_layers)), (
"MiniMax-M3 expects a contiguous dense prefix in moe_layer_freq, got "
f"{args.moe_layer_ids}"
)
rotary_config = RotaryConfig(
head_dim=args.head_dim,
rotary_dim=args.rotary_dim,
max_position=args.max_position,
base=args.rope_theta,
scaling=None, # plain NeoX partial rope, no scaling
)
# Routed experts: NVFP4 per the modelopt MIXED_PRECISION quantized_layers map
# (checked structurally by the bank loader's shape/dtype asserts). The generic
# detect_expert_quant would report "mixed_precision", which names the checkpoint,
# not the experts -- so the expert format is pinned here (qwen3_5_moe precedent).
quant = getattr(hf_config, "quantization_config", None)
expert_quant = "nvfp4" if quant is not None else "none"
# The activation is model-constant swigluoai: assert instead of consume.
# Raw checkpoints declare "swigluoai"; the native transformers TextConfig
# force-normalizes hidden_act to "silu", which would send the backend
# selection down a silu-only path.
declared_act = getattr(text, "hidden_act", "swigluoai")
assert declared_act in ("swigluoai", "silu"), (
f"MiniMax-M3 support implements swigluoai only, got hidden_act={declared_act!r}"
)
return ModelConfig(
num_layers=num_layers,
num_qo_heads=args.num_heads,
num_kv_heads=args.num_kv_heads,
head_dim=args.head_dim,
hidden_size=args.hidden_size,
vocab_size=text.vocab_size,
intermediate_size=args.dense_intermediate_size,
hidden_act="swigluoai", # pinned; see the declared_act assert above
rms_norm_eps=text.rms_norm_eps,
tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)),
rotary_config=rotary_config,
# One group over all layers: GQA K/V everywhere, plus the index-key slab for
# the sparse layers (mla=False + index dims -> AttnType.BSA -> BSAKVCache).
# The same spec drives the pool factory and the KV cost model, so they can
# never disagree. FREETOKEN_M3_SPARSE=0 zeroes the index dims (dense ablation:
# plain MHAKVCache, generic FULL backends serve every layer).
attention_groups=(
FullAttentionGroupConfig(
name="full",
layer_ids=tuple(range(num_layers)),
num_kv_heads=args.num_kv_heads,
head_dim=args.head_dim,
rotary_config=rotary_config,
mla=False,
index_head_dim=args.index_dim if args.use_sparse else 0,
num_index_layers=(
sum(1 for lid in args.sparse_layer_ids if lid < num_layers)
if args.use_sparse
else 0
),
),
),
num_experts=int(getattr(text, "num_local_experts", 0)),
num_experts_per_tok=text.num_experts_per_tok,
# Routed experts use the text tower's intermediate_size (3072); the dense
# MLP width (12288) rides ModelConfig.intermediate_size above.
moe_intermediate_size=text.intermediate_size,
norm_topk_prob=True, # MiniMax always renormalizes the selected expert weights
model_type=getattr(hf_config, "model_type", "minimax_m3_vl"),
architectures=getattr(
hf_config, "architectures", ["MiniMaxM3SparseForConditionalGeneration"]
),
moe_enabled=True,
expert_quant=expert_quant,
first_k_dense_replace=first_k_dense,
n_shared_experts=int(getattr(text, "n_shared_experts", 0)),
shared_expert_intermediate_size=args.shared_intermediate_size,
routed_scaling_factor=float(getattr(text, "routed_scaling_factor", 1.0)),
has_router_bias=bool(getattr(text, "use_routing_bias", False)),
use_qk_norm=bool(getattr(text, "use_qk_norm", True)),
swiglu_limit=args.swiglu_limit,
hidden_act_alpha=args.swiglu_alpha,
# Resident-weight quant modes (see the module docstring): recorded here so the
# resolved config describes the served weights; the constructors and
# iter_weights read these fields, never the env directly. lm_head stays bf16
# (the checkpoint excludes it from quantization).
attn_quant="mxfp8" if dense_mxfp8 else "none",
dense_quant="mxfp8" if dense_mxfp8 else "none",
lm_head_quant="none",
m3_args=args,
vision_config=parse_vision_config(hf_config),
# the native config maps image_token_id onto the checkpoint's image_token_index
image_token_id=getattr(hf_config, "image_token_id", getattr(hf_config, "image_token_index", None)),
)
__all__ = ["IMAGE_END_ID", "IMAGE_START_ID", "VisionConfig", "parse_config", "parse_vision_config"]