FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Weight loading for MiniMax-M3 (``minimax_m3``).
The checkpoint is the multimodal wrapper layout: every text-tower tensor carries a
``language_model.`` prefix (``language_model.model.layers.N...``,
``language_model.lm_head.weight``); the ViT stack (``vision_tower.`` /
``multi_modal_projector.`` / ``patch_merge_mlp.``) is read by ``_iter_vision`` only when
the engine builds the vision encoder (``include_vision``).
Resident (non routed-expert) dense projections are MXFP8 in the checkpoint
(fp8-e4m3 ``weight`` + uint8 e8m0 block-32 ``weight_scale_inv``): the fp8 weight and
its scale codes stream through verbatim (merged output-wise for the fused qkv /
index-qk / gate-up projections -- the scales are per-output-row, so fusion is exact).
Norms, the router gate, ``e_score_correction_bias`` (kept fp32), embeddings and
lm_head are unquantized and stream through verbatim. Routed experts are NVFP4
(``w1/w3/w2`` = gate/up/down, same ModelOpt layout as MiniMax-M2) and go to the
offload cache from their NVFP4 pieces; expert ``input_scale`` calibration
tensors are unused (W4A16) and never match the bank spec.
"""
from __future__ import annotations
import json
import os
import re
from typing import Iterator
import safetensors
import torch
from freetoken.distributed import get_tp_info
from freetoken.models.loader import drop_page_cache
from freetoken.models.nvfp4_banks import (
Nvfp4ExpertSourceSpec,
)
from freetoken.utils import cached_load_hf_config, download_hf_weight
from tqdm import tqdm
from .config import parse_config
_EXPERT_KEY_RE = re.compile(
r"^language_model\.model\.layers\.(?P<layer>\d+)\.block_sparse_moe\."
r"experts\.(?P<expert>\d+)\.(?P<proj>w1|w2|w3)\."
r"(?P<kind>weight|weight_scale|weight_scale_2)$"
)
_NVFP4_SOURCE_SPEC = Nvfp4ExpertSourceSpec(
key_pattern=_EXPERT_KEY_RE,
proj_to_role={"w1": "gate", "w3": "up", "w2": "down"},
# Experts exist only for layers [first_k_dense_replace, num_layers); banks pack by
# MoE-layer index so the leading dense layers leave no holes (GLM precedent).
layer_to_bank=lambda layer, config: (
None
if layer < config.first_k_dense_replace or layer >= config.num_layers
else layer - config.first_k_dense_replace
),
desc="MiniMax-M3 NVFP4 experts",
)
class _ShardReader:
"""Random-access reader over the safetensors index (GLM-5.2 precedent): the M3
dense pass walks layers in model order and fuses q/k/v (etc.) output-wise, which
a shard-ordered stream cannot do without buffering whole projections anyway."""
def __init__(self, folder: str, weight_map: dict, device: torch.device):
self._folder = folder
self._weight_map = weight_map
self._device = device
self._handles: dict[str, object] = {}
def has(self, name: str) -> bool:
return name in self._weight_map
def get(self, name: str) -> torch.Tensor:
shard = self._weight_map[name]
handle = self._handles.get(shard)
if handle is None:
handle = safetensors.safe_open(
os.path.join(self._folder, shard), framework="pt", device=str(self._device)
).__enter__()
self._handles[shard] = handle
return handle.get_tensor(name)
def close(self) -> None:
for shard, handle in self._handles.items():
try:
handle.__exit__(None, None, None)
except Exception: # pragma: no cover - best effort
pass
drop_page_cache(os.path.join(self._folder, shard))
self._handles.clear()
def _read_proj(reader: _ShardReader, key: str) -> tuple[torch.Tensor, torch.Tensor | None]:
"""One dense projection as ``(weight, scale_codes | None)``. MXFP8 tensors carry a
``weight_scale_inv`` sibling; a bf16 tensor (nothing quantized under this name)
comes back scale-less and passes through every mode unchanged."""
w = reader.get(f"{key}.weight")
scale_key = f"{key}.weight_scale_inv"
if reader.has(scale_key):
s = reader.get(scale_key)
assert w.dtype == torch.float8_e4m3fn and s.dtype == torch.uint8, (
f"unexpected MXFP8 dtypes at {key}: {w.dtype}/{s.dtype}"
)
# The GEMV folds w * 2**(code-127) into the 16-bit compute dtype, which
# is lossless only while the product stays in range; codes above 245
# could overflow bf16. Real checkpoints sit near 127 -- pin at load.
assert int(s.max()) <= 245, (
f"e8m0 scale code {int(s.max())} at {key} exceeds the bf16-exact "
"fold bound (245); the W8A16 kernels would overflow"
)
return w, s
return w, None
def _emit_fused(
reader: _ShardReader,
out_key: str,
part_keys: list[str],
keep_mxfp8: bool,
) -> Iterator[tuple[str, torch.Tensor]]:
"""Emit one (possibly multi-part output-wise fused) projection in the resolved
quant mode. MXFP8 scales are per-output-row / per-32-input-block, so concatenating
the parts' weights AND scales along the output dim is exact."""
parts = [_read_proj(reader, key) for key in part_keys]
if keep_mxfp8 and all(s is not None for _, s in parts):
yield f"{out_key}.weight", torch.cat([w for w, _ in parts], dim=0)
yield f"{out_key}.weight_scale_inv", torch.cat([s for _, s in parts], dim=0)
return
from freetoken.kernel.triton.mxfp8_linear import mxfp8_dequant
dequant = [
mxfp8_dequant(w, s, dtype=torch.bfloat16) if s is not None else w
for w, s in parts
]
yield f"{out_key}.weight", (
torch.cat(dequant, dim=0) if len(dequant) > 1 else dequant[0]
)
_VISION_ATTN = "vision_tower.vision_model.encoder.layers.{}.self_attn"
def _iter_vision(reader: _ShardReader, weight_map: dict, num_layers: int) -> Iterator[tuple[str, torch.Tensor]]:
"""The tower under its checkpoint names with q/k/v fused, the projector and patch-merge MLP under the tower prefix; all bf16 (the patch embedding is stored fp32)."""
for name in weight_map:
if name.startswith(("multi_modal_projector.", "patch_merge_mlp.")):
yield "vision_tower." + name, reader.get(name).to(torch.bfloat16)
elif name.startswith("vision_tower.") and ".self_attn." not in name:
yield name, reader.get(name).to(torch.bfloat16)
for layer in range(num_layers):
attn = _VISION_ATTN.format(layer)
for kind in ("weight", "bias"):
parts = [reader.get(f"{attn}.{proj}.{kind}") for proj in ("q_proj", "k_proj", "v_proj")]
yield f"{attn}.qkv.{kind}", torch.cat(parts, dim=0).to(torch.bfloat16)
yield f"{attn}.out_proj.{kind}", reader.get(f"{attn}.out_proj.{kind}").to(torch.bfloat16)
def iter_weights(
model_path: str,
device: torch.device,
*,
include_moe_experts: bool,
include_non_moe: bool,
include_vision: bool = True,
) -> Iterator[tuple[str, torch.Tensor]]:
assert not include_moe_experts, (
"MiniMax-M3 stores routed experts as NVFP4 and only supports the offload MoE "
"backend; experts are loaded into the offload cache from their NVFP4 pieces."
)
assert include_non_moe
config = parse_config(cached_load_hf_config(model_path))
args = config.m3_args
folder = download_hf_weight(model_path)
with open(os.path.join(folder, "model.safetensors.index.json")) as f:
weight_map = json.load(f)["weight_map"]
reader = _ShardReader(folder, weight_map, device)
primary = get_tp_info().is_primary()
attn_mx = config.attn_quant == "mxfp8"
mlp_mx = config.dense_quant == "mxfp8"
if primary:
from freetoken.utils import init_logger
init_logger(__name__).info(
f"MiniMax-M3 resident quant: attn={config.attn_quant} dense={config.dense_quant} "
f"lm_head={config.lm_head_quant}"
)
try:
for layer in tqdm(
range(config.num_layers),
desc="Loading MiniMax-M3 dense weights",
disable=not primary,
):
src = f"language_model.model.layers.{layer}"
dst = f"model.layers.{layer}"
a_src, a_dst = f"{src}.self_attn", f"{dst}.self_attn"
yield from _emit_fused(
reader,
f"{a_dst}.qkv_proj",
[f"{a_src}.q_proj", f"{a_src}.k_proj", f"{a_src}.v_proj"],
attn_mx,
)
yield from _emit_fused(reader, f"{a_dst}.o_proj", [f"{a_src}.o_proj"], attn_mx)
for norm in ("q_norm", "k_norm"):
yield f"{a_dst}.{norm}.weight", reader.get(f"{a_src}.{norm}.weight")
# Block-sparse indexer (sparse layers only; the dense ablation builds no
# indexer modules, so its tensors are skipped entirely).
if args.is_sparse_layer(layer):
yield from _emit_fused(
reader,
f"{a_dst}.index_qk_proj",
[f"{a_src}.index_q_proj", f"{a_src}.index_k_proj"],
attn_mx,
)
for norm in ("index_q_norm", "index_k_norm"):
yield f"{a_dst}.{norm}.weight", reader.get(f"{a_src}.{norm}.weight")
for norm in ("input_layernorm", "post_attention_layernorm"):
yield f"{dst}.{norm}.weight", reader.get(f"{src}.{norm}.weight")
if layer in args.moe_layer_ids:
m_src, m_dst = f"{src}.block_sparse_moe", f"{dst}.block_sparse_moe"
# fp32 like the bias: both sit on the top-4 selection boundary.
yield (
f"{m_dst}.gate.weight",
reader.get(f"{m_src}.gate.weight").to(torch.float32),
)
# fp32 in the checkpoint AND the module (top-k selection boundary).
yield (
f"{m_dst}.e_score_correction_bias",
reader.get(f"{m_src}.e_score_correction_bias").to(torch.float32),
)
s_src, s_dst = f"{m_src}.shared_experts", f"{m_dst}.shared_experts"
yield from _emit_fused(
reader,
f"{s_dst}.gate_up_proj",
[f"{s_src}.gate_proj", f"{s_src}.up_proj"],
mlp_mx,
)
yield from _emit_fused(
reader, f"{s_dst}.down_proj", [f"{s_src}.down_proj"], mlp_mx
)
else:
yield from _emit_fused(
reader,
f"{dst}.mlp.gate_up_proj",
[f"{src}.mlp.gate_proj", f"{src}.mlp.up_proj"],
mlp_mx,
)
yield from _emit_fused(
reader, f"{dst}.mlp.down_proj", [f"{src}.mlp.down_proj"], mlp_mx
)
yield "model.embed_tokens.weight", reader.get("language_model.model.embed_tokens.weight")
yield "model.norm.weight", reader.get("language_model.model.norm.weight")
yield "lm_head.weight", reader.get("language_model.lm_head.weight")
if include_vision and config.vision_config is not None:
yield from _iter_vision(reader, weight_map, config.vision_config.num_layers)
finally:
reader.close()
def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuple[str, torch.Tensor]]:
"""The vision tower alone, named as iter_weights names it."""
config = parse_config(cached_load_hf_config(model_path))
if config.vision_config is None:
return
folder = download_hf_weight(model_path)
with open(os.path.join(folder, "model.safetensors.index.json")) as f:
weight_map = json.load(f)["weight_map"]
reader = _ShardReader(folder, weight_map, device)
try:
yield from _iter_vision(reader, weight_map, config.vision_config.num_layers)
finally:
reader.close()
def nvfp4_expert_spec(model_path: str, config):
return _NVFP4_SOURCE_SPEC
__all__ = ["iter_weights", "nvfp4_expert_spec"]