Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(gguf): serve MiniMax-M2 and MiniMax-M3 GGUF checkpoints (#725)

Xiaoze Fan4663b2b89e60
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py
index eca18d00..f0b70e20 100644
--- a/python/freetoken/kernel/aot_models.py
+++ b/python/freetoken/kernel/aot_models.py
@@ -300,6 +300,7 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
moe_intermediate_size=1536, # experts reuse the dense intermediate_size
expert_formats=_NVFP4_FORMATS,
aliases=("MiniMaxAI/MiniMax-M2.5",),
+ arch_aliases=("MiniMaxM2GGUFForCausalLM",),
),
AotModel(
# GQA block-sparse (BSA): the BSAKVCache stores K/V through store_cache
@@ -318,7 +319,7 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
moe_intermediate_size=3072,
expert_formats=("nvfp4",),
aliases=("MiniMaxAI/MiniMax-M3",),
- arch_aliases=("MiniMaxM3SparseForCausalLM",),
+ arch_aliases=("MiniMaxM3SparseForCausalLM", "MiniMaxM3GGUFForCausalLM"),
),
AotModel(
name="deepseek-ai/DeepSeek-V4-Flash",
diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.py
index 6e31b8ff..677adff6 100644
--- a/python/freetoken/models/gguf/config.py
+++ b/python/freetoken/models/gguf/config.py
@@ -19,6 +19,8 @@ from .reader import gguf_architecture, load_gguf_metadata, gguf_tensor_names
GGUF_ARCH_TO_REGISTRY: dict[str, str] = {
"gemma4": "Gemma4GGUFForCausalLM",
"llama": "LlamaGGUFForCausalLM",
+ "minimax-m2": "MiniMaxM2GGUFForCausalLM",
+ "minimax-m3": "MiniMaxM3GGUFForCausalLM",
"muse-glimmer": "MuseGlimmerGGUFForCausalLM",
"qwen2": "Qwen2GGUFForCausalLM",
"qwen3": "Qwen3GGUFForCausalLM",
diff --git a/python/freetoken/models/minimax_m2/__init__.py b/python/freetoken/models/minimax_m2/__init__.py
index ad276f84..edb9896d 100644
--- a/python/freetoken/models/minimax_m2/__init__.py
+++ b/python/freetoken/models/minimax_m2/__init__.py
@@ -1,4 +1,5 @@
from .config import parse_config
+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf
from .model import MiniMaxM2ForCausalLM
from .weight import iter_weights, nvfp4_expert_spec
@@ -7,4 +8,7 @@ __all__ = [
"MiniMaxM2ForCausalLM",
"parse_config",
"iter_weights",
+ "iter_gguf_weights",
+ "parse_gguf_config",
+ "to_gguf",
]
diff --git a/python/freetoken/models/minimax_m2/gguf.py b/python/freetoken/models/minimax_m2/gguf.py
new file mode 100644
index 00000000..99047b40
--- /dev/null
+++ b/python/freetoken/models/minimax_m2/gguf.py
@@ -0,0 +1,161 @@
+"""MiniMax-M2 from a llama.cpp ``minimax-m2`` GGUF: config from the GGUF metadata, tensors by name; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``).
+
+The llama.cpp converter (conversion/minimax.py) only stacks the w1 / w2 / w3 experts: both sides run NeoX rope, so q/k carry no rope permute, and the RMSNorm weights are the HF ones.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import replace
+from types import SimpleNamespace
+from typing import TYPE_CHECKING, Iterator
+
+import torch
+
+from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index
+from freetoken.models.gguf.tensors import float_tensor, fused, matrix
+
+from .config import parse_config
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+ from freetoken.models.gguf.config import GgufConfigShim
+
+_ARCH = "minimax-m2"
+_LAYER_LEAVES = {
+ "self_attn.q_proj": "attn_q",
+ "self_attn.k_proj": "attn_k",
+ "self_attn.v_proj": "attn_v",
+ "self_attn.o_proj": "attn_output",
+ "block_sparse_moe.gate": "ffn_gate_inp",
+ "block_sparse_moe.experts.0.w1": "ffn_gate_exps",
+ "block_sparse_moe.experts.0.w3": "ffn_up_exps",
+ "block_sparse_moe.experts.0.w2": "ffn_down_exps",
+}
+_QKV_LEAVES = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj")
+_TOP_LEAVES = {"model.embed_tokens": "token_embd", "lm_head": "output"}
+_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")
+_SIGMOID_GATING = 2
+
+
+def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""
+ if name in _TOP_LEAVES:
+ return (_TOP_LEAVES[name],)
+ m = _LAYER_RE.match(name)
+ if m is None:
+ return ()
+ blk, leaf = f"blk.{m.group(1)}", m.group(2)
+ if leaf in _QKV_LEAVES and f"{blk}.attn_qkv.weight" in tensors:
+ return (f"{blk}.attn_qkv",)
+ tensor = _LAYER_LEAVES.get(leaf)
+ return (f"{blk}.{tensor}",) if tensor is not None else ()
+
+
+def _check_supported(shim: GgufConfigShim) -> None:
+ if any(k.startswith("prism.hadamard.") for k in shim.metadata):
+ raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")
+ index = gguf_tensor_index(shim.model_path)
+ scaled = sorted(n for n in index if n.endswith((".scale", ".input_scale")))
+ if scaled:
+ raise NotImplementedError(f"{shim.model_path}: GGUF tensors with per-tensor scales are not supported (e.g. {scaled[0]})")
+ # a metadata-only GGUF (an FTW dir) has no tensor table to check
+ if index and "blk.0.exp_probs_b.bias" not in index:
+ raise NotImplementedError(f"{shim.model_path}: converted by an older llama.cpp that names the router bias exp_probs_b, which current llama.cpp refuses too")
+
+
+def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:
+ """The HF config fields ``parse_config`` reads, from the GGUF metadata."""
+ md = shim.metadata
+
+ def kv(key: str):
+ return md[f"{_ARCH}.{key}"]
+
+ _check_supported(shim)
+ gating = int(md.get(f"{_ARCH}.expert_gating_func", 0))
+ if gating != _SIGMOID_GATING:
+ raise NotImplementedError(f"{shim.model_path}: expert gating function {gating} is not supported; MiniMax-M2 routes with a sigmoid (2)")
+ scaling = md.get(f"{_ARCH}.rope.scaling.type", "none")
+ if scaling != "none":
+ raise NotImplementedError(f"{shim.model_path}: rope scaling {scaling!r} is not supported")
+ # a list holds one size per layer
+ expert_ff = kv("expert_feed_forward_length")
+ if isinstance(expert_ff, list):
+ if len(set(expert_ff)) != 1:
+ raise NotImplementedError(f"{shim.model_path}: per-layer expert sizes {sorted(set(expert_ff))} are not supported")
+ expert_ff = expert_ff[0]
+ return SimpleNamespace(
+ model_type="minimax_m2",
+ architectures=list(shim.architectures),
+ hidden_act="silu",
+ hidden_size=int(kv("embedding_length")),
+ num_hidden_layers=int(kv("block_count")),
+ num_attention_heads=int(kv("attention.head_count")),
+ num_key_value_heads=int(kv("attention.head_count_kv")),
+ head_dim=int(kv("attention.key_length")),
+ rotary_dim=int(kv("rope.dimension_count")),
+ rope_theta=float(kv("rope.freq_base")),
+ max_position_embeddings=int(kv("context_length")),
+ rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),
+ vocab_size=int(shim.vocab_size),
+ tie_word_embeddings=bool(shim.tie_word_embeddings),
+ intermediate_size=int(expert_ff),
+ num_local_experts=int(kv("expert_count")),
+ num_experts_per_tok=int(kv("expert_used_count")),
+ # llama.cpp always builds the per-layer q/k norms of this arch
+ use_qk_norm=True,
+ )
+
+
+def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:
+ return replace(parse_config(_hf_config(shim)), expert_quant="gguf")
+
+
+def iter_gguf_weights(
+ model_path: str,
+ device: torch.device,
+ *,
+ include_moe_experts: bool,
+ include_non_moe: bool,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ """The dense state dict from the GGUF; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``)."""
+ if not include_non_moe:
+ return
+ from freetoken.utils import cached_load_hf_config
+
+ hf = _hf_config(cached_load_hf_config(model_path))
+ index = gguf_tensor_index(model_path)
+ hidden = hf.hidden_size
+ q_rows, kv_rows = hf.num_attention_heads * hf.head_dim, hf.num_key_value_heads * hf.head_dim
+
+ def t(name: str) -> GgufTensorInfo:
+ return index[name if name in index else f"{name}.weight"]
+
+ yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)
+ if not hf.tie_word_embeddings:
+ yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)
+ yield "model.norm.weight", float_tensor(t("output_norm"))
+
+ for layer in range(hf.num_hidden_layers):
+ blk, pre = f"blk.{layer}", f"model.layers.{layer}"
+ yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm"))
+ yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.ffn_norm"))
+ sa = f"{pre}.self_attn"
+ # convert_hf_to_gguf.py --fuse-qkv stores q, k and v as one attn_qkv matrix
+ if f"{blk}.attn_qkv.weight" in index:
+ qkv = [t(f"{blk}.attn_qkv")] * 3
+ parts = list(matrix(qkv[0]).split([q_rows, kv_rows, kv_rows]))
+ else:
+ qkv = [t(f"{blk}.attn_{p}") for p in ("q", "k", "v")]
+ parts = [matrix(p) for p in qkv]
+ yield from fused(f"{sa}.qkv_proj", qkv, parts, hidden)
+ yield f"{sa}.q_norm.weight", float_tensor(t(f"{blk}.attn_q_norm"))
+ yield f"{sa}.k_norm.weight", float_tensor(t(f"{blk}.attn_k_norm"))
+ out = t(f"{blk}.attn_output")
+ yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))
+ moe = f"{pre}.block_sparse_moe"
+ yield f"{moe}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp"))
+ yield f"{moe}.e_score_correction_bias", float_tensor(t(f"{blk}.exp_probs_b.bias"))
+
+
+__all__ = ["iter_gguf_weights", "parse_gguf_config", "to_gguf"]
diff --git a/python/freetoken/models/minimax_m2/model.py b/python/freetoken/models/minimax_m2/model.py
index 457ab565..2f29a873 100644
--- a/python/freetoken/models/minimax_m2/model.py
+++ b/python/freetoken/models/minimax_m2/model.py
@@ -41,6 +41,8 @@ class MiniMaxM2Model(BaseOP):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
+ quant_config=config.quant,
+ prefix=f"{prefix}.embed_tokens",
)
self.layers = OPList(
[
diff --git a/python/freetoken/models/minimax_m3/__init__.py b/python/freetoken/models/minimax_m3/__init__.py
index 4abf9c87..8d0aea79 100644
--- a/python/freetoken/models/minimax_m3/__init__.py
+++ b/python/freetoken/models/minimax_m3/__init__.py
@@ -1,4 +1,5 @@
from .config import VisionConfig, parse_config, parse_vision_config
+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf
from .model import MiniMaxM3ForCausalLM, MiniMaxM3ForConditionalGeneration
from .vision import MiniMaxM3VisionModel
from .weight import (
@@ -17,4 +18,7 @@ __all__ = [
"parse_vision_config",
"iter_vision_weights",
"iter_weights",
+ "iter_gguf_weights",
+ "parse_gguf_config",
+ "to_gguf",
]
diff --git a/python/freetoken/models/minimax_m3/attention.py b/python/freetoken/models/minimax_m3/attention.py
index 0d55c072..4fcf57e1 100644
--- a/python/freetoken/models/minimax_m3/attention.py
+++ b/python/freetoken/models/minimax_m3/attention.py
@@ -45,6 +45,7 @@ class MiniMaxM3Attention(BaseOP):
self.qkv_proj = LinearReplicated(
args.hidden_size, self.qo_attn_dim + 2 * self.kv_attn_dim, has_bias=False,
+ output_sizes=[self.qo_attn_dim, self.kv_attn_dim, self.kv_attn_dim],
quant_config=config.quant, prefix=f"{prefix}.qkv_proj",
)
self.o_proj = LinearReplicated(
diff --git a/python/freetoken/models/minimax_m3/gguf.py b/python/freetoken/models/minimax_m3/gguf.py
new file mode 100644
index 00000000..92742c3a
--- /dev/null
+++ b/python/freetoken/models/minimax_m3/gguf.py
@@ -0,0 +1,198 @@
+"""MiniMax-M3 text from a llama.cpp ``minimax-m3`` GGUF: config from the GGUF metadata, tensors by name; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``).
+
+The llama.cpp converter (conversion/minimax.py) stacks the w1 / w2 / w3 experts, stores the indexer projections as F32, and adds 1 to every RMSNorm weight so llama.cpp can run plain RMSNorm; the loader subtracts the 1 for this family's (1 + w) norms.
+Both sides run NeoX rope, so q/k carry no rope permute.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import replace
+from types import SimpleNamespace
+from typing import TYPE_CHECKING, Iterator
+
+import torch
+
+from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index
+from freetoken.models.gguf.tensors import float_tensor, fused, matrix
+
+from .config import parse_config
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+ from freetoken.models.gguf.config import GgufConfigShim
+
+_ARCH = "minimax-m3"
+_LAYER_LEAVES = {
+ "self_attn.q_proj": "attn_q",
+ "self_attn.k_proj": "attn_k",
+ "self_attn.v_proj": "attn_v",
+ "self_attn.o_proj": "attn_output",
+ "self_attn.index_q_proj": "indexer.q_proj",
+ "self_attn.index_k_proj": "indexer.k_proj",
+ "mlp.gate_proj": "ffn_gate",
+ "mlp.up_proj": "ffn_up",
+ "mlp.down_proj": "ffn_down",
+ "block_sparse_moe.gate": "ffn_gate_inp",
+ "block_sparse_moe.shared_experts.gate_proj": "ffn_gate_shexp",
+ "block_sparse_moe.shared_experts.up_proj": "ffn_up_shexp",
+ "block_sparse_moe.shared_experts.down_proj": "ffn_down_shexp",
+ "block_sparse_moe.experts.0.w1": "ffn_gate_exps",
+ "block_sparse_moe.experts.0.w3": "ffn_up_exps",
+ "block_sparse_moe.experts.0.w2": "ffn_down_exps",
+}
+_TOP_LEAVES = {"model.embed_tokens": "token_embd", "lm_head": "output"}
+_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")
+_SIGMOID_GATING = 2
+# llama.cpp hard-codes these for the arch (src/llama-graph.cpp LLM_FFN_SWIGLU_OAI_MOE)
+_SWIGLU_ALPHA, _SWIGLU_LIMIT = 1.702, 7.0
+
+
+def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""
+ if name in _TOP_LEAVES:
+ return (_TOP_LEAVES[name],)
+ m = _LAYER_RE.match(name)
+ if m is None:
+ return ()
+ tensor = _LAYER_LEAVES.get(m.group(2))
+ return (f"blk.{m.group(1)}.{tensor}",) if tensor is not None else ()
+
+
+def _check_supported(shim: GgufConfigShim) -> None:
+ if any(k.startswith("prism.hadamard.") for k in shim.metadata):
+ raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")
+ scaled = sorted(n for n in gguf_tensor_index(shim.model_path) if n.endswith((".scale", ".input_scale")))
+ if scaled:
+ raise NotImplementedError(f"{shim.model_path}: GGUF tensors with per-tensor scales are not supported (e.g. {scaled[0]})")
+
+
+def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:
+ """The HF text config fields ``parse_config`` reads, from the GGUF metadata."""
+ md = shim.metadata
+
+ def kv(key: str):
+ return md[f"{_ARCH}.{key}"]
+
+ _check_supported(shim)
+ if f"{_ARCH}.attention.indexer.head_count" not in md:
+ raise NotImplementedError(f"{shim.model_path}: converted by an older llama.cpp without the sparse-attention indexer, which current llama.cpp refuses too")
+ gating = int(kv("expert_gating_func"))
+ if gating != _SIGMOID_GATING:
+ raise NotImplementedError(f"{shim.model_path}: expert gating function {gating} is not supported; MiniMax-M3 routes with a sigmoid (2)")
+ if not md.get(f"{_ARCH}.expert_weights_norm", False):
+ raise NotImplementedError(f"{shim.model_path}: routing without renormalized expert weights is not supported")
+ scaling = md.get(f"{_ARCH}.rope.scaling.type", "none")
+ if scaling != "none":
+ raise NotImplementedError(f"{shim.model_path}: rope scaling {scaling!r} is not supported")
+ expert_ff = kv("expert_feed_forward_length")
+ if isinstance(expert_ff, list):
+ if len(set(expert_ff)) != 1:
+ raise NotImplementedError(f"{shim.model_path}: per-layer expert sizes {sorted(set(expert_ff))} are not supported")
+ expert_ff = expert_ff[0]
+ num_layers = int(kv("block_count"))
+ dense = int(md.get(f"{_ARCH}.leading_dense_block_count", 0))
+ # llama.cpp runs sparse attention and the MoE FFN on every layer after the leading dense ones
+ sparse = [0] * dense + [1] * (num_layers - dense)
+ return SimpleNamespace(
+ model_type="minimax_m3",
+ architectures=list(shim.architectures),
+ hidden_act="swigluoai",
+ swiglu_alpha=_SWIGLU_ALPHA,
+ swiglu_limit=_SWIGLU_LIMIT,
+ hidden_size=int(kv("embedding_length")),
+ num_hidden_layers=num_layers,
+ num_attention_heads=int(kv("attention.head_count")),
+ num_key_value_heads=int(kv("attention.head_count_kv")),
+ head_dim=int(kv("attention.key_length")),
+ rotary_dim=int(kv("rope.dimension_count")),
+ rope_theta=float(kv("rope.freq_base")),
+ max_position_embeddings=int(kv("context_length")),
+ rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),
+ vocab_size=int(shim.vocab_size),
+ tie_word_embeddings=bool(shim.tie_word_embeddings),
+ dense_intermediate_size=int(kv("feed_forward_length")),
+ intermediate_size=int(expert_ff),
+ shared_intermediate_size=int(expert_ff),
+ n_shared_experts=int(kv("expert_shared_count")),
+ num_local_experts=int(kv("expert_count")),
+ num_experts_per_tok=int(kv("expert_used_count")),
+ # llama.cpp skips the scale when the key is absent or 0
+ routed_scaling_factor=float(md.get(f"{_ARCH}.expert_weights_scale", 0.0)) or 1.0,
+ use_routing_bias=True,
+ moe_layer_freq=sparse,
+ sparse_attention_config={
+ "use_sparse_attention": True,
+ "sparse_attention_freq": sparse,
+ "sparse_num_index_heads": int(kv("attention.indexer.head_count")),
+ "sparse_index_dim": int(kv("attention.indexer.key_length")),
+ "sparse_topk_blocks": int(kv("attention.indexer.top_k")),
+ "sparse_block_size": int(kv("attention.indexer.block_size")),
+ "sparse_local_block": int(kv("attention.indexer.local_blocks")),
+ "sparse_init_block": 0,
+ "sparse_score_type": "max",
+ },
+ )
+
+
+def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:
+ return replace(parse_config(_hf_config(shim)), expert_quant="gguf")
+
+
+def iter_gguf_weights(
+ model_path: str,
+ device: torch.device,
+ *,
+ include_moe_experts: bool,
+ include_non_moe: bool,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ """The dense text state dict from the GGUF; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``)."""
+ if not include_non_moe:
+ return
+ from freetoken.utils import cached_load_hf_config
+
+ config = parse_gguf_config(cached_load_hf_config(model_path))
+ args = config.m3_args
+ index = gguf_tensor_index(model_path)
+ hidden = config.hidden_size
+
+ def t(name: str) -> GgufTensorInfo:
+ return index[name if name in index else f"{name}.weight"]
+
+ def norm(name: str) -> torch.Tensor:
+ return float_tensor(t(name)).float() - 1.0
+
+ yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)
+ if not config.tie_word_embeddings:
+ yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)
+ yield "model.norm.weight", norm("output_norm")
+
+ for layer in range(config.num_layers):
+ blk, pre = f"blk.{layer}", f"model.layers.{layer}"
+ yield f"{pre}.input_layernorm.weight", norm(f"{blk}.attn_norm")
+ yield f"{pre}.post_attention_layernorm.weight", norm(f"{blk}.ffn_norm")
+ sa = f"{pre}.self_attn"
+ qkv = [t(f"{blk}.attn_{p}") for p in ("q", "k", "v")]
+ yield from fused(f"{sa}.qkv_proj", qkv, [matrix(p) for p in qkv], hidden)
+ out = t(f"{blk}.attn_output")
+ yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))
+ yield f"{sa}.q_norm.weight", norm(f"{blk}.attn_q_norm")
+ yield f"{sa}.k_norm.weight", norm(f"{blk}.attn_k_norm")
+ if args.is_sparse_layer(layer):
+ iq, ik = t(f"{blk}.indexer.q_proj"), t(f"{blk}.indexer.k_proj")
+ yield from fused(f"{sa}.index_qk_proj", [iq, ik], [matrix(iq), matrix(ik)], hidden)
+ yield f"{sa}.index_q_norm.weight", norm(f"{blk}.indexer.q_norm")
+ yield f"{sa}.index_k_norm.weight", norm(f"{blk}.indexer.k_norm")
+ if layer in args.moe_layer_ids:
+ moe = f"{pre}.block_sparse_moe"
+ yield f"{moe}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp")).float()
+ yield f"{moe}.e_score_correction_bias", float_tensor(t(f"{blk}.exp_probs_b.bias")).float()
+ mlp, parts = f"{moe}.shared_experts", [t(f"{blk}.ffn_{p}_shexp") for p in ("gate", "up", "down")]
+ else:
+ mlp, parts = f"{pre}.mlp", [t(f"{blk}.ffn_{p}") for p in ("gate", "up", "down")]
+ gate, up, down = parts
+ yield from fused(f"{mlp}.gate_up_proj", [gate, up], [matrix(gate), matrix(up)], hidden)
+ yield from fused(f"{mlp}.down_proj", [down], [matrix(down)], int(down.ne[0]))
+
+
+__all__ = ["iter_gguf_weights", "parse_gguf_config", "to_gguf"]
diff --git a/python/freetoken/models/minimax_m3/mlp.py b/python/freetoken/models/minimax_m3/mlp.py
index d5332eed..62b02e80 100644
--- a/python/freetoken/models/minimax_m3/mlp.py
+++ b/python/freetoken/models/minimax_m3/mlp.py
@@ -36,6 +36,7 @@ class MiniMaxM3MLP(BaseOP):
):
self.gate_up_proj = LinearReplicated(
hidden_size, 2 * intermediate_size, has_bias=False,
+ output_sizes=[intermediate_size, intermediate_size],
quant_config=quant_config, prefix=f"{prefix}.gate_up_proj",
)
self.down_proj = LinearReplicated(
diff --git a/python/freetoken/models/minimax_m3/model.py b/python/freetoken/models/minimax_m3/model.py
index d78fc10a..b6f271b3 100644
--- a/python/freetoken/models/minimax_m3/model.py
+++ b/python/freetoken/models/minimax_m3/model.py
@@ -71,6 +71,8 @@ class MiniMaxM3Model(BaseOP):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
+ quant_config=config.quant,
+ prefix=f"{prefix}.embed_tokens",
)
self.layers = OPList(
[
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index 08b70e83..bfa0a1d6 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -48,6 +48,8 @@ _DENSE_PACKED = (
# per-expert checkpoints: probe the first expert's projections for the experts container
_EXPERTS_PACKED = (("experts", ("experts.0.gate_proj", "experts.0.up_proj", "experts.0.down_proj")),)
_EXPERTS_W123_PACKED = (("experts", ("experts.0.w1", "experts.0.w2", "experts.0.w3")),)
+# the GGUF scheme reads expert types in (gate, up, down) order
+_EXPERTS_W132_PACKED = (("experts", ("experts.0.w1", "experts.0.w3", "experts.0.w2")),)
# pre-stacked expert checkpoints (Qwen3-VL-MoE): the container's own two tensors are the probe
_STACKED_EXPERTS_PACKED = (("experts", ("experts.gate_up_proj", "experts.down_proj")),)
_QWEN3_5_PACKED = _DENSE_PACKED + (
@@ -86,6 +88,7 @@ _MUSE_GLIMMER_ENCODERS = (EncoderSpec("vision", "vision_config", ("image",)),)
_MINIMAX_M3_PACKED = _DENSE_PACKED + (
("index_qk_proj", ("index_q_proj", "index_k_proj")),
) + _EXPERTS_W123_PACKED
+_MINIMAX_M3_GGUF_PACKED = _DENSE_PACKED + (("index_qk_proj", ("index_q_proj", "index_k_proj")),) + _EXPERTS_W132_PACKED
_MINIMAX_M3_PROCESSOR = "freetoken.mm.processors.minimax_m3:MiniMaxM3MMProcessor"
_MINIMAX_M3_ENCODERS = (EncoderSpec("vision", "vision_config", ("image",)),)
# DeepSeek-V4's vision dims sit at the top level of the checkpoint config, next to the
@@ -166,6 +169,14 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"MiniMaxM2ForCausalLM",
packed_modules_mapping=_DENSE_PACKED + _EXPERTS_W123_PACKED,
),
+ # llama.cpp minimax-m2 GGUF: the same model classes with every tensor in its packed ggml type
+ "MiniMaxM2GGUFForCausalLM": ModelSpec(
+ "freetoken.models.minimax_m2",
+ "MiniMaxM2ForCausalLM",
+ parse_config="parse_gguf_config",
+ iter_weights="iter_gguf_weights",
+ packed_modules_mapping=_DENSE_PACKED + _EXPERTS_W132_PACKED,
+ ),
# MiniMax-M3 (model_type minimax_m3_vl): multimodal wrapper config (text tower in
# text_config, weights under language_model.); the CLIP-style tower under vision_tower.
# serves image input. GQA + block-sparse attention (lightning indexer, top-k 128-token
@@ -185,6 +196,14 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"MiniMaxM3ForCausalLM",
packed_modules_mapping=_MINIMAX_M3_PACKED,
),
+ # llama.cpp minimax-m3 GGUF (text only): the same model classes with every tensor in its packed ggml type
+ "MiniMaxM3GGUFForCausalLM": ModelSpec(
+ "freetoken.models.minimax_m3",
+ "MiniMaxM3ForCausalLM",
+ parse_config="parse_gguf_config",
+ iter_weights="iter_gguf_weights",
+ packed_modules_mapping=_MINIMAX_M3_GGUF_PACKED,
+ ),
"DeepseekV4ForCausalLM": ModelSpec(
"freetoken.models.deepseek_v4",
"DeepseekV4ForCausalLM",
diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.py
index 7e25e58f..b79d2860 100644
--- a/python/freetoken/moe/expert_banks.py
+++ b/python/freetoken/moe/expert_banks.py
@@ -423,7 +423,7 @@ def load_expert_banks(
from freetoken.models.gguf.experts import load_gguf_expert_banks
# each layer keeps its own ggml types, so the banks come from the file, not from one method's layout
- return load_gguf_expert_banks(model_path, model_config.num_moe_layers, dummy=dummy, device=device if resident else None)
+ return load_gguf_expert_banks(model_path, model_config.num_moe_layers, first_layer=model_config.first_k_dense_replace, dummy=dummy, device=device if resident else None)
if method is None:
raise ValueError("the experts of a raw checkpoint are packed by the offload layers' MoE quant method; pass method=")
return _method_expert_banks(model_path, model_config, method, device, dummy, par, workers, chunk, layer_sink, resident)