Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

10.9 KB
Raw Blame
"""Engine-facing config for MiniMax-M3 (``minimax_m3``).

The checkpoint is a multimodal wrapper (``model_type=minimax_m3_vl``): the text tower
lives in ``text_config`` and the weights carry a ``language_model.`` prefix. Its vision
section becomes ``ModelConfig.vision_config`` for the tower in vision.py; an engine that
builds no vision encoder hands parse_config a config without that section.

Attention is GQA with block-sparse selection on the trailing layers: parse_config
declares ONE full-attention group over all layers carrying ``mla=False`` plus the
index dims, which resolves to ``AttnType.BSA`` -- the pool factory builds
``BSAKVCache`` (paged GQA K/V + the index-key slab), the KV cost model budgets the
slab from the same spec, and the ``m3_sparse`` backend serves both the dense leading
layers and the sparse layers. ``FREETOKEN_M3_SPARSE=0`` zeroes the index dims (dense
ablation: plain MHAKVCache, every layer attends its whole history through a generic
FULL backend). ``page_size`` is pinned to the 128-token sparse block by the backend's
``page_sizes`` declaration, so one KV page == one sparse block.

The resident projections are served in the precision the checkpoint stores (MXFP8 for the
official export); ``attn_quant`` / ``dense_quant`` record what was detected for the weight
loader. The routed experts are NVFP4
(same ModelOpt layout as MiniMax-M2 / GLM) and always live in the offload cache;
their swigluoai activation restricts the NVFP4 GEMM backend to the Triton kernels
(the NVFP4 MoE kernel objects). lm_head / embeddings are BF16 in the checkpoint.
"""

from __future__ import annotations

import os
from dataclasses import dataclass
from typing import Any

from freetoken.models.config import (
    FullAttentionGroupConfig,
    ModelConfig,
    RotaryConfig,
)

from .args import load_args

# the config names only the image token; the reference processor reads these two from the tokenizer's fixed added tokens
IMAGE_START_ID = 200029  # ]<]start of image[>[
IMAGE_END_ID = 200030  # ]<]end of image[>[


def _text_config(hf_config: Any) -> Any:
    return getattr(hf_config, "text_config", None) or hf_config


@dataclass(frozen=True)
class VisionConfig:
    hidden_size: int
    num_layers: int
    num_heads: int
    intermediate_size: int
    num_channels: int
    patch_size: int
    temporal_patch_size: int
    spatial_merge_size: int
    layer_norm_eps: float
    rope_theta: float
    projector_hidden_size: int
    text_hidden_size: int


def _compression(vc: Any, key: str) -> int:
    """The native vision config carries the merge sizes flat; the checkpoint's own config nests them under img_token_compression_config."""
    value = getattr(vc, key, None)
    return int(vc.img_token_compression_config[key] if value is None else value)


def parse_vision_config(hf_config: Any) -> VisionConfig | None:
    """None when the config carries no vision section, which is how a text-only engine asks for no tower."""
    vc = getattr(hf_config, "vision_config", None)
    if vc is None:
        return None
    if vc.hidden_act != "gelu":
        raise NotImplementedError(f"minimax_m3 vision tower activation {vc.hidden_act!r}; only gelu is implemented")
    # the native config moves rope_theta into rope_parameters; the checkpoint's own config keeps it flat
    rope_params = getattr(vc, "rope_parameters", None) or {}
    text_hidden_size = _text_config(hf_config).hidden_size
    return VisionConfig(
        hidden_size=vc.hidden_size,
        num_layers=vc.num_hidden_layers,
        num_heads=vc.num_attention_heads,
        intermediate_size=vc.intermediate_size,
        num_channels=vc.num_channels,
        patch_size=vc.patch_size,
        temporal_patch_size=_compression(vc, "temporal_patch_size"),
        spatial_merge_size=_compression(vc, "spatial_merge_size"),
        layer_norm_eps=vc.layer_norm_eps,
        rope_theta=float(rope_params["rope_theta"] if "rope_theta" in rope_params else vc.rope_theta),
        projector_hidden_size=getattr(hf_config, "projector_hidden_size", None) or text_hidden_size,
        text_hidden_size=text_hidden_size,
    )


# House rule: an env switch that changes WHAT IS SERVED must leave a server-log
# trace. parse_config runs a few times per process (engine + weight loader), so
# each resolved-mode line is logged once per distinct value.
_LOGGED_MODES: set = set()


def _log_mode_once(key: str, message: str) -> None:
    if key in _LOGGED_MODES:
        return
    _LOGGED_MODES.add(key)
    from freetoken.utils import init_logger

    init_logger(__name__).info(message)


def parse_config(hf_config: Any) -> ModelConfig:
    text = _text_config(hf_config)

    num_layers = text.num_hidden_layers
    # Dev/testing only: cap the layer count (e.g. FREETOKEN_M3_MAX_LAYERS=5 -> 3 dense
    # + 2 MoE layers) so the forward path / KV / offload cache can be exercised without
    # pinning the full ~230 GB of experts. Unset in normal use (GLM-5.2 precedent).
    _cap = os.environ.get("FREETOKEN_M3_MAX_LAYERS")
    if _cap and int(_cap) < num_layers:  # a cap >= the real count changes nothing
        num_layers = int(_cap)
        _log_mode_once(
            f"cap={num_layers}",
            f"FREETOKEN_M3_MAX_LAYERS: serving a TRUNCATED model "
            f"({num_layers}/{text.num_hidden_layers} layers) -- dev/testing only, "
            "outputs are garbage; unset the env for real serving.",
        )

    sparse_enabled = os.getenv("FREETOKEN_M3_SPARSE", "1") != "0"
    if not sparse_enabled:
        _log_mode_once(
            "sparse=0",
            "FREETOKEN_M3_SPARSE=0: serving the DENSE-attention ablation (plain "
            "MHA pool, no indexer; different math and outputs from the shipped "
            "block-sparse model).",
        )
    args = load_args(text, num_layers, sparse_enabled=sparse_enabled)

    # the quantized release stores the dense projections MXFP8; the reader keeps them native and the layers take their scheme from the QuantConfig
    dense_mxfp8 = getattr(hf_config, "quantization_config", None) is not None

    # Leading dense-FFN layers: M3's moe_layer_freq is a contiguous 0-prefix; the
    # offload cache and the generic num_moe_layers arithmetic rely on that shape.
    first_k_dense = num_layers - len(args.moe_layer_ids)
    assert args.moe_layer_ids == tuple(range(first_k_dense, num_layers)), (
        "MiniMax-M3 expects a contiguous dense prefix in moe_layer_freq, got "
        f"{args.moe_layer_ids}"
    )

    rotary_config = RotaryConfig(
        head_dim=args.head_dim,
        rotary_dim=args.rotary_dim,
        max_position=args.max_position,
        base=args.rope_theta,
        scaling=None,  # plain NeoX partial rope, no scaling
    )

    # Routed experts: NVFP4 per the modelopt MIXED_PRECISION quantized_layers map
    # (checked structurally by the bank loader's shape/dtype asserts). The generic
    # detect_expert_quant would report "mixed_precision", which names the checkpoint,
    # not the experts -- so the expert format is pinned here (qwen3_5_moe precedent).
    quant = getattr(hf_config, "quantization_config", None)
    expert_quant = "nvfp4" if quant is not None else "none"

    # The activation is model-constant swigluoai: assert instead of consume.
    # Raw checkpoints declare "swigluoai"; the native transformers TextConfig
    # force-normalizes hidden_act to "silu", which would send the backend
    # selection down a silu-only path.
    declared_act = getattr(text, "hidden_act", "swigluoai")
    assert declared_act in ("swigluoai", "silu"), (
        f"MiniMax-M3 support implements swigluoai only, got hidden_act={declared_act!r}"
    )

    return ModelConfig(
        num_layers=num_layers,
        num_qo_heads=args.num_heads,
        num_kv_heads=args.num_kv_heads,
        head_dim=args.head_dim,
        hidden_size=args.hidden_size,
        vocab_size=text.vocab_size,
        intermediate_size=args.dense_intermediate_size,
        hidden_act="swigluoai",  # pinned; see the declared_act assert above
        rms_norm_eps=text.rms_norm_eps,
        tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)),
        rotary_config=rotary_config,
        # One group over all layers: GQA K/V everywhere, plus the index-key slab for
        # the sparse layers (mla=False + index dims -> AttnType.BSA -> BSAKVCache).
        # The same spec drives the pool factory and the KV cost model, so they can
        # never disagree. FREETOKEN_M3_SPARSE=0 zeroes the index dims (dense ablation:
        # plain MHAKVCache, generic FULL backends serve every layer).
        attention_groups=(
            FullAttentionGroupConfig(
                name="full",
                layer_ids=tuple(range(num_layers)),
                num_kv_heads=args.num_kv_heads,
                head_dim=args.head_dim,
                rotary_config=rotary_config,
                mla=False,
                index_head_dim=args.index_dim if args.use_sparse else 0,
                num_index_layers=(
                    sum(1 for lid in args.sparse_layer_ids if lid < num_layers)
                    if args.use_sparse
                    else 0
                ),
            ),
        ),
        num_experts=int(getattr(text, "num_local_experts", 0)),
        num_experts_per_tok=text.num_experts_per_tok,
        # Routed experts use the text tower's intermediate_size (3072); the dense
        # MLP width (12288) rides ModelConfig.intermediate_size above.
        moe_intermediate_size=text.intermediate_size,
        norm_topk_prob=True,  # MiniMax always renormalizes the selected expert weights
        model_type=getattr(hf_config, "model_type", "minimax_m3_vl"),
        architectures=getattr(
            hf_config, "architectures", ["MiniMaxM3SparseForConditionalGeneration"]
        ),
        moe_enabled=True,
        expert_quant=expert_quant,
        first_k_dense_replace=first_k_dense,
        n_shared_experts=int(getattr(text, "n_shared_experts", 0)),
        shared_expert_intermediate_size=args.shared_intermediate_size,
        routed_scaling_factor=float(getattr(text, "routed_scaling_factor", 1.0)),
        has_router_bias=bool(getattr(text, "use_routing_bias", False)),
        use_qk_norm=bool(getattr(text, "use_qk_norm", True)),
        swiglu_limit=args.swiglu_limit,
        hidden_act_alpha=args.swiglu_alpha,
        # Resident-weight quant modes (see the module docstring): recorded here so the
        # resolved config describes the served weights; the constructors and
        # iter_weights read these fields, never the env directly. lm_head stays bf16
        # (the checkpoint excludes it from quantization).
        attn_quant="mxfp8" if dense_mxfp8 else "none",
        dense_quant="mxfp8" if dense_mxfp8 else "none",
        lm_head_quant="none",
        m3_args=args,
        vision_config=parse_vision_config(hf_config),
        # the native config maps image_token_id onto the checkpoint's image_token_index
        image_token_id=getattr(hf_config, "image_token_id", getattr(hf_config, "image_token_index", None)),
    )


__all__ = ["IMAGE_END_ID", "IMAGE_START_ID", "VisionConfig", "parse_config", "parse_vision_config"]