Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

17.0 KB
Raw Blame
"""MiniMax-M3 from a llama.cpp ``minimax-m3`` GGUF: config from the GGUF metadata, tensors by name; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``).

The llama.cpp converter (conversion/minimax.py) stacks the w1 / w2 / w3 experts, stores the indexer projections as F32, and adds 1 to every RMSNorm weight so llama.cpp can run plain RMSNorm; the loader subtracts the 1 for this family's (1 + w) norms.
Both sides run NeoX rope, so q/k carry no rope permute.

A ``minimax_m3`` mmproj beside the text file adds the vision tower under ``vision_tower.``. The converter splits the Conv3d patch embedding into its two temporal halves (``v.patch_embd.weight`` and ``.weight.1``) and reorders the rows of each ViT head's q / k from HF's 3-axis rope layout [t h w | t h w | pad] to [t t | h h | w w | pad]; both are undone at load, every other tower tensor is stored in HF layout.
"""

from __future__ import annotations

import re
from dataclasses import replace
from types import SimpleNamespace
from typing import TYPE_CHECKING, Any, Iterator

import numpy as np
import torch

from freetoken.models.gguf.mmproj import Mmproj, find_mmproj, log_mmproj, mmproj_tensor
from freetoken.models.gguf.reader import (
    GgufTensorInfo,
    gguf_tensor_index,
    load_gguf_metadata,
)
from freetoken.models.gguf.tensors import float_tensor, fused, matrix

from .config import IMAGE_END_ID, IMAGE_START_ID, parse_config

if TYPE_CHECKING:
    from freetoken.models.config import ModelConfig
    from freetoken.models.gguf.config import GgufConfigShim

_ARCH = "minimax-m3"
_LAYER_LEAVES = {
    "self_attn.q_proj": "attn_q",
    "self_attn.k_proj": "attn_k",
    "self_attn.v_proj": "attn_v",
    "self_attn.o_proj": "attn_output",
    "self_attn.index_q_proj": "indexer.q_proj",
    "self_attn.index_k_proj": "indexer.k_proj",
    "mlp.gate_proj": "ffn_gate",
    "mlp.up_proj": "ffn_up",
    "mlp.down_proj": "ffn_down",
    "block_sparse_moe.gate": "ffn_gate_inp",
    "block_sparse_moe.shared_experts.gate_proj": "ffn_gate_shexp",
    "block_sparse_moe.shared_experts.up_proj": "ffn_up_shexp",
    "block_sparse_moe.shared_experts.down_proj": "ffn_down_shexp",
    "block_sparse_moe.experts.0.w1": "ffn_gate_exps",
    "block_sparse_moe.experts.0.w3": "ffn_up_exps",
    "block_sparse_moe.experts.0.w2": "ffn_down_exps",
}
_TOP_LEAVES = {"model.embed_tokens": "token_embd", "lm_head": "output"}
_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")
_SIGMOID_GATING = 2
# llama.cpp hard-codes these for the arch (src/llama-graph.cpp LLM_FFN_SWIGLU_OAI_MOE)
_SWIGLU_ALPHA, _SWIGLU_LIMIT = 1.702, 7.0


def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
    """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""
    if name in _TOP_LEAVES:
        return (_TOP_LEAVES[name],)
    m = _LAYER_RE.match(name)
    if m is None:
        return ()
    tensor = _LAYER_LEAVES.get(m.group(2))
    return (f"blk.{m.group(1)}.{tensor}",) if tensor is not None else ()


def _check_supported(shim: GgufConfigShim) -> None:
    if any(k.startswith("prism.hadamard.") for k in shim.metadata):
        raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")
    scaled = sorted(n for n in gguf_tensor_index(shim.model_path) if n.endswith((".scale", ".input_scale")))
    if scaled:
        raise NotImplementedError(f"{shim.model_path}: GGUF tensors with per-tensor scales are not supported (e.g. {scaled[0]})")


def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:
    """The HF text config fields ``parse_config`` reads, from the GGUF metadata."""
    md = shim.metadata

    def kv(key: str):
        return md[f"{_ARCH}.{key}"]

    _check_supported(shim)
    if f"{_ARCH}.attention.indexer.head_count" not in md:
        raise NotImplementedError(f"{shim.model_path}: converted by an older llama.cpp without the sparse-attention indexer, which current llama.cpp refuses too")
    gating = int(kv("expert_gating_func"))
    if gating != _SIGMOID_GATING:
        raise NotImplementedError(f"{shim.model_path}: expert gating function {gating} is not supported; MiniMax-M3 routes with a sigmoid (2)")
    if not md.get(f"{_ARCH}.expert_weights_norm", False):
        raise NotImplementedError(f"{shim.model_path}: routing without renormalized expert weights is not supported")
    scaling = md.get(f"{_ARCH}.rope.scaling.type", "none")
    if scaling != "none":
        raise NotImplementedError(f"{shim.model_path}: rope scaling {scaling!r} is not supported")
    expert_ff = kv("expert_feed_forward_length")
    if isinstance(expert_ff, list):
        if len(set(expert_ff)) != 1:
            raise NotImplementedError(f"{shim.model_path}: per-layer expert sizes {sorted(set(expert_ff))} are not supported")
        expert_ff = expert_ff[0]
    num_layers = int(kv("block_count"))
    dense = int(md.get(f"{_ARCH}.leading_dense_block_count", 0))
    # llama.cpp runs sparse attention and the MoE FFN on every layer after the leading dense ones
    sparse = [0] * dense + [1] * (num_layers - dense)
    return SimpleNamespace(
        model_type="minimax_m3",
        architectures=list(shim.architectures),
        # filled by a minimax_m3 mmproj (gguf_mm_config); a text-only process nulls vision_config on its copy
        vision_config=getattr(shim, "vision_config", None),
        projector_hidden_size=getattr(shim, "projector_hidden_size", None),
        image_token_id=getattr(shim, "image_token_id", None),
        hidden_act="swigluoai",
        swiglu_alpha=_SWIGLU_ALPHA,
        swiglu_limit=_SWIGLU_LIMIT,
        hidden_size=int(kv("embedding_length")),
        num_hidden_layers=num_layers,
        num_attention_heads=int(kv("attention.head_count")),
        num_key_value_heads=int(kv("attention.head_count_kv")),
        head_dim=int(kv("attention.key_length")),
        rotary_dim=int(kv("rope.dimension_count")),
        rope_theta=float(kv("rope.freq_base")),
        max_position_embeddings=int(kv("context_length")),
        rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),
        vocab_size=int(shim.vocab_size),
        tie_word_embeddings=bool(shim.tie_word_embeddings),
        dense_intermediate_size=int(kv("feed_forward_length")),
        intermediate_size=int(expert_ff),
        shared_intermediate_size=int(expert_ff),
        n_shared_experts=int(kv("expert_shared_count")),
        num_local_experts=int(kv("expert_count")),
        num_experts_per_tok=int(kv("expert_used_count")),
        # llama.cpp skips the scale when the key is absent or 0
        routed_scaling_factor=float(md.get(f"{_ARCH}.expert_weights_scale", 0.0)) or 1.0,
        use_routing_bias=True,
        moe_layer_freq=sparse,
        sparse_attention_config={
            "use_sparse_attention": True,
            "sparse_attention_freq": sparse,
            "sparse_num_index_heads": int(kv("attention.indexer.head_count")),
            "sparse_index_dim": int(kv("attention.indexer.key_length")),
            "sparse_topk_blocks": int(kv("attention.indexer.top_k")),
            "sparse_block_size": int(kv("attention.indexer.block_size")),
            "sparse_local_block": int(kv("attention.indexer.local_blocks")),
            "sparse_init_block": 0,
            "sparse_score_type": "max",
        },
    )


def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:
    return replace(parse_config(_hf_config(shim)), expert_quant="gguf")


# ----------------------------------------------------------------------------------------------
# Vision tower (a minimax_m3 mmproj)
# ----------------------------------------------------------------------------------------------

_PROJECTOR = "minimax_m3"
_IMAGE_TOKEN = "]<]image[>["
_WRAPPERS = {"]<]start of image[>[": IMAGE_START_ID, "]<]end of image[>[": IMAGE_END_ID}
# no mmproj key carries the rope theta; llama.cpp hardcodes the HF value too (clip.cpp)
_ROPE_THETA = 10000.0
# MiniMaxM3VLImageProcessor's own size: the HF preprocessor_config.json stores a stale [672, 672] that the class replaces with this
_IMAGE_SIZE = {"shortest_edge": 3136, "longest_edge": 451584}
_PATCH = ("v.patch_embd.weight", "v.patch_embd.weight.1")
_TOWER = "vision_tower"

_TOP = {
    "v.pre_ln.weight": "vision_model.pre_layrnorm.weight",
    "v.pre_ln.bias": "vision_model.pre_layrnorm.bias",
    "mm.1.weight": "multi_modal_projector.linear_1.weight",
    "mm.1.bias": "multi_modal_projector.linear_1.bias",
    "mm.2.weight": "multi_modal_projector.linear_2.weight",
    "mm.2.bias": "multi_modal_projector.linear_2.bias",
    "mm.merger.fc1.weight": "patch_merge_mlp.linear_1.weight",
    "mm.merger.fc1.bias": "patch_merge_mlp.linear_1.bias",
    "mm.merger.fc2.weight": "patch_merge_mlp.linear_2.weight",
    "mm.merger.fc2.bias": "patch_merge_mlp.linear_2.bias",
}
_BLOCK = {
    "attn_out": "self_attn.out_proj",
    "ln1": "layer_norm1",
    "ln2": "layer_norm2",
    "ffn_up": "mlp.fc1",
    "ffn_down": "mlp.fc2",
}
_QKV = ("attn_q", "attn_k", "attn_v")
_BLOCK_RE = re.compile(r"^v\.blk\.(\d+)\.(\w+)\.(weight|bias)$")


def _f32_value(v: float) -> float:
    """The shortest decimal of a float32 metadata value: 1e-05, as the HF config writes it, not 9.99999975e-06."""
    return float(str(np.float32(v)))


def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int, *, model_path: str = "") -> tuple[dict[str, Any], dict[str, Any]]:
    """The HF config.json keys and the preprocessor_config.json of the tower in ``mmproj``, for the text GGUF whose metadata is ``text_md``."""
    if mmproj.projector != _PROJECTOR:
        raise ValueError(f"mmproj {mmproj.path} is a {mmproj.projector!r} tower; MiniMax-M3 needs {_PROJECTOR!r}")
    md, index = load_gguf_metadata(mmproj.path), gguf_tensor_index(mmproj.path)

    def kv(key: str):
        return md[f"clip.vision.{key}"]

    text_hidden = int(text_md[f"{_ARCH}.embedding_length"])
    if int(kv("projection_dim")) != text_hidden:
        raise ValueError(f"mmproj {mmproj.path} projects to {int(kv('projection_dim'))} but the text model is {text_hidden} wide")
    tokens = text_md["tokenizer.ggml.tokens"]
    if _IMAGE_TOKEN not in tokens:
        raise ValueError(f"the text GGUF tokenizer has no {_IMAGE_TOKEN} token")
    # the processor hardcodes the wrapper ids; a tokenizer with them elsewhere would fail per request instead of here
    for token, want in _WRAPPERS.items():
        got = tokens.index(token) if token in tokens else None
        if got != want:
            raise ValueError(f"the text GGUF tokenizer has {token} at id {got}, not the family's {want}")
    # exactly the tensor names llama.cpp loads, both patch-embedding halves included; other spellings (mm.merge.*) fail there too
    unexpected = [n for n in index if n not in _PATCH and n not in _TOP and not ((m := _BLOCK_RE.match(n)) and m[2] in (*_BLOCK, *_QKV))]
    missing = [n for n in (*_PATCH, *_TOP) if n not in index]
    if unexpected or missing:
        raise ValueError(f"mmproj {mmproj.path} is not in the minimax_m3 tensor layout (unexpected {unexpected[:3]}, missing {missing[:3]})")
    merge = int(kv("spatial_merge_size"))
    vision_config = {
        # llama.cpp's minimax_m3 graph runs erf-GELU whatever clip.use_gelu says
        "hidden_act": "gelu",
        "hidden_size": int(kv("embedding_length")),
        "num_hidden_layers": int(kv("block_count")),
        "num_attention_heads": int(kv("attention.head_count")),
        "intermediate_size": int(kv("feed_forward_length")),
        # ggml order [kw, kh, in, out]
        "num_channels": int(index[_PATCH[0]].ne[2]),
        "patch_size": int(kv("patch_size")),
        "temporal_patch_size": 2,
        "spatial_merge_size": merge,
        "layer_norm_eps": _f32_value(kv("attention.layer_norm_epsilon")),
        "rope_theta": _ROPE_THETA,
    }
    hf_fields = {
        "vision_config": vision_config,
        # ggml order [in, out]
        "projector_hidden_size": int(index["mm.1.weight"].ne[1]),
        "image_token_id": tokens.index(_IMAGE_TOKEN),
    }
    preprocessor = {
        "image_processor_type": "MiniMaxM3VLImageProcessor",
        "size": dict(_IMAGE_SIZE),
        "patch_size": vision_config["patch_size"],
        "temporal_patch_size": 2,
        "merge_size": merge,
        "image_mean": [float(v) for v in kv("image_mean")],
        "image_std": [float(v) for v in kv("image_std")],
    }
    return hf_fields, preprocessor


def _hf_qk_rows(head_dim: int) -> torch.Tensor:
    """The mmproj row of each HF q / k row within one head (the inverse of conversion/minimax.py ``_permute_vit_qk``)."""
    band = (2 * (head_dim // 2) // 3) // 2
    stored = []
    for axis in range(3):
        stored += [*range(axis * band, (axis + 1) * band), *range((3 + axis) * band, (4 + axis) * band)]
    stored += range(6 * band, head_dim)
    return torch.argsort(torch.tensor(stored))


def iter_mmproj_weights(mmproj: Mmproj) -> Iterator[tuple[str, torch.Tensor]]:
    """The tower state dict under ``vision_tower.``, named as the HF checkpoint loads (q / k / v fused)."""
    md, index = load_gguf_metadata(mmproj.path), gguf_tensor_index(mmproj.path)
    heads = int(md["clip.vision.attention.head_count"])
    rows = _hf_qk_rows(int(md["clip.vision.embedding_length"]) // heads)

    def hf_qk(t: torch.Tensor) -> torch.Tensor:
        return t.reshape(heads, rows.numel(), -1)[:, rows].reshape(t.shape)

    yield f"{_TOWER}.vision_model.embeddings.patch_embedding.weight", torch.stack([mmproj_tensor(index[n]) for n in _PATCH], dim=2)
    for name, info in index.items():
        if name in _PATCH:
            continue
        if name in _TOP:
            yield f"{_TOWER}.{_TOP[name]}", mmproj_tensor(info)
            continue
        # gguf_mm_config refused any other name
        layer, part, kind = _BLOCK_RE.match(name).groups()
        dst = f"{_TOWER}.vision_model.encoder.layers.{layer}"
        if part in _BLOCK:
            yield f"{dst}.{_BLOCK[part]}.{kind}", mmproj_tensor(info)
        elif part == "attn_q":
            q, k, v = (mmproj_tensor(index[f"v.blk.{layer}.{p}.{kind}"]) for p in _QKV)
            yield f"{dst}.self_attn.qkv.{kind}", torch.cat([hf_qk(q), hf_qk(k), v])


def iter_gguf_weights(
    model_path: str,
    device: torch.device,
    *,
    include_moe_experts: bool,
    include_non_moe: bool,
    include_vision: bool = True,
) -> Iterator[tuple[str, torch.Tensor]]:
    """The dense state dict from the GGUF (and the tower from its mmproj); the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``)."""
    if not include_non_moe:
        return
    from freetoken.utils import cached_load_hf_config

    config = parse_gguf_config(cached_load_hf_config(model_path))
    vision = include_vision and config.is_multimodal
    log_mmproj(model_path, vision)
    args = config.m3_args
    index = gguf_tensor_index(model_path)
    hidden = config.hidden_size

    def t(name: str) -> GgufTensorInfo:
        return index[name if name in index else f"{name}.weight"]

    def norm(name: str) -> torch.Tensor:
        return float_tensor(t(name)).float() - 1.0

    yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)
    if not config.tie_word_embeddings:
        yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)
    yield "model.norm.weight", norm("output_norm")

    for layer in range(config.num_layers):
        blk, pre = f"blk.{layer}", f"model.layers.{layer}"
        yield f"{pre}.input_layernorm.weight", norm(f"{blk}.attn_norm")
        yield f"{pre}.post_attention_layernorm.weight", norm(f"{blk}.ffn_norm")
        sa = f"{pre}.self_attn"
        qkv = [t(f"{blk}.attn_{p}") for p in ("q", "k", "v")]
        yield from fused(f"{sa}.qkv_proj", qkv, [matrix(p) for p in qkv], hidden)
        out = t(f"{blk}.attn_output")
        yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))
        yield f"{sa}.q_norm.weight", norm(f"{blk}.attn_q_norm")
        yield f"{sa}.k_norm.weight", norm(f"{blk}.attn_k_norm")
        if args.is_sparse_layer(layer):
            iq, ik = t(f"{blk}.indexer.q_proj"), t(f"{blk}.indexer.k_proj")
            yield from fused(f"{sa}.index_qk_proj", [iq, ik], [matrix(iq), matrix(ik)], hidden)
            yield f"{sa}.index_q_norm.weight", norm(f"{blk}.indexer.q_norm")
            yield f"{sa}.index_k_norm.weight", norm(f"{blk}.indexer.k_norm")
        if layer in args.moe_layer_ids:
            moe = f"{pre}.block_sparse_moe"
            yield f"{moe}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp")).float()
            yield f"{moe}.e_score_correction_bias", float_tensor(t(f"{blk}.exp_probs_b.bias")).float()
            mlp, parts = f"{moe}.shared_experts", [t(f"{blk}.ffn_{p}_shexp") for p in ("gate", "up", "down")]
        else:
            mlp, parts = f"{pre}.mlp", [t(f"{blk}.ffn_{p}") for p in ("gate", "up", "down")]
        gate, up, down = parts
        yield from fused(f"{mlp}.gate_up_proj", [gate, up], [matrix(gate), matrix(up)], hidden)
        yield from fused(f"{mlp}.down_proj", [down], [matrix(down)], int(down.ne[0]))

    if vision:
        yield from iter_mmproj_weights(find_mmproj(model_path))


__all__ = ["gguf_mm_config", "iter_gguf_weights", "iter_mmproj_weights", "parse_gguf_config", "to_gguf"]