Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

34.9 KB
Raw Blame
"""What each checkpoint's quant config resolves to: the dialect, the method behind every layer, and agreement with the stored tensors.

Rows need the checkpoint's config.json locally (/mnt/nvme/models or the HF cache); absent ones skip. The
stored-tensor scan also reads $FREETOKEN_TEST_MODELS_ROOT.
"""

from __future__ import annotations

import glob
import json
import os
import re
import struct
from collections import defaultdict
from dataclasses import dataclass, field
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Callable

import pytest
import torch

from freetoken.distributed.info import set_tp_info, try_get_tp_info
from freetoken.engine.config import EngineConfig
from freetoken.layers import set_rope_device
from freetoken.layers.quantization import (
    CompressedTensorsConfig,
    Fp8BlockConfig,
    ModelOptConfig,
    Mxfp4Config,
    NoQuantConfig,
    QuantConfig,
    QuantKind,
)
from freetoken.layers.quantization.linear import (
    Fp8BlockLinearMethod,
    Fp8TensorLinearMethod,
    Mxfp8LinearMethod,
    Nvfp4LinearMethod,
    UnquantizedLinearMethod,
)
from freetoken.layers.quantization.moe import Fp8BlockMoEMethod, Mxfp4MoEMethod, Nvfp4MoEMethod, UnquantizedMoEMethod
from freetoken.models import create_model
from freetoken.models.gpt_oss.moe import GptOssMoELayer, GptOssOffloadMoELayer
from freetoken.models.register import checkpoint_quant_config, get_model_spec
from freetoken.utils.hf import sidecar_quantization_config
from freetoken.utils.torch_utils import torch_dtype

MODELS = "/mnt/nvme/models"
# the stored-tensor scan takes every checkpoint under this root (the case table above matches by name)
SCAN_ROOT = os.environ.get("FREETOKEN_TEST_MODELS_ROOT", MODELS)
HF_CACHE = os.path.expanduser("~/.cache/huggingface/hub")

BF16, FP8B, FP8T, MXFP8, NVFP4 = UnquantizedLinearMethod, Fp8BlockLinearMethod, Fp8TensorLinearMethod, Mxfp8LinearMethod, Nvfp4LinearMethod


def model_dir(name: str) -> str | None:
    path = os.path.join(MODELS, name.split("/")[-1])
    if os.path.isfile(os.path.join(path, "config.json")):  # a download in progress has the dir but no config yet
        return path
    snaps = glob.glob(os.path.join(HF_CACHE, "models--" + name.replace("/", "--"), "snapshots", "*", "config.json"))
    return os.path.dirname(snaps[0]) if snaps else None


def op_at(model, path: str):
    op = model
    for part in path.split("."):
        op = op.op_list[int(part)] if part.isdigit() else getattr(op, part)
    return op


def build_meta_model(path: str, strategy: str):
    if try_get_tp_info() is None:
        set_tp_info(0, 1)
    set_rope_device(torch.device("cpu"))
    config = EngineConfig(model_path=path, tp_info=try_get_tp_info(), dtype=torch.bfloat16, moe_strategy=strategy)
    from freetoken.engine.engine import _decode_target

    object.__setattr__(config.model_config, "moe_strategy", strategy)
    object.__setattr__(config.model_config, "decode_target", _decode_target(config))
    with torch.device("meta"), torch_dtype(torch.bfloat16):
        return create_model(config.model_config), config.model_config


def _layer_indices(model) -> dict[str, int | None]:
    """First layer of each kind, for the {moe} {dense} {lin} {full} {dsa} {kda} {ple} placeholders in probe paths."""
    layers = model.model.layers.op_list

    def first(pred):
        return next((i for i, layer in enumerate(layers) if pred(layer)), None)

    def has_experts(layer):
        if getattr(layer, "is_moe_layer", None) is not None:
            return layer.is_moe_layer
        block = next((getattr(layer, n) for n in ("mlp", "feed_forward", "block_sparse_moe", "ffn") if hasattr(layer, n)), None)
        return block is not None and hasattr(block, "experts")

    def is_linear_attn(layer):
        return getattr(layer, "_is_linear", False) or hasattr(layer, "linear_attn")

    attn = lambda layer: getattr(layer, "self_attn", None)
    ple = getattr(getattr(model, "_config", None), "qwen4_args", None)
    return {
        "moe": first(has_experts),
        "dense": first(lambda l: not has_experts(l)),
        "lin": first(is_linear_attn),
        "full": first(lambda l: not is_linear_attn(l)),
        "dsa": first(lambda l: getattr(attn(l), "indexer", None) is not None),
        "kda": first(lambda l: hasattr(attn(l), "in_proj")),
        "ple": ple.ple_layer_ids[0] if ple is not None else None,
    }


@dataclass(frozen=True)
class Case:
    name: str
    dialect: type
    methods: dict[str, Any]  # probe path -> method class, (class, kernel name), or None for a layer without a method
    strategy: str = "offload"
    raises: type | None = None
    skip_if: Callable[[], bool] | None = None
    env: dict[str, str] = field(default_factory=dict)
    check: Callable[[Any, dict], None] | None = None  # family extras beyond the method table


def _qwen35_fp8(model, idx):
    assert model.model.layers.op_list[idx["lin"]].linear_attn._split_in_proj


def _qwen35_nvfp4(model, idx):
    assert model.model.layers.op_list[idx["lin"]].linear_attn._split_in_proj
    assert op_at(model, f"model.layers.{idx['lin']}.linear_attn.in_proj_qkvz").quant_method.scheme.has("input_scale")
    assert model.lm_head.weight.dtype is torch.uint8


def _no_split(model, idx):
    assert not model.model.layers.op_list[idx["lin"]].linear_attn._split_in_proj


def _dsv4(model, idx):
    block = op_at(model, "model.layers.2")
    for proj in (block.attn.wq_a, block.ffn.shared_experts.w1):
        assert proj.weight.dtype == torch.float8_e4m3fn and proj.weight_scale_inv.dtype == torch.float8_e8m0fnu
        assert proj.weight_scale_inv.shape == (proj.out_features // 128, proj.in_features // 128) and proj.bias is None
    assert set(block.attn.compressor.wkv.state_dict()) == {"weight"}
    experts = block.ffn.experts
    assert experts.limit == 10.0 and experts.quant_method.cpu_format == "ds_fp4" and not experts.has_bias
    H, I = 4096, 2048
    assert {r: (s.shape, s.dtype) for r, s in experts.quant_method.layout().items()} == {
        "gate_up": ((2 * I, H // 2), torch.uint8), "gate_up_scale": ((2 * I, H // 32), torch.float8_e8m0fnu),
        "down": ((H, I // 2), torch.uint8), "down_scale": ((H, I // 32), torch.float8_e8m0fnu),
    }
    keys = set(model.state_dict())
    assert "model.layers.2.attn.wq_a.weight_scale_inv" in keys and "model.layers.2.attn.wq_a.scale" not in keys
    assert "model.layers.2.attn.compressor.wkv.weight_scale_inv" not in keys
    assert "model.head.weight" in keys and not any(".ffn.experts." in k for k in keys)


def _banked_experts(model, idx):
    experts = op_at(model, f"model.layers.{idx['lin']}.mlp.experts")
    assert not experts.state_dict()


def _gpt_oss(cls):
    def check(model, idx):
        experts = op_at(model, "model.layers.0.mlp.experts")
        assert type(experts) is cls
        assert experts.interleaved and experts.alpha == pytest.approx(1.702) and experts.limit == 7.0
        if cls is GptOssMoELayer:
            assert not experts.state_dict()

    return check


def _m3_sparse(model, idx):
    layers = model.model.layers.op_list
    sparse = next((i for i, l in enumerate(layers) if getattr(l.self_attn, "is_sparse", False)), None)
    if sparse is not None:
        assert type(op_at(model, f"model.layers.{sparse}.self_attn.index_qk_proj").quant_method) is MXFP8


def _gemma_tied_head(model, idx):
    assert model.lm_head.quant_method is None


CASES = [
    Case("Qwen3.6-35B-A3B-FP8", Fp8BlockConfig, {
        "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8B, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
        "model.layers.{lin}.linear_attn.out_proj": FP8B, "model.layers.{full}.self_attn.qkv_proj": FP8B,
        "model.layers.{full}.self_attn.o_proj": FP8B, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": FP8B,
        "model.layers.{lin}.mlp.shared_expert.down_proj": FP8B, "model.layers.{lin}.mlp.experts": Fp8BlockMoEMethod,
        "model.layers.{lin}.mlp.gate": BF16, "model.layers.{lin}.mlp.shared_expert_gate": BF16, "lm_head": BF16,
    }, check=_qwen35_fp8),
    Case("Qwen3.6-35B-A3B-NVFP4", ModelOptConfig, {
        "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
        "model.layers.{full}.self_attn.qkv_proj": FP8T, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": NVFP4,
        "model.layers.{lin}.mlp.shared_expert.down_proj": NVFP4, "model.layers.{lin}.mlp.experts": Nvfp4MoEMethod,
        "lm_head": NVFP4,
    }, check=_qwen35_nvfp4),
    # resident NVFP4 experts load through the expert-bank loader, not the state dict
    Case("Qwen3.6-35B-A3B-NVFP4", ModelOptConfig, {"model.layers.{lin}.mlp.experts": (Nvfp4MoEMethod, "triton")}, strategy="fused", check=_banked_experts),
    Case("Qwen3.8-27B-NVFP4", ModelOptConfig, {
        "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{full}.self_attn.o_proj": FP8T,
        "model.layers.{lin}.mlp.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.down_proj": NVFP4, "lm_head": NVFP4,
    }, strategy="auto"),
    # llm-compressor exports of the dense 27B: every Linear NVFP4, GDN in_proj included (sakamakismile); channel-fp8 attention / GDN / lm_head over NVFP4 MLPs (unsloth)
    Case("sakamakismile/Qwen3.6-27B-NVFP4", CompressedTensorsConfig, {
        "model.layers.{lin}.linear_attn.in_proj_qkvz": NVFP4, "model.layers.{lin}.linear_attn.in_proj_ba": NVFP4,
        "model.layers.{lin}.linear_attn.out_proj": NVFP4, "model.layers.{full}.self_attn.qkv_proj": NVFP4,
        "model.layers.{lin}.mlp.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.down_proj": NVFP4, "lm_head": BF16,
    }, strategy="auto", check=_qwen35_fp8),
    Case("unsloth/Qwen3.8-27B-NVFP4", CompressedTensorsConfig, {
        "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
        "model.layers.{lin}.linear_attn.out_proj": FP8T, "model.layers.{full}.self_attn.o_proj": FP8T,
        "model.layers.{lin}.mlp.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.down_proj": NVFP4, "lm_head": FP8T,
    }, strategy="auto", check=_qwen35_fp8),
    # llm-compressor exports of the 35B MoE: NVFP4 everywhere but the GDN and the routers (RedHatAI); channel-fp8 attention / GDN / lm_head with NVFP4 experts and shared expert (unsloth)
    Case("RedHatAI/Qwen3.6-35B-A3B-NVFP4", CompressedTensorsConfig, {
        "model.layers.{lin}.linear_attn.in_proj": BF16, "model.layers.{lin}.linear_attn.out_proj": BF16,
        "model.layers.{full}.self_attn.qkv_proj": NVFP4, "model.layers.{full}.self_attn.o_proj": NVFP4,
        "model.layers.{lin}.mlp.shared_expert.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.experts": Nvfp4MoEMethod,
        "model.layers.{lin}.mlp.gate": BF16, "lm_head": BF16,
    }, check=_no_split),
    # static per-tensor fp8 attention / GDN / shared expert with NVFP4 experts; the ignore list names every ``experts.N`` container
    Case("primitive-ai/Ornith-1.5-35B-A3B-mixed-NVFP4-FP8", CompressedTensorsConfig, {
        "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
        "model.layers.{full}.self_attn.qkv_proj": FP8T, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": FP8T,
        "model.layers.{lin}.mlp.experts": Nvfp4MoEMethod, "model.layers.{lin}.mlp.gate": BF16, "lm_head": BF16,
    }, check=_qwen35_fp8),
    # block-fp8 attention / GDN / shared expert with NVFP4 experts, shipped as one model.safetensors without an index
    Case("kyaky/Qwen3.6-35B-A3B-Uncensored-NVFP4", CompressedTensorsConfig, {
        "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8B, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
        "model.layers.{full}.self_attn.qkv_proj": FP8B, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": FP8B,
        "model.layers.{lin}.mlp.experts": Nvfp4MoEMethod, "model.layers.{lin}.mlp.gate": BF16, "lm_head": BF16,
    }, check=_qwen35_fp8),
    Case("unsloth/Qwen3.6-35B-A3B-NVFP4-Fast", CompressedTensorsConfig, {
        "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
        "model.layers.{lin}.linear_attn.out_proj": FP8T, "model.layers.{full}.self_attn.qkv_proj": FP8T,
        "model.layers.{lin}.mlp.shared_expert.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.experts": Nvfp4MoEMethod,
        "model.layers.{lin}.mlp.gate": BF16, "lm_head": FP8T,
    }, check=_qwen35_fp8),
    Case("Qwen3.6-27B", NoQuantConfig, {
        "model.layers.{lin}.linear_attn.in_proj": BF16, "model.layers.{lin}.mlp.gate_up_proj": BF16, "lm_head": BF16,
    }, strategy="auto", check=_no_split),
    Case("Qwen3.6-35B-A3B", NoQuantConfig, {
        "model.layers.{lin}.mlp.experts": UnquantizedMoEMethod, "model.layers.{lin}.mlp.shared_expert.down_proj": BF16,
    }),
    Case("nvidia/Qwen3.6-27B-NVFP4", ModelOptConfig, {
        "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
        "model.layers.{lin}.linear_attn.out_proj": FP8T, "model.layers.{full}.self_attn.qkv_proj": FP8T,
        "model.layers.{lin}.mlp.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.down_proj": NVFP4, "lm_head": NVFP4,
    }, strategy="auto"),
    *[Case(name, Fp8BlockConfig, {
        "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8B, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
        "model.layers.{lin}.linear_attn.out_proj": FP8B, "model.layers.{full}.self_attn.qkv_proj": FP8B,
        "model.layers.{full}.self_attn.o_proj": FP8B, "model.layers.{lin}.mlp.gate_up_proj": FP8B,
        "model.layers.{lin}.mlp.down_proj": FP8B, "lm_head": BF16,
    }, strategy="auto") for name in ("Qwen/Qwen3.8-27B-FP8", "Qwen/Qwen3.6-27B-FP8")],
    Case("RedHatAI/Muse-Glimmer-30B-NVFP4", CompressedTensorsConfig, {
        "model.layers.0.self_attn.qkv_proj": NVFP4, "model.layers.0.self_attn.gate_proj": NVFP4,
        "model.layers.0.self_attn.o_proj": NVFP4,
        "model.layers.0.mlp.gate_up_proj": NVFP4, "model.layers.0.mlp.down_proj": NVFP4, "lm_head": BF16,
    }, strategy="auto"),
    Case("nvidia/Gemma-4-31B-IT-NVFP4", ModelOptConfig, {
        "model.layers.0.feed_forward.shared_mlp.gate_up_proj": NVFP4, "model.layers.0.feed_forward.shared_mlp.down_proj": NVFP4,
        "model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,
    }, strategy="auto", check=_gemma_tied_head),
    Case("google/gemma-4-12B-it", NoQuantConfig, {
        "model.layers.0.feed_forward.shared_mlp.gate_up_proj": BF16, "model.layers.0.self_attn.qkv_proj": BF16,
    }, strategy="auto", check=_gemma_tied_head),
    Case("Qwen/Qwen3-4B", NoQuantConfig, {
        "model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,
        "model.layers.0.mlp.gate_up_proj": BF16, "model.layers.0.mlp.down_proj": BF16,
    }, strategy="auto"),
    *[Case(name, NoQuantConfig, {
        "model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,
        "model.layers.0.mlp.gate_up_proj": BF16, "model.layers.0.mlp.down_proj": BF16,
    }, strategy="auto") for name in ("Qwen/Qwen2.5-1.5B-Instruct", "unsloth/Llama-3.2-1B-Instruct", "mistralai/Mistral-7B-Instruct-v0.3")],
    *[Case("Qwen/Qwen3-30B-A3B", NoQuantConfig, {
        "model.layers.0.mlp.experts": UnquantizedMoEMethod, "model.layers.0.mlp.gate": BF16,
        "model.layers.0.self_attn.qkv_proj": BF16, "lm_head": BF16,
    }, strategy=strategy) for strategy in ("fused", "offload")],
    Case("RadixArk/Qwen3.8-Flash-Next-NVFP4", ModelOptConfig, {
        "model.layers.{lin}.mlp.experts": Nvfp4MoEMethod, "model.layers.{lin}.linear_attn.in_proj": BF16,
        "model.layers.{lin}.linear_attn.out_proj": BF16, "model.layers.{full}.self_attn.qkv_proj": BF16,
        "model.layers.{full}.self_attn.o_proj": BF16, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": BF16,
        "model.layers.{lin}.mlp.shared_expert.down_proj": BF16, "model.layers.{lin}.mlp.gate": BF16, "lm_head": BF16,
        # the indexer, hyper-connection and PLE projections ask the config too
        "model.layers.{full}.self_attn.indexer.index_qk_proj": BF16,
        "model.layers.{lin}.attn_hyper_connection.input_mix_weight_down_block_inject": BF16,
        "model.layers.{lin}.mlp_hyper_connection.input_mix_weight_up": BF16,
        "model.hyper_connection_mixer.input_mix_weight_down": BF16,
        "model.layers.{ple}.ple.key_proj": BF16, "model.layers.{ple}.ple.value_proj": BF16,
    }, check=_no_split),
    Case("deepseek-ai/DeepSeek-V4-Flash-0731", Fp8BlockConfig, {
        **{f"model.layers.2.{p}": (FP8B, "dsv4") for p in (
            "attn.wq_a", "attn.wq_b", "attn.wkv", "attn.wo_b", "attn.indexer.wq_b",
            "ffn.shared_experts.w1", "ffn.shared_experts.w2", "ffn.shared_experts.w3")},
        # the KV compressors and the indexer's scorer ship bf16; the fp8 config has no modules_to_not_convert
        **{f"model.layers.2.{p}": (BF16, "torch") for p in (
            "attn.compressor.wkv", "attn.compressor.wgate", "attn.indexer.weights_proj", "attn.indexer.compressor.wkv")},
        "model.head": (BF16, "torch"), "model.layers.2.ffn.experts": (Mxfp4MoEMethod, "triton"),
    }, check=_dsv4),
    Case("nvidia/MiniMax-M2.5-NVFP4", ModelOptConfig, {
        "model.layers.3.block_sparse_moe.experts": Nvfp4MoEMethod, "model.layers.3.self_attn.qkv_proj": BF16,
        "model.layers.3.self_attn.o_proj": BF16, "model.layers.3.block_sparse_moe.gate": BF16, "lm_head": BF16,
    }),
    Case("nvidia/MiniMax-M3-NVFP4", ModelOptConfig, {
        "model.layers.{moe}.block_sparse_moe.experts": Nvfp4MoEMethod,
        "model.layers.{moe}.block_sparse_moe.shared_experts.gate_up_proj": MXFP8,
        "model.layers.{moe}.block_sparse_moe.shared_experts.down_proj": MXFP8, "model.layers.{dense}.mlp.gate_up_proj": MXFP8,
        "model.layers.{moe}.self_attn.qkv_proj": MXFP8, "model.layers.{moe}.self_attn.o_proj": MXFP8, "lm_head": BF16,
    }, env={"FREETOKEN_M3_MAX_LAYERS": "8"}, check=_m3_sparse),
    Case("nvidia/Gemma-4-26B-A4B-NVFP4", ModelOptConfig, {
        "model.layers.0.feed_forward.experts": Nvfp4MoEMethod, "model.layers.0.feed_forward.shared_mlp.gate_up_proj": BF16,
        "model.layers.0.feed_forward.shared_mlp.down_proj": BF16, "model.layers.0.feed_forward.router.proj": BF16,
        "model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,
    }, check=_gemma_tied_head),
    Case("google/gemma-4-26B-A4B-it", NoQuantConfig, {
        "model.layers.0.feed_forward.experts": UnquantizedMoEMethod, "model.layers.0.feed_forward.shared_mlp.gate_up_proj": BF16,
    }),
    Case("RedHatAI/GLM-5.3-Flash-NVFP4", CompressedTensorsConfig, {
        "model.layers.{moe}.mlp.experts": Nvfp4MoEMethod, "model.layers.{moe}.mlp.shared_experts.gate_proj": BF16,
        "model.layers.{moe}.mlp.shared_experts.down_proj": BF16, "model.layers.{dense}.mlp.gate_proj": BF16,
        "model.layers.{moe}.self_attn.o_proj": BF16, "lm_head": BF16,
        # the indexer, the MLA kv_b_proj and the KDA projections ask the config too
        "model.layers.{dsa}.self_attn.indexer.wq_b": BF16, "model.layers.{dsa}.self_attn.kv_b_proj": BF16,
        "model.layers.{kda}.self_attn.in_proj": BF16, "model.layers.{kda}.self_attn.f_b_proj": BF16,
        "model.layers.{kda}.self_attn.g_b_proj": BF16,
    }),
    Case("zai-org/GLM-5.3-Flash", Fp8BlockConfig, {
        "model.layers.{moe}.mlp.experts": (Fp8BlockMoEMethod, "triton"), "model.layers.{moe}.mlp.shared_experts.gate_proj": FP8B,
        "model.layers.{moe}.mlp.shared_experts.down_proj": FP8B, "model.layers.{dense}.mlp.gate_proj": FP8B,
        "model.layers.{dsa}.self_attn.q_b_proj": FP8B, "model.layers.{dsa}.self_attn.kv_b_proj": BF16,
        "model.layers.{dsa}.self_attn.o_proj": FP8B, "model.layers.{kda}.self_attn.in_proj": BF16,
        "model.layers.{kda}.self_attn.o_proj": BF16, "lm_head": BF16,
    }),
    Case("nvidia/GLM-4.7-NVFP4", ModelOptConfig, {
        "model.layers.{moe}.mlp.experts": Nvfp4MoEMethod, "model.layers.{moe}.mlp.shared_experts.gate_proj": NVFP4,
        "model.layers.{moe}.mlp.shared_experts.down_proj": NVFP4, "model.layers.{moe}.mlp.gate": BF16, "lm_head": BF16,
    }),
    Case("nvidia/GLM-5.3-NVFP4", ModelOptConfig, {
        "model.layers.{moe}.mlp.experts": Nvfp4MoEMethod, "model.layers.{moe}.mlp.gate": BF16,
        "model.layers.{moe}.mlp.shared_experts.gate_proj": BF16, "model.layers.{moe}.self_attn.o_proj": BF16, "lm_head": BF16,
        "model.layers.{dsa}.self_attn.indexer.wq_b": BF16, "model.layers.{moe}.self_attn.kv_b_proj": BF16,
    }),
    *[Case("openai/gpt-oss-20b", Mxfp4Config, {
        "model.layers.0.mlp.experts": Mxfp4MoEMethod, "model.layers.0.self_attn.qkv_proj": BF16,
        "model.layers.0.self_attn.o_proj": BF16, "model.layers.0.mlp.router": BF16, "lm_head": BF16,
    }, strategy=strategy, check=_gpt_oss(cls)) for strategy, cls in (
        ("fused", GptOssMoELayer), ("offload", GptOssOffloadMoELayer), ("cpu", GptOssOffloadMoELayer), ("hybrid", GptOssOffloadMoELayer))],
    # hybrid decodes on the CPU executor too, so the expert kernel must have a CPU format
    Case("Qwen3.6-35B-A3B-NVFP4", ModelOptConfig, {"model.layers.{lin}.mlp.experts": (Nvfp4MoEMethod, "triton")}, strategy="hybrid"),
    Case("Qwen3.6-35B-A3B", NoQuantConfig, {"model.layers.{lin}.mlp.experts": (UnquantizedMoEMethod, "fused")}, strategy="hybrid"),
    Case("Qwen3.6-35B-A3B-FP8", Fp8BlockConfig, {"model.layers.{lin}.mlp.experts": (Fp8BlockMoEMethod, "triton")}, strategy="hybrid"),
]


@pytest.mark.parametrize("case", CASES, ids=[f"{c.name.split('/')[-1]}:{c.strategy}" for c in CASES])
def test_probed_layers_get_the_method_their_config_says(case: Case, monkeypatch):
    path = model_dir(case.name)
    if path is None:
        pytest.skip(f"{case.name} not present locally")
    if case.skip_if is not None and case.skip_if():
        pytest.skip("kernel availability differs on this host")
    for key, value in case.env.items():
        monkeypatch.setenv(key, value)
    if case.raises is not None:
        with pytest.raises(case.raises):
            build_meta_model(path, case.strategy)
        return
    model, model_config = build_meta_model(path, case.strategy)
    assert type(model_config.quant) is case.dialect
    idx = _layer_indices(model)
    for template, expected in case.methods.items():
        probe = template.format(**{k: v for k, v in idx.items() if v is not None})
        op = op_at(model, probe)
        if getattr(op, "prefix", ""):  # routers are built without one
            assert op.prefix == probe
        if expected is None:
            assert op.quant_method is None, probe
            continue
        cls, kernel = expected if isinstance(expected, tuple) else (expected, None)
        assert type(op.quant_method) is cls, f"{probe}: {type(op.quant_method).__name__}"
        if kernel is not None:
            assert op.quant_method.kernel.name == kernel, probe
    if case.check is not None:
        case.check(model, idx)


# --------------------------------------------------------------------------- config without local weights


def test_the_modelopt_sidecar_is_folded_into_the_hf_config(tmp_path):
    """An old ModelOpt export keeps its quantization config only in hf_quant_config.json; the config loader folds it in, so every reader of the config sees it."""
    from freetoken.utils import cached_load_hf_config

    (tmp_path / "config.json").write_text(json.dumps({"model_type": "qwen3_moe", "architectures": ["Qwen3MoeForCausalLM"]}))
    (tmp_path / "hf_quant_config.json").write_text(json.dumps({"producer": {"name": "modelopt", "version": "0.29.0"}, "quantization": {"quant_algo": "FP8", "exclude_modules": ["lm_head"]}}))
    hf = cached_load_hf_config(str(tmp_path))
    assert hf.quantization_config == {"quant_method": "modelopt", "quant_algo": "FP8", "exclude_modules": ["lm_head"]}
    quant = checkpoint_quant_config(str(tmp_path), hf, get_model_spec("Qwen3MoeForCausalLM"))
    assert type(quant) is ModelOptConfig
    assert quant.scheme_for("model.layers.0.self_attn.q_proj").kind is QuantKind.FP8_TENSOR
    assert quant.scheme_for("lm_head") is None


def test_compressed_tensors_ignore_names_the_module_alone():
    """llm-compressor lists every skipped module, containers included: an ignored ``linear_attn`` must not shield the quantized projections under it (unsloth/Qwen3.6-35B-A3B-NVFP4-Fast)."""
    q = {
        "quant_method": "compressed-tensors", "format": "mixed-precision",
        "config_groups": {"group_0": {
            "targets": [r"re:.*linear_attn\.(in_proj_qkv|in_proj_z|out_proj)$"],
            "weights": {"num_bits": 8, "type": "float", "strategy": "channel"}, "input_activations": {"dynamic": True},
        }},
        "ignore": ["model.language_model.layers.0.linear_attn", "model.language_model.layers.0.linear_attn.in_proj_b"],
    }
    quant = QuantConfig.from_hf(SimpleNamespace(quantization_config=q))
    assert quant.scheme_for("model.language_model.layers.0.linear_attn.in_proj_qkv").kind is QuantKind.FP8_TENSOR
    assert quant.scheme_for("model.language_model.layers.0.linear_attn.in_proj_b") is None
    assert quant.scheme_for("model.language_model.layers.0.linear_attn") is None


_NVFP4_GROUP = {"targets": ["Linear"], "weights": {"num_bits": 4, "type": "float", "group_size": 16}}


@pytest.mark.parametrize("extra, has_input_scale", [
    ({"config_groups": {"group_0": {**_NVFP4_GROUP, "input_activations": None}}}, False),
    ({"config_groups": {"group_0": {**_NVFP4_GROUP, "input_activations": {"num_bits": 4, "type": "float", "group_size": 16}}}}, True),
    ({}, True),
    ({"with_input_scale": False}, False),
    ({"config_groups": {"group_0": {**_NVFP4_GROUP, "input_activations": None}}, "with_input_scale": True}, True),
], ids=["no input_activations", "input_activations", "no config_groups", "with_input_scale false", "with_input_scale true wins"])
def test_modelopt_nvfp4_reads_the_activation_quantizer_from_config_groups(extra, has_input_scale):
    """A ModelOpt export can say ``NVFP4`` with no activation quantizer: every config group then has ``input_activations`` null (vLLM #54427); an explicit ``with_input_scale`` wins."""
    q = {"quant_method": "modelopt", "quant_algo": "NVFP4", **extra}
    scheme = QuantConfig.from_hf(SimpleNamespace(quantization_config=q)).scheme_for("model.layers.0.mlp.down_proj")
    assert scheme.kind is QuantKind.NVFP4 and scheme.has("input_scale") is has_input_scale


def test_fp8_dialect_reads_the_block_size_and_the_nested_expert_dtype():
    """DeepSeek-V4.1 keeps the ``fp8`` dialect with 32x32 blocks and puts ``expert_dtype`` inside quantization_config."""
    from freetoken.layers.quantization.scheme import fp8_block_size

    q = {"quant_method": "fp8", "activation_scheme": "dynamic", "weight_block_size": [32, 32], "scale_fmt": "ue8m0", "expert_dtype": "fp4"}
    quant = QuantConfig.from_hf(SimpleNamespace(quantization_config=q))
    assert type(quant) is Fp8BlockConfig and quant.block == 32
    dense = quant.scheme_for("layers.3.attn.wq_a")
    assert dense.kind is QuantKind.FP8_BLOCK and fp8_block_size(dense) == 32 and dense.weight.scale == "e8m0"
    assert quant.storage(dense)["weight_scale_inv"].name == "scale"
    assert quant.scheme_for("layers.3.ffn.experts.7.w1").kind is QuantKind.MXFP4
    # the V4 128-block export still resolves to the 128 scheme
    q128 = {"quant_method": "fp8", "activation_scheme": "dynamic", "weight_block_size": [128, 128], "scale_fmt": "ue8m0"}
    assert fp8_block_size(QuantConfig.from_hf(SimpleNamespace(quantization_config=q128)).scheme_for("layers.3.attn.wq_a")) == 128
    for bad in ([64, 64], [32, 128], [128]):
        with pytest.raises(NotImplementedError):
            QuantConfig.from_hf(SimpleNamespace(quantization_config={**q, "weight_block_size": bad}))


def test_every_dialect_names_the_tensors_behind_its_schemes():
    from freetoken.layers.quantization.registry import dialects

    for cls in dialects():
        for scheme in getattr(cls, "SCHEMES", {}).values():
            assert set(cls.STORAGE[scheme.kind]) >= scheme.roles, f"{cls.__name__} does not name every tensor of {scheme}"


def test_compressed_tensors_target_classes_other_than_linear_fail_closed():
    q = {"quant_method": "compressed-tensors", "config_groups": {"group_0": {
        "targets": ["Embedding"], "weights": {"num_bits": 8, "type": "float", "strategy": "channel"}, "input_activations": {"dynamic": True}}}}
    with pytest.raises(NotImplementedError, match="Embedding"):
        QuantConfig.from_hf(SimpleNamespace(quantization_config=q))


def test_unsupported_dialects_fail_closed(tmp_path):
    hf_config = SimpleNamespace(architectures=["Qwen3MoeForCausalLM"], quantization_config={"quant_method": "gptq", "bits": 4})
    with pytest.raises(NotImplementedError, match="gptq"):
        checkpoint_quant_config(str(tmp_path), hf_config, get_model_spec("Qwen3MoeForCausalLM"))


# --------------------------------------------------------------------------- config against the stored tensors

# leaf names FreeToken builds as Linear / MoE layers; routers, norms, convs never ask for a scheme
_LINEAR_LEAVES = {
    "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "w1", "w2", "w3",
    "in_proj_qkv", "in_proj_z", "in_proj_a", "in_proj_b", "in_proj_ba", "out_proj", "qkv_proj", "gate_up_proj",
    "lm_head", "q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "wq_a", "wq_b", "wkv_a", "wkv_b", "wo",
    "wkv", "wo_a", "wo_b", "wgate", "weights_proj", "experts",
    "b_proj", "f_a_proj", "f_b_proj", "g_a_proj", "g_b_proj", "index_qk_proj", "key_proj", "value_proj",
    "input_mix_weight_down", "input_mix_weight_up", "block_inject_weight",
}
_META_SUFFIXES = {"bias", "k_scale", "v_scale"}
_QUANT_SUFFIXES = {
    "weight_scale", "weight_scale_2", "weight_scale_inv", "weight_packed", "weight_global_scale",
    "input_scale", "input_scale_2", "input_global_scale", "scale",
    "gate_up_proj_blocks", "gate_up_proj_scales", "gate_up_proj_bias", "down_proj_blocks", "down_proj_scales", "down_proj_bias",
}
# unsupported dialects: from_hf must refuse them, nothing else is checked
_UNSUPPORTED = {"quark", "mxfp8", "fp_quant"}
# tables the model code reads itself; RadixArk lists the fp8 PLE table in ``ignore`` yet ships it quantized
_MODEL_READS = ("ple.ple_embedding",)
# non-Linear leaves whose parameters happen to be called scale
_NOT_LINEAR = {"router", "gate", "shared_expert_gate"}


def _weight_map(p: Path) -> dict[str, str] | None:
    """Tensor name -> shard file, from the index or from a single model.safetensors."""
    if (p / "model.safetensors.index.json").exists():
        return json.load(open(p / "model.safetensors.index.json"))["weight_map"]
    if (p / "model.safetensors").exists():
        return {name: "model.safetensors" for name in _safetensors_header(p / "model.safetensors") if name != "__metadata__"}
    return None


def _candidate_dirs() -> list[Path]:
    roots = glob.glob(os.path.join(HF_CACHE, "models--*/snapshots/*/")) + glob.glob(os.path.join(SCAN_ROOT, "*/"))
    return [Path(d) for d in roots if (Path(d) / "config.json").exists() and _weight_map(Path(d)) is not None]


def _label(p: Path) -> str:
    m = re.search(r"models--([^/]+)/snapshots", str(p))
    name = m.group(1).replace("--", "/") if m else f"nvme:{p.name}"
    # an HF cache entry may hold the config and index without the shards; the tensor check then only has the names
    shards = set(_weight_map(p).values())
    return name if all((p / s).exists() for s in shards) else f"{name}[index-only]"


def _safetensors_header(path: Path) -> dict:
    with open(path, "rb") as f:
        n = struct.unpack("<Q", f.read(8))[0]
        return json.loads(f.read(n))


def _tensor_info(p: Path, weight_map: dict[str, str]) -> dict[str, dict]:
    info: dict[str, dict] = {}
    for shard in sorted(set(weight_map.values())):
        if not (p / shard).exists():
            return {}
        for name, meta in _safetensors_header(p / shard).items():
            if name != "__metadata__":
                info[name] = meta
    return info


def _expected_kinds(suffixes: set[str], tensors: dict[str, dict], module: str) -> set[QuantKind]:
    s = suffixes - _META_SUFFIXES
    if not s & {"weight", "weight_packed", "gate_up_proj_blocks"} or not s & _QUANT_SUFFIXES:
        return {QuantKind.NONE}
    if "weight_packed" in s or "weight_scale_2" in s:
        return {QuantKind.NVFP4}
    if "gate_up_proj_blocks" in s:
        return {QuantKind.MXFP4}

    def dtype(suffix):
        return (tensors.get(f"{module}.{suffix}") or {}).get("dtype")

    def shape(suffix):
        return (tensors.get(f"{module}.{suffix}") or {}).get("shape")

    if "weight_scale_inv" in s:
        if tensors:
            return {QuantKind.MXFP8} if dtype("weight_scale_inv") == "U8" else {QuantKind.FP8_BLOCK}
        return {QuantKind.FP8_BLOCK, QuantKind.MXFP8}
    if "scale" in s:
        if tensors:
            return {QuantKind.FP8_BLOCK} if dtype("weight") == "F8_E4M3" else {QuantKind.MXFP4}
        return {QuantKind.FP8_BLOCK, QuantKind.MXFP4}
    if "weight_scale" in s:
        if tensors:
            if dtype("weight") == "U8":
                return {QuantKind.MXFP4}
            if dtype("weight_scale") == "U8":
                return {QuantKind.MXFP8}
            sh = shape("weight_scale") or []
            return {QuantKind.FP8_BLOCK} if len(sh) == 2 and sh[0] > 1 and sh[1] > 1 else {QuantKind.FP8_TENSOR}
        return {QuantKind.FP8_TENSOR, QuantKind.FP8_BLOCK, QuantKind.MXFP4, QuantKind.MXFP8}
    return {QuantKind.NONE}


def _probes(weight_map: dict[str, str]):
    mods: dict[str, set[str]] = defaultdict(set)
    for name in weight_map:
        module, _, suffix = name.rpartition(".")
        mods[module].add(suffix)
    for module, suffixes in mods.items():
        leaf = module.rpartition(".")[2]
        if leaf in _NOT_LINEAR or any(m in module for m in _MODEL_READS) or suffixes <= _META_SUFFIXES:
            continue
        if leaf in _LINEAR_LEAVES or suffixes & _QUANT_SUFFIXES:
            yield module, suffixes


DIRS = _candidate_dirs()


@pytest.mark.skipif(not DIRS, reason="no local checkpoint with config.json + safetensors")
@pytest.mark.parametrize("ckpt", DIRS, ids=_label)
def test_scheme_for_agrees_with_the_stored_tensors(ckpt: Path):
    cfg = json.load(open(ckpt / "config.json"))
    quant = cfg.get("quantization_config") or (cfg.get("text_config") or {}).get("quantization_config") or {}
    if str(quant.get("quant_method") or "").lower() in _UNSUPPORTED:
        with pytest.raises(NotImplementedError):
            QuantConfig.from_hf(cfg)
        return
    if not quant:
        cfg["quantization_config"] = sidecar_quantization_config(str(ckpt))
    try:
        spec = get_model_spec((cfg.get("architectures") or [""])[0])
    except Exception:
        spec = None
    qc = QuantConfig.from_hf(cfg, unquantized=spec.unquantized_modules if spec else ())
    weight_map = _weight_map(ckpt)
    tensors = _tensor_info(ckpt, weight_map)
    mismatches, checked = [], 0
    for module, suffixes in _probes(weight_map):
        expected = _expected_kinds(suffixes, tensors, module)
        scheme = qc.scheme_for(module)
        kind = scheme.kind if scheme else QuantKind.NONE
        checked += 1
        if kind not in expected:
            mismatches.append(f"{module}: config says {kind}, tensors {sorted(suffixes)} say {sorted(k.value for k in expected)}")
    assert checked > 0
    assert not mismatches, f"{len(mismatches)}/{checked} modules disagree:\n  " + "\n  ".join(mismatches[:40])