FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""What each checkpoint's quant config resolves to: the dialect, the method behind every layer, and agreement with the stored tensors.
Rows need the checkpoint's config.json locally (/mnt/nvme/models or the HF cache); absent ones skip. The
stored-tensor scan also reads $FREETOKEN_TEST_MODELS_ROOT.
"""
from __future__ import annotations
import glob
import json
import os
import re
import struct
from collections import defaultdict
from dataclasses import dataclass, field
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Callable
import pytest
import torch
from freetoken.distributed.info import set_tp_info, try_get_tp_info
from freetoken.engine.config import EngineConfig
from freetoken.layers import set_rope_device
from freetoken.layers.quantization import (
CompressedTensorsConfig,
Fp8BlockConfig,
ModelOptConfig,
Mxfp4Config,
NoQuantConfig,
QuantConfig,
QuantKind,
)
from freetoken.layers.quantization.linear import (
Fp8BlockLinearMethod,
Fp8TensorLinearMethod,
Mxfp8LinearMethod,
Nvfp4LinearMethod,
UnquantizedLinearMethod,
)
from freetoken.layers.quantization.moe import Fp8BlockMoEMethod, Mxfp4MoEMethod, Nvfp4MoEMethod, UnquantizedMoEMethod
from freetoken.models import create_model
from freetoken.models.gpt_oss.moe import GptOssMoELayer, GptOssOffloadMoELayer
from freetoken.models.register import checkpoint_quant_config, get_model_spec
from freetoken.utils.hf import sidecar_quantization_config
from freetoken.utils.torch_utils import torch_dtype
MODELS = "/mnt/nvme/models"
# the stored-tensor scan takes every checkpoint under this root (the case table above matches by name)
SCAN_ROOT = os.environ.get("FREETOKEN_TEST_MODELS_ROOT", MODELS)
HF_CACHE = os.path.expanduser("~/.cache/huggingface/hub")
BF16, FP8B, FP8T, MXFP8, NVFP4 = UnquantizedLinearMethod, Fp8BlockLinearMethod, Fp8TensorLinearMethod, Mxfp8LinearMethod, Nvfp4LinearMethod
def model_dir(name: str) -> str | None:
path = os.path.join(MODELS, name.split("/")[-1])
if os.path.isfile(os.path.join(path, "config.json")): # a download in progress has the dir but no config yet
return path
snaps = glob.glob(os.path.join(HF_CACHE, "models--" + name.replace("/", "--"), "snapshots", "*", "config.json"))
return os.path.dirname(snaps[0]) if snaps else None
def op_at(model, path: str):
op = model
for part in path.split("."):
op = op.op_list[int(part)] if part.isdigit() else getattr(op, part)
return op
def build_meta_model(path: str, strategy: str):
if try_get_tp_info() is None:
set_tp_info(0, 1)
set_rope_device(torch.device("cpu"))
config = EngineConfig(model_path=path, tp_info=try_get_tp_info(), dtype=torch.bfloat16, moe_strategy=strategy)
from freetoken.engine.engine import _decode_target
object.__setattr__(config.model_config, "moe_strategy", strategy)
object.__setattr__(config.model_config, "decode_target", _decode_target(config))
with torch.device("meta"), torch_dtype(torch.bfloat16):
return create_model(config.model_config), config.model_config
def _layer_indices(model) -> dict[str, int | None]:
"""First layer of each kind, for the {moe} {dense} {lin} {full} {dsa} {kda} {ple} placeholders in probe paths."""
layers = model.model.layers.op_list
def first(pred):
return next((i for i, layer in enumerate(layers) if pred(layer)), None)
def has_experts(layer):
if getattr(layer, "is_moe_layer", None) is not None:
return layer.is_moe_layer
block = next((getattr(layer, n) for n in ("mlp", "feed_forward", "block_sparse_moe", "ffn") if hasattr(layer, n)), None)
return block is not None and hasattr(block, "experts")
def is_linear_attn(layer):
return getattr(layer, "_is_linear", False) or hasattr(layer, "linear_attn")
attn = lambda layer: getattr(layer, "self_attn", None)
ple = getattr(getattr(model, "_config", None), "qwen4_args", None)
return {
"moe": first(has_experts),
"dense": first(lambda l: not has_experts(l)),
"lin": first(is_linear_attn),
"full": first(lambda l: not is_linear_attn(l)),
"dsa": first(lambda l: getattr(attn(l), "indexer", None) is not None),
"kda": first(lambda l: hasattr(attn(l), "in_proj")),
"ple": ple.ple_layer_ids[0] if ple is not None else None,
}
@dataclass(frozen=True)
class Case:
name: str
dialect: type
methods: dict[str, Any] # probe path -> method class, (class, kernel name), or None for a layer without a method
strategy: str = "offload"
raises: type | None = None
skip_if: Callable[[], bool] | None = None
env: dict[str, str] = field(default_factory=dict)
check: Callable[[Any, dict], None] | None = None # family extras beyond the method table
def _qwen35_fp8(model, idx):
assert model.model.layers.op_list[idx["lin"]].linear_attn._split_in_proj
def _qwen35_nvfp4(model, idx):
assert model.model.layers.op_list[idx["lin"]].linear_attn._split_in_proj
assert op_at(model, f"model.layers.{idx['lin']}.linear_attn.in_proj_qkvz").quant_method.scheme.has("input_scale")
assert model.lm_head.weight.dtype is torch.uint8
def _no_split(model, idx):
assert not model.model.layers.op_list[idx["lin"]].linear_attn._split_in_proj
def _dsv4(model, idx):
block = op_at(model, "model.layers.2")
for proj in (block.attn.wq_a, block.ffn.shared_experts.w1):
assert proj.weight.dtype == torch.float8_e4m3fn and proj.weight_scale_inv.dtype == torch.float8_e8m0fnu
assert proj.weight_scale_inv.shape == (proj.out_features // 128, proj.in_features // 128) and proj.bias is None
assert set(block.attn.compressor.wkv.state_dict()) == {"weight"}
experts = block.ffn.experts
assert experts.limit == 10.0 and experts.quant_method.cpu_format == "ds_fp4" and not experts.has_bias
H, I = 4096, 2048
assert {r: (s.shape, s.dtype) for r, s in experts.quant_method.layout().items()} == {
"gate_up": ((2 * I, H // 2), torch.uint8), "gate_up_scale": ((2 * I, H // 32), torch.float8_e8m0fnu),
"down": ((H, I // 2), torch.uint8), "down_scale": ((H, I // 32), torch.float8_e8m0fnu),
}
keys = set(model.state_dict())
assert "model.layers.2.attn.wq_a.weight_scale_inv" in keys and "model.layers.2.attn.wq_a.scale" not in keys
assert "model.layers.2.attn.compressor.wkv.weight_scale_inv" not in keys
assert "model.head.weight" in keys and not any(".ffn.experts." in k for k in keys)
def _banked_experts(model, idx):
experts = op_at(model, f"model.layers.{idx['lin']}.mlp.experts")
assert not experts.state_dict()
def _gpt_oss(cls):
def check(model, idx):
experts = op_at(model, "model.layers.0.mlp.experts")
assert type(experts) is cls
assert experts.interleaved and experts.alpha == pytest.approx(1.702) and experts.limit == 7.0
if cls is GptOssMoELayer:
assert not experts.state_dict()
return check
def _m3_sparse(model, idx):
layers = model.model.layers.op_list
sparse = next((i for i, l in enumerate(layers) if getattr(l.self_attn, "is_sparse", False)), None)
if sparse is not None:
assert type(op_at(model, f"model.layers.{sparse}.self_attn.index_qk_proj").quant_method) is MXFP8
def _gemma_tied_head(model, idx):
assert model.lm_head.quant_method is None
CASES = [
Case("Qwen3.6-35B-A3B-FP8", Fp8BlockConfig, {
"model.layers.{lin}.linear_attn.in_proj_qkvz": FP8B, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
"model.layers.{lin}.linear_attn.out_proj": FP8B, "model.layers.{full}.self_attn.qkv_proj": FP8B,
"model.layers.{full}.self_attn.o_proj": FP8B, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": FP8B,
"model.layers.{lin}.mlp.shared_expert.down_proj": FP8B, "model.layers.{lin}.mlp.experts": Fp8BlockMoEMethod,
"model.layers.{lin}.mlp.gate": BF16, "model.layers.{lin}.mlp.shared_expert_gate": BF16, "lm_head": BF16,
}, check=_qwen35_fp8),
Case("Qwen3.6-35B-A3B-NVFP4", ModelOptConfig, {
"model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
"model.layers.{full}.self_attn.qkv_proj": FP8T, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": NVFP4,
"model.layers.{lin}.mlp.shared_expert.down_proj": NVFP4, "model.layers.{lin}.mlp.experts": Nvfp4MoEMethod,
"lm_head": NVFP4,
}, check=_qwen35_nvfp4),
# resident NVFP4 experts load through the expert-bank loader, not the state dict
Case("Qwen3.6-35B-A3B-NVFP4", ModelOptConfig, {"model.layers.{lin}.mlp.experts": (Nvfp4MoEMethod, "triton")}, strategy="fused", check=_banked_experts),
Case("Qwen3.8-27B-NVFP4", ModelOptConfig, {
"model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{full}.self_attn.o_proj": FP8T,
"model.layers.{lin}.mlp.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.down_proj": NVFP4, "lm_head": NVFP4,
}, strategy="auto"),
# llm-compressor exports of the dense 27B: every Linear NVFP4, GDN in_proj included (sakamakismile); channel-fp8 attention / GDN / lm_head over NVFP4 MLPs (unsloth)
Case("sakamakismile/Qwen3.6-27B-NVFP4", CompressedTensorsConfig, {
"model.layers.{lin}.linear_attn.in_proj_qkvz": NVFP4, "model.layers.{lin}.linear_attn.in_proj_ba": NVFP4,
"model.layers.{lin}.linear_attn.out_proj": NVFP4, "model.layers.{full}.self_attn.qkv_proj": NVFP4,
"model.layers.{lin}.mlp.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.down_proj": NVFP4, "lm_head": BF16,
}, strategy="auto", check=_qwen35_fp8),
Case("unsloth/Qwen3.8-27B-NVFP4", CompressedTensorsConfig, {
"model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
"model.layers.{lin}.linear_attn.out_proj": FP8T, "model.layers.{full}.self_attn.o_proj": FP8T,
"model.layers.{lin}.mlp.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.down_proj": NVFP4, "lm_head": FP8T,
}, strategy="auto", check=_qwen35_fp8),
# llm-compressor exports of the 35B MoE: NVFP4 everywhere but the GDN and the routers (RedHatAI); channel-fp8 attention / GDN / lm_head with NVFP4 experts and shared expert (unsloth)
Case("RedHatAI/Qwen3.6-35B-A3B-NVFP4", CompressedTensorsConfig, {
"model.layers.{lin}.linear_attn.in_proj": BF16, "model.layers.{lin}.linear_attn.out_proj": BF16,
"model.layers.{full}.self_attn.qkv_proj": NVFP4, "model.layers.{full}.self_attn.o_proj": NVFP4,
"model.layers.{lin}.mlp.shared_expert.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.experts": Nvfp4MoEMethod,
"model.layers.{lin}.mlp.gate": BF16, "lm_head": BF16,
}, check=_no_split),
# static per-tensor fp8 attention / GDN / shared expert with NVFP4 experts; the ignore list names every ``experts.N`` container
Case("primitive-ai/Ornith-1.5-35B-A3B-mixed-NVFP4-FP8", CompressedTensorsConfig, {
"model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
"model.layers.{full}.self_attn.qkv_proj": FP8T, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": FP8T,
"model.layers.{lin}.mlp.experts": Nvfp4MoEMethod, "model.layers.{lin}.mlp.gate": BF16, "lm_head": BF16,
}, check=_qwen35_fp8),
# block-fp8 attention / GDN / shared expert with NVFP4 experts, shipped as one model.safetensors without an index
Case("kyaky/Qwen3.6-35B-A3B-Uncensored-NVFP4", CompressedTensorsConfig, {
"model.layers.{lin}.linear_attn.in_proj_qkvz": FP8B, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
"model.layers.{full}.self_attn.qkv_proj": FP8B, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": FP8B,
"model.layers.{lin}.mlp.experts": Nvfp4MoEMethod, "model.layers.{lin}.mlp.gate": BF16, "lm_head": BF16,
}, check=_qwen35_fp8),
Case("unsloth/Qwen3.6-35B-A3B-NVFP4-Fast", CompressedTensorsConfig, {
"model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
"model.layers.{lin}.linear_attn.out_proj": FP8T, "model.layers.{full}.self_attn.qkv_proj": FP8T,
"model.layers.{lin}.mlp.shared_expert.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.experts": Nvfp4MoEMethod,
"model.layers.{lin}.mlp.gate": BF16, "lm_head": FP8T,
}, check=_qwen35_fp8),
Case("Qwen3.6-27B", NoQuantConfig, {
"model.layers.{lin}.linear_attn.in_proj": BF16, "model.layers.{lin}.mlp.gate_up_proj": BF16, "lm_head": BF16,
}, strategy="auto", check=_no_split),
Case("Qwen3.6-35B-A3B", NoQuantConfig, {
"model.layers.{lin}.mlp.experts": UnquantizedMoEMethod, "model.layers.{lin}.mlp.shared_expert.down_proj": BF16,
}),
Case("nvidia/Qwen3.6-27B-NVFP4", ModelOptConfig, {
"model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
"model.layers.{lin}.linear_attn.out_proj": FP8T, "model.layers.{full}.self_attn.qkv_proj": FP8T,
"model.layers.{lin}.mlp.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.down_proj": NVFP4, "lm_head": NVFP4,
}, strategy="auto"),
*[Case(name, Fp8BlockConfig, {
"model.layers.{lin}.linear_attn.in_proj_qkvz": FP8B, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
"model.layers.{lin}.linear_attn.out_proj": FP8B, "model.layers.{full}.self_attn.qkv_proj": FP8B,
"model.layers.{full}.self_attn.o_proj": FP8B, "model.layers.{lin}.mlp.gate_up_proj": FP8B,
"model.layers.{lin}.mlp.down_proj": FP8B, "lm_head": BF16,
}, strategy="auto") for name in ("Qwen/Qwen3.8-27B-FP8", "Qwen/Qwen3.6-27B-FP8")],
Case("RedHatAI/Muse-Glimmer-30B-NVFP4", CompressedTensorsConfig, {
"model.layers.0.self_attn.qkv_proj": NVFP4, "model.layers.0.self_attn.gate_proj": NVFP4,
"model.layers.0.self_attn.o_proj": NVFP4,
"model.layers.0.mlp.gate_up_proj": NVFP4, "model.layers.0.mlp.down_proj": NVFP4, "lm_head": BF16,
}, strategy="auto"),
Case("nvidia/Gemma-4-31B-IT-NVFP4", ModelOptConfig, {
"model.layers.0.feed_forward.shared_mlp.gate_up_proj": NVFP4, "model.layers.0.feed_forward.shared_mlp.down_proj": NVFP4,
"model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,
}, strategy="auto", check=_gemma_tied_head),
Case("google/gemma-4-12B-it", NoQuantConfig, {
"model.layers.0.feed_forward.shared_mlp.gate_up_proj": BF16, "model.layers.0.self_attn.qkv_proj": BF16,
}, strategy="auto", check=_gemma_tied_head),
Case("Qwen/Qwen3-4B", NoQuantConfig, {
"model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,
"model.layers.0.mlp.gate_up_proj": BF16, "model.layers.0.mlp.down_proj": BF16,
}, strategy="auto"),
*[Case(name, NoQuantConfig, {
"model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,
"model.layers.0.mlp.gate_up_proj": BF16, "model.layers.0.mlp.down_proj": BF16,
}, strategy="auto") for name in ("Qwen/Qwen2.5-1.5B-Instruct", "unsloth/Llama-3.2-1B-Instruct", "mistralai/Mistral-7B-Instruct-v0.3")],
*[Case("Qwen/Qwen3-30B-A3B", NoQuantConfig, {
"model.layers.0.mlp.experts": UnquantizedMoEMethod, "model.layers.0.mlp.gate": BF16,
"model.layers.0.self_attn.qkv_proj": BF16, "lm_head": BF16,
}, strategy=strategy) for strategy in ("fused", "offload")],
Case("RadixArk/Qwen3.8-Flash-Next-NVFP4", ModelOptConfig, {
"model.layers.{lin}.mlp.experts": Nvfp4MoEMethod, "model.layers.{lin}.linear_attn.in_proj": BF16,
"model.layers.{lin}.linear_attn.out_proj": BF16, "model.layers.{full}.self_attn.qkv_proj": BF16,
"model.layers.{full}.self_attn.o_proj": BF16, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": BF16,
"model.layers.{lin}.mlp.shared_expert.down_proj": BF16, "model.layers.{lin}.mlp.gate": BF16, "lm_head": BF16,
# the indexer, hyper-connection and PLE projections ask the config too
"model.layers.{full}.self_attn.indexer.index_qk_proj": BF16,
"model.layers.{lin}.attn_hyper_connection.input_mix_weight_down_block_inject": BF16,
"model.layers.{lin}.mlp_hyper_connection.input_mix_weight_up": BF16,
"model.hyper_connection_mixer.input_mix_weight_down": BF16,
"model.layers.{ple}.ple.key_proj": BF16, "model.layers.{ple}.ple.value_proj": BF16,
}, check=_no_split),
Case("deepseek-ai/DeepSeek-V4-Flash-0731", Fp8BlockConfig, {
**{f"model.layers.2.{p}": (FP8B, "dsv4") for p in (
"attn.wq_a", "attn.wq_b", "attn.wkv", "attn.wo_b", "attn.indexer.wq_b",
"ffn.shared_experts.w1", "ffn.shared_experts.w2", "ffn.shared_experts.w3")},
# the KV compressors and the indexer's scorer ship bf16; the fp8 config has no modules_to_not_convert
**{f"model.layers.2.{p}": (BF16, "torch") for p in (
"attn.compressor.wkv", "attn.compressor.wgate", "attn.indexer.weights_proj", "attn.indexer.compressor.wkv")},
"model.head": (BF16, "torch"), "model.layers.2.ffn.experts": (Mxfp4MoEMethod, "triton"),
}, check=_dsv4),
Case("nvidia/MiniMax-M2.5-NVFP4", ModelOptConfig, {
"model.layers.3.block_sparse_moe.experts": Nvfp4MoEMethod, "model.layers.3.self_attn.qkv_proj": BF16,
"model.layers.3.self_attn.o_proj": BF16, "model.layers.3.block_sparse_moe.gate": BF16, "lm_head": BF16,
}),
Case("nvidia/MiniMax-M3-NVFP4", ModelOptConfig, {
"model.layers.{moe}.block_sparse_moe.experts": Nvfp4MoEMethod,
"model.layers.{moe}.block_sparse_moe.shared_experts.gate_up_proj": MXFP8,
"model.layers.{moe}.block_sparse_moe.shared_experts.down_proj": MXFP8, "model.layers.{dense}.mlp.gate_up_proj": MXFP8,
"model.layers.{moe}.self_attn.qkv_proj": MXFP8, "model.layers.{moe}.self_attn.o_proj": MXFP8, "lm_head": BF16,
}, env={"FREETOKEN_M3_MAX_LAYERS": "8"}, check=_m3_sparse),
Case("nvidia/Gemma-4-26B-A4B-NVFP4", ModelOptConfig, {
"model.layers.0.feed_forward.experts": Nvfp4MoEMethod, "model.layers.0.feed_forward.shared_mlp.gate_up_proj": BF16,
"model.layers.0.feed_forward.shared_mlp.down_proj": BF16, "model.layers.0.feed_forward.router.proj": BF16,
"model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,
}, check=_gemma_tied_head),
Case("google/gemma-4-26B-A4B-it", NoQuantConfig, {
"model.layers.0.feed_forward.experts": UnquantizedMoEMethod, "model.layers.0.feed_forward.shared_mlp.gate_up_proj": BF16,
}),
Case("RedHatAI/GLM-5.3-Flash-NVFP4", CompressedTensorsConfig, {
"model.layers.{moe}.mlp.experts": Nvfp4MoEMethod, "model.layers.{moe}.mlp.shared_experts.gate_proj": BF16,
"model.layers.{moe}.mlp.shared_experts.down_proj": BF16, "model.layers.{dense}.mlp.gate_proj": BF16,
"model.layers.{moe}.self_attn.o_proj": BF16, "lm_head": BF16,
# the indexer, the MLA kv_b_proj and the KDA projections ask the config too
"model.layers.{dsa}.self_attn.indexer.wq_b": BF16, "model.layers.{dsa}.self_attn.kv_b_proj": BF16,
"model.layers.{kda}.self_attn.in_proj": BF16, "model.layers.{kda}.self_attn.f_b_proj": BF16,
"model.layers.{kda}.self_attn.g_b_proj": BF16,
}),
Case("zai-org/GLM-5.3-Flash", Fp8BlockConfig, {
"model.layers.{moe}.mlp.experts": (Fp8BlockMoEMethod, "triton"), "model.layers.{moe}.mlp.shared_experts.gate_proj": FP8B,
"model.layers.{moe}.mlp.shared_experts.down_proj": FP8B, "model.layers.{dense}.mlp.gate_proj": FP8B,
"model.layers.{dsa}.self_attn.q_b_proj": FP8B, "model.layers.{dsa}.self_attn.kv_b_proj": BF16,
"model.layers.{dsa}.self_attn.o_proj": FP8B, "model.layers.{kda}.self_attn.in_proj": BF16,
"model.layers.{kda}.self_attn.o_proj": BF16, "lm_head": BF16,
}),
Case("nvidia/GLM-4.7-NVFP4", ModelOptConfig, {
"model.layers.{moe}.mlp.experts": Nvfp4MoEMethod, "model.layers.{moe}.mlp.shared_experts.gate_proj": NVFP4,
"model.layers.{moe}.mlp.shared_experts.down_proj": NVFP4, "model.layers.{moe}.mlp.gate": BF16, "lm_head": BF16,
}),
Case("nvidia/GLM-5.3-NVFP4", ModelOptConfig, {
"model.layers.{moe}.mlp.experts": Nvfp4MoEMethod, "model.layers.{moe}.mlp.gate": BF16,
"model.layers.{moe}.mlp.shared_experts.gate_proj": BF16, "model.layers.{moe}.self_attn.o_proj": BF16, "lm_head": BF16,
"model.layers.{dsa}.self_attn.indexer.wq_b": BF16, "model.layers.{moe}.self_attn.kv_b_proj": BF16,
}),
*[Case("openai/gpt-oss-20b", Mxfp4Config, {
"model.layers.0.mlp.experts": Mxfp4MoEMethod, "model.layers.0.self_attn.qkv_proj": BF16,
"model.layers.0.self_attn.o_proj": BF16, "model.layers.0.mlp.router": BF16, "lm_head": BF16,
}, strategy=strategy, check=_gpt_oss(cls)) for strategy, cls in (
("fused", GptOssMoELayer), ("offload", GptOssOffloadMoELayer), ("cpu", GptOssOffloadMoELayer), ("hybrid", GptOssOffloadMoELayer))],
# hybrid decodes on the CPU executor too, so the expert kernel must have a CPU format
Case("Qwen3.6-35B-A3B-NVFP4", ModelOptConfig, {"model.layers.{lin}.mlp.experts": (Nvfp4MoEMethod, "triton")}, strategy="hybrid"),
Case("Qwen3.6-35B-A3B", NoQuantConfig, {"model.layers.{lin}.mlp.experts": (UnquantizedMoEMethod, "fused")}, strategy="hybrid"),
Case("Qwen3.6-35B-A3B-FP8", Fp8BlockConfig, {"model.layers.{lin}.mlp.experts": (Fp8BlockMoEMethod, "triton")}, strategy="hybrid"),
]
@pytest.mark.parametrize("case", CASES, ids=[f"{c.name.split('/')[-1]}:{c.strategy}" for c in CASES])
def test_probed_layers_get_the_method_their_config_says(case: Case, monkeypatch):
path = model_dir(case.name)
if path is None:
pytest.skip(f"{case.name} not present locally")
if case.skip_if is not None and case.skip_if():
pytest.skip("kernel availability differs on this host")
for key, value in case.env.items():
monkeypatch.setenv(key, value)
if case.raises is not None:
with pytest.raises(case.raises):
build_meta_model(path, case.strategy)
return
model, model_config = build_meta_model(path, case.strategy)
assert type(model_config.quant) is case.dialect
idx = _layer_indices(model)
for template, expected in case.methods.items():
probe = template.format(**{k: v for k, v in idx.items() if v is not None})
op = op_at(model, probe)
if getattr(op, "prefix", ""): # routers are built without one
assert op.prefix == probe
if expected is None:
assert op.quant_method is None, probe
continue
cls, kernel = expected if isinstance(expected, tuple) else (expected, None)
assert type(op.quant_method) is cls, f"{probe}: {type(op.quant_method).__name__}"
if kernel is not None:
assert op.quant_method.kernel.name == kernel, probe
if case.check is not None:
case.check(model, idx)
# --------------------------------------------------------------------------- config without local weights
def test_the_modelopt_sidecar_is_folded_into_the_hf_config(tmp_path):
"""An old ModelOpt export keeps its quantization config only in hf_quant_config.json; the config loader folds it in, so every reader of the config sees it."""
from freetoken.utils import cached_load_hf_config
(tmp_path / "config.json").write_text(json.dumps({"model_type": "qwen3_moe", "architectures": ["Qwen3MoeForCausalLM"]}))
(tmp_path / "hf_quant_config.json").write_text(json.dumps({"producer": {"name": "modelopt", "version": "0.29.0"}, "quantization": {"quant_algo": "FP8", "exclude_modules": ["lm_head"]}}))
hf = cached_load_hf_config(str(tmp_path))
assert hf.quantization_config == {"quant_method": "modelopt", "quant_algo": "FP8", "exclude_modules": ["lm_head"]}
quant = checkpoint_quant_config(str(tmp_path), hf, get_model_spec("Qwen3MoeForCausalLM"))
assert type(quant) is ModelOptConfig
assert quant.scheme_for("model.layers.0.self_attn.q_proj").kind is QuantKind.FP8_TENSOR
assert quant.scheme_for("lm_head") is None
def test_compressed_tensors_ignore_names_the_module_alone():
"""llm-compressor lists every skipped module, containers included: an ignored ``linear_attn`` must not shield the quantized projections under it (unsloth/Qwen3.6-35B-A3B-NVFP4-Fast)."""
q = {
"quant_method": "compressed-tensors", "format": "mixed-precision",
"config_groups": {"group_0": {
"targets": [r"re:.*linear_attn\.(in_proj_qkv|in_proj_z|out_proj)$"],
"weights": {"num_bits": 8, "type": "float", "strategy": "channel"}, "input_activations": {"dynamic": True},
}},
"ignore": ["model.language_model.layers.0.linear_attn", "model.language_model.layers.0.linear_attn.in_proj_b"],
}
quant = QuantConfig.from_hf(SimpleNamespace(quantization_config=q))
assert quant.scheme_for("model.language_model.layers.0.linear_attn.in_proj_qkv").kind is QuantKind.FP8_TENSOR
assert quant.scheme_for("model.language_model.layers.0.linear_attn.in_proj_b") is None
assert quant.scheme_for("model.language_model.layers.0.linear_attn") is None
_NVFP4_GROUP = {"targets": ["Linear"], "weights": {"num_bits": 4, "type": "float", "group_size": 16}}
@pytest.mark.parametrize("extra, has_input_scale", [
({"config_groups": {"group_0": {**_NVFP4_GROUP, "input_activations": None}}}, False),
({"config_groups": {"group_0": {**_NVFP4_GROUP, "input_activations": {"num_bits": 4, "type": "float", "group_size": 16}}}}, True),
({}, True),
({"with_input_scale": False}, False),
({"config_groups": {"group_0": {**_NVFP4_GROUP, "input_activations": None}}, "with_input_scale": True}, True),
], ids=["no input_activations", "input_activations", "no config_groups", "with_input_scale false", "with_input_scale true wins"])
def test_modelopt_nvfp4_reads_the_activation_quantizer_from_config_groups(extra, has_input_scale):
"""A ModelOpt export can say ``NVFP4`` with no activation quantizer: every config group then has ``input_activations`` null (vLLM #54427); an explicit ``with_input_scale`` wins."""
q = {"quant_method": "modelopt", "quant_algo": "NVFP4", **extra}
scheme = QuantConfig.from_hf(SimpleNamespace(quantization_config=q)).scheme_for("model.layers.0.mlp.down_proj")
assert scheme.kind is QuantKind.NVFP4 and scheme.has("input_scale") is has_input_scale
def test_fp8_dialect_reads_the_block_size_and_the_nested_expert_dtype():
"""DeepSeek-V4.1 keeps the ``fp8`` dialect with 32x32 blocks and puts ``expert_dtype`` inside quantization_config."""
from freetoken.layers.quantization.scheme import fp8_block_size
q = {"quant_method": "fp8", "activation_scheme": "dynamic", "weight_block_size": [32, 32], "scale_fmt": "ue8m0", "expert_dtype": "fp4"}
quant = QuantConfig.from_hf(SimpleNamespace(quantization_config=q))
assert type(quant) is Fp8BlockConfig and quant.block == 32
dense = quant.scheme_for("layers.3.attn.wq_a")
assert dense.kind is QuantKind.FP8_BLOCK and fp8_block_size(dense) == 32 and dense.weight.scale == "e8m0"
assert quant.storage(dense)["weight_scale_inv"].name == "scale"
assert quant.scheme_for("layers.3.ffn.experts.7.w1").kind is QuantKind.MXFP4
# the V4 128-block export still resolves to the 128 scheme
q128 = {"quant_method": "fp8", "activation_scheme": "dynamic", "weight_block_size": [128, 128], "scale_fmt": "ue8m0"}
assert fp8_block_size(QuantConfig.from_hf(SimpleNamespace(quantization_config=q128)).scheme_for("layers.3.attn.wq_a")) == 128
for bad in ([64, 64], [32, 128], [128]):
with pytest.raises(NotImplementedError):
QuantConfig.from_hf(SimpleNamespace(quantization_config={**q, "weight_block_size": bad}))
def test_every_dialect_names_the_tensors_behind_its_schemes():
from freetoken.layers.quantization.registry import dialects
for cls in dialects():
for scheme in getattr(cls, "SCHEMES", {}).values():
assert set(cls.STORAGE[scheme.kind]) >= scheme.roles, f"{cls.__name__} does not name every tensor of {scheme}"
def test_compressed_tensors_target_classes_other_than_linear_fail_closed():
q = {"quant_method": "compressed-tensors", "config_groups": {"group_0": {
"targets": ["Embedding"], "weights": {"num_bits": 8, "type": "float", "strategy": "channel"}, "input_activations": {"dynamic": True}}}}
with pytest.raises(NotImplementedError, match="Embedding"):
QuantConfig.from_hf(SimpleNamespace(quantization_config=q))
def test_unsupported_dialects_fail_closed(tmp_path):
hf_config = SimpleNamespace(architectures=["Qwen3MoeForCausalLM"], quantization_config={"quant_method": "gptq", "bits": 4})
with pytest.raises(NotImplementedError, match="gptq"):
checkpoint_quant_config(str(tmp_path), hf_config, get_model_spec("Qwen3MoeForCausalLM"))
# --------------------------------------------------------------------------- config against the stored tensors
# leaf names FreeToken builds as Linear / MoE layers; routers, norms, convs never ask for a scheme
_LINEAR_LEAVES = {
"q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "w1", "w2", "w3",
"in_proj_qkv", "in_proj_z", "in_proj_a", "in_proj_b", "in_proj_ba", "out_proj", "qkv_proj", "gate_up_proj",
"lm_head", "q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "wq_a", "wq_b", "wkv_a", "wkv_b", "wo",
"wkv", "wo_a", "wo_b", "wgate", "weights_proj", "experts",
"b_proj", "f_a_proj", "f_b_proj", "g_a_proj", "g_b_proj", "index_qk_proj", "key_proj", "value_proj",
"input_mix_weight_down", "input_mix_weight_up", "block_inject_weight",
}
_META_SUFFIXES = {"bias", "k_scale", "v_scale"}
_QUANT_SUFFIXES = {
"weight_scale", "weight_scale_2", "weight_scale_inv", "weight_packed", "weight_global_scale",
"input_scale", "input_scale_2", "input_global_scale", "scale",
"gate_up_proj_blocks", "gate_up_proj_scales", "gate_up_proj_bias", "down_proj_blocks", "down_proj_scales", "down_proj_bias",
}
# unsupported dialects: from_hf must refuse them, nothing else is checked
_UNSUPPORTED = {"quark", "mxfp8", "fp_quant"}
# tables the model code reads itself; RadixArk lists the fp8 PLE table in ``ignore`` yet ships it quantized
_MODEL_READS = ("ple.ple_embedding",)
# non-Linear leaves whose parameters happen to be called scale
_NOT_LINEAR = {"router", "gate", "shared_expert_gate"}
def _weight_map(p: Path) -> dict[str, str] | None:
"""Tensor name -> shard file, from the index or from a single model.safetensors."""
if (p / "model.safetensors.index.json").exists():
return json.load(open(p / "model.safetensors.index.json"))["weight_map"]
if (p / "model.safetensors").exists():
return {name: "model.safetensors" for name in _safetensors_header(p / "model.safetensors") if name != "__metadata__"}
return None
def _candidate_dirs() -> list[Path]:
roots = glob.glob(os.path.join(HF_CACHE, "models--*/snapshots/*/")) + glob.glob(os.path.join(SCAN_ROOT, "*/"))
return [Path(d) for d in roots if (Path(d) / "config.json").exists() and _weight_map(Path(d)) is not None]
def _label(p: Path) -> str:
m = re.search(r"models--([^/]+)/snapshots", str(p))
name = m.group(1).replace("--", "/") if m else f"nvme:{p.name}"
# an HF cache entry may hold the config and index without the shards; the tensor check then only has the names
shards = set(_weight_map(p).values())
return name if all((p / s).exists() for s in shards) else f"{name}[index-only]"
def _safetensors_header(path: Path) -> dict:
with open(path, "rb") as f:
n = struct.unpack("<Q", f.read(8))[0]
return json.loads(f.read(n))
def _tensor_info(p: Path, weight_map: dict[str, str]) -> dict[str, dict]:
info: dict[str, dict] = {}
for shard in sorted(set(weight_map.values())):
if not (p / shard).exists():
return {}
for name, meta in _safetensors_header(p / shard).items():
if name != "__metadata__":
info[name] = meta
return info
def _expected_kinds(suffixes: set[str], tensors: dict[str, dict], module: str) -> set[QuantKind]:
s = suffixes - _META_SUFFIXES
if not s & {"weight", "weight_packed", "gate_up_proj_blocks"} or not s & _QUANT_SUFFIXES:
return {QuantKind.NONE}
if "weight_packed" in s or "weight_scale_2" in s:
return {QuantKind.NVFP4}
if "gate_up_proj_blocks" in s:
return {QuantKind.MXFP4}
def dtype(suffix):
return (tensors.get(f"{module}.{suffix}") or {}).get("dtype")
def shape(suffix):
return (tensors.get(f"{module}.{suffix}") or {}).get("shape")
if "weight_scale_inv" in s:
if tensors:
return {QuantKind.MXFP8} if dtype("weight_scale_inv") == "U8" else {QuantKind.FP8_BLOCK}
return {QuantKind.FP8_BLOCK, QuantKind.MXFP8}
if "scale" in s:
if tensors:
return {QuantKind.FP8_BLOCK} if dtype("weight") == "F8_E4M3" else {QuantKind.MXFP4}
return {QuantKind.FP8_BLOCK, QuantKind.MXFP4}
if "weight_scale" in s:
if tensors:
if dtype("weight") == "U8":
return {QuantKind.MXFP4}
if dtype("weight_scale") == "U8":
return {QuantKind.MXFP8}
sh = shape("weight_scale") or []
return {QuantKind.FP8_BLOCK} if len(sh) == 2 and sh[0] > 1 and sh[1] > 1 else {QuantKind.FP8_TENSOR}
return {QuantKind.FP8_TENSOR, QuantKind.FP8_BLOCK, QuantKind.MXFP4, QuantKind.MXFP8}
return {QuantKind.NONE}
def _probes(weight_map: dict[str, str]):
mods: dict[str, set[str]] = defaultdict(set)
for name in weight_map:
module, _, suffix = name.rpartition(".")
mods[module].add(suffix)
for module, suffixes in mods.items():
leaf = module.rpartition(".")[2]
if leaf in _NOT_LINEAR or any(m in module for m in _MODEL_READS) or suffixes <= _META_SUFFIXES:
continue
if leaf in _LINEAR_LEAVES or suffixes & _QUANT_SUFFIXES:
yield module, suffixes
DIRS = _candidate_dirs()
@pytest.mark.skipif(not DIRS, reason="no local checkpoint with config.json + safetensors")
@pytest.mark.parametrize("ckpt", DIRS, ids=_label)
def test_scheme_for_agrees_with_the_stored_tensors(ckpt: Path):
cfg = json.load(open(ckpt / "config.json"))
quant = cfg.get("quantization_config") or (cfg.get("text_config") or {}).get("quantization_config") or {}
if str(quant.get("quant_method") or "").lower() in _UNSUPPORTED:
with pytest.raises(NotImplementedError):
QuantConfig.from_hf(cfg)
return
if not quant:
cfg["quantization_config"] = sidecar_quantization_config(str(ckpt))
try:
spec = get_model_spec((cfg.get("architectures") or [""])[0])
except Exception:
spec = None
qc = QuantConfig.from_hf(cfg, unquantized=spec.unquantized_modules if spec else ())
weight_map = _weight_map(ckpt)
tensors = _tensor_info(ckpt, weight_map)
mismatches, checked = [], 0
for module, suffixes in _probes(weight_map):
expected = _expected_kinds(suffixes, tensors, module)
scheme = qc.scheme_for(module)
kind = scheme.kind if scheme else QuantKind.NONE
checked += 1
if kind not in expected:
mismatches.append(f"{module}: config says {kind}, tensors {sorted(suffixes)} say {sorted(k.value for k in expected)}")
assert checked > 0
assert not mismatches, f"{len(mismatches)}/{checked} modules disagree:\n " + "\n ".join(mismatches[:40])