PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
import os
import shutil
from test.helpers import get_model_path, get_tiny_model, opt_name_or_path, qwen_name_or_path, save_tiny_model
import pytest
import torch
import torch.nn as nn
import transformers
from auto_round import AutoRound
from auto_round.schemes import QuantizationScheme, _handle_special_schemes
class TestAutoRound:
@pytest.fixture(autouse=True)
def setup_save_folder(self, tmp_path):
self.save_folder = str(tmp_path / "saved")
yield
shutil.rmtree(self.save_folder, ignore_errors=True)
@classmethod
def teardown_class(self):
shutil.rmtree("runs", ignore_errors=True)
def test_gguf(self, tiny_qwen_model_path, dataloader):
ar = AutoRound(
tiny_qwen_model_path,
scheme="W2A16",
nsamples=1,
iters=1,
seqlen=2,
dataset=dataloader,
)
ar.quantize_and_save(self.save_folder, format="gguf:q4_k_m")
assert ar.bits == 4
def test_w4a16(self, tiny_opt_model_path, dataloader):
ar = AutoRound(tiny_opt_model_path, scheme="W4A16", nsamples=1, iters=1, seqlen=2, dataset=dataloader)
ar.post_init()
assert ar.bits == 4
def test_w2a16_rtn(self, tiny_opt_model_path, dataloader):
ar = AutoRound(tiny_opt_model_path, scheme="W2A16", nsamples=1, iters=0, seqlen=2, dataset=dataloader)
ar.post_init()
assert ar.bits == 2
def test_w4a16_mixed(self, micro_qwen_moe_model_path, dataloader):
layer_config = {
"model.layers.0.self_attn.k_proj": {"bits": 16},
}
ar = AutoRound(
micro_qwen_moe_model_path,
scheme="W4A16_MIXED",
nsamples=1,
iters=0,
seqlen=2,
dataset=dataloader,
low_cpu_mem_usage=False,
layer_config=layer_config,
)
_, quantized_model_path = ar.quantize_and_save(self.save_folder)
assert ar.bits == 4
assert ar.model.model.layers[0].self_attn.q_proj.bits == 8
assert ar.model.model.layers[0].self_attn.k_proj.bits == 16
assert ar.model.model.layers[0].mlp.experts[0].up_proj.bits == 4
# assert ar.model.model.layers[0].mlp.shared_expert.gate_proj.bits == 8 # gate has been added to ignore_layers
model = transformers.AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True)
assert model is not None, "Model loading failed after quantization with W4A16_MIXED scheme on MoE"
def test_w4a16_mixed_mllm(self, tiny_qwen_2_5_vl_model_path, dataloader):
ar = AutoRound(
tiny_qwen_2_5_vl_model_path,
scheme="W4A16_MIXED",
nsamples=1,
batch_size=1,
iters=0,
seqlen=2,
disable_opt_rtn=True,
dataset=dataloader,
low_cpu_mem_usage=False,
disable_model_free=True,
)
_, quantized_model_path = ar.quantize_and_save(self.save_folder)
model = transformers.Qwen2_5_VLForConditionalGeneration.from_pretrained(quantized_model_path)
assert model is not None, "Model loading failed after quantization with W4A16_MIXED scheme on MLLM"
assert ar.bits == 4
assert ar.model.model.language_model.layers[0].self_attn.q_proj.bits == 16
assert ar.model.model.visual.blocks[0].attn.qkv.bits == 16
def test_mxfp4(self, tiny_opt_model_path, dataloader):
ar = AutoRound(tiny_opt_model_path, scheme="MXFP4", nsamples=1, iters=1, seqlen=2, dataset=dataloader)
ar.post_init()
assert ar.bits == 4
assert ar.act_bits == 4
assert ar.data_type == "mx_fp"
assert ar.act_data_type == "mx_fp"
def test_mxfp4_rceil(self, tiny_opt_model_path):
ar = AutoRound(tiny_opt_model_path, scheme="MXFP4_RCEIL", nsamples=1, iters=1)
ar.post_init()
assert ar.bits == 4
assert ar.act_bits == 4
assert ar.data_type == "mx_fp"
assert ar.act_data_type == "mx_fp_rceil"
_, quantized_model_path = ar.quantize_and_save()
model = transformers.AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True)
assert model is not None, "Model loading failed after quantization with MXFP4 scheme"
def test_vlm(self, tiny_qwen_vl_model_path):
from auto_round import AutoRound
ar = AutoRound(tiny_qwen_vl_model_path, scheme="W2A16", nsamples=1, iters=1, seqlen=2)
ar.post_init()
assert ar.bits == 2
assert ar.act_bits == 16
def test_nvfp4(self, tiny_opt_model_path, dataloader):
ar = AutoRound(tiny_opt_model_path, scheme="NVFP4", nsamples=1, iters=1, seqlen=2, dataset=dataloader)
ar.post_init()
assert ar.bits == 4
assert ar.act_bits == 4
assert ar.data_type == "nv_fp"
assert ar.act_data_type == "nv_fp4_with_static_gs"
_, quantized_model_path = ar.quantize_and_save(self.save_folder)
model = transformers.AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True)
assert model is not None, "Model loading failed after quantization with NVFP4 scheme"
@pytest.mark.parametrize(
"scheme", ["W8A16", "MXFP8", "FPW8A16", "FP8_BLOCK", "FP8_STATIC", "GGUF:Q2_K_S", "GGUF:Q4_K_M"]
)
def test_all_scheme(self, scheme, tiny_qwen_model_path, dataloader):
print(f"scheme={scheme}")
ar = AutoRound(tiny_qwen_model_path, scheme=scheme, nsamples=1, iters=1, seqlen=2, dataset=dataloader)
ar.post_init()
assert ar.bits > 0
def test_scheme_in_layer_config(self, dataloader):
model = get_tiny_model(opt_name_or_path, num_layers=5)
tokenizer = transformers.AutoTokenizer.from_pretrained(opt_name_or_path, trust_remote_code=True)
layer_config = {
"model.decoder.layers.2.self_attn": {"bits": 2},
"model.decoder.layers.3.self_attn.v_proj": "W8A16",
"model.decoder.layers.4.self_attn.k_proj": QuantizationScheme.from_dict({"group_size": 64}),
}
ar = AutoRound(
model,
tokenizer,
scheme="W3A16",
nsamples=1,
iters=1,
layer_config=layer_config,
seqlen=2,
dataset=dataloader,
)
ar.quantize_and_save(self.save_folder)
for n, m in ar.model.named_modules():
if n == "model.decoder.layers.2.self_attn.q_proj":
assert m.bits == 2
if n == "model.decoder.layers.2.self_attn.k_proj":
assert m.bits == 2
if n == "model.decoder.layers.3.self_attn.v_proj":
assert m.bits == 8
if n == "model.decoder.layers.4.self_attn.k_proj":
assert m.group_size == 64
def test_parse_available_devices(self):
from auto_round.utils.device import parse_available_devices
from auto_round.utils.device_manager import get_major_device
device_list = parse_available_devices("auto")
# ``parse_available_devices`` resolves to ``cuda:0`` while
# ``get_major_device`` may return the backend name ``cuda``.
assert device_list[0].split(":", 1)[0] == get_major_device("auto").split(":", 1)[0]
device_list = parse_available_devices("a:cuda:0,b:cuda:1,c:cpu")
assert len(device_list) == 3
assert device_list == ["cuda:0", "cuda:1", "cpu"]
device_list = parse_available_devices("0,1")
major_type = get_major_device("0").split(":", 1)[0]
if major_type == "cpu":
assert device_list == ["cpu"]
else:
assert device_list == [f"{major_type}:0", f"{major_type}:1"]
def test_set_scheme(self, tiny_qwen_model_path):
pytest.importorskip("sentencepiece")
ar = AutoRound(
tiny_qwen_model_path,
scheme="gguf:q2_k_s",
data_type="fp",
nsamples=1,
disable_opt_rtn=True,
iters=0,
seqlen=2,
)
ar.quantize_and_save(self.save_folder)
from auto_round.schemes import QuantizationScheme
qs = QuantizationScheme.from_dict({"bits": 4, "group_size": 64})
ar = AutoRound(
tiny_qwen_model_path,
scheme=qs,
bits=2,
data_type="int_asym_dq",
nsamples=1,
iters=0,
disable_opt_rtn=True,
seqlen=2,
)
ar.post_init()
assert ar.scheme == "GGUF:Q2_K_S"
assert ar.scheme_context.bits == 2
def test_fp8_static(self, tiny_opt_model_path):
ar = AutoRound(tiny_opt_model_path, scheme="FP8_STATIC", nsamples=1, iters=1)
ar.post_init()
assert ar.bits == 8
assert ar.act_bits == 8
assert ar.data_type == "fp"
assert ar.act_data_type == "fp"
assert ar.group_size == -1
assert ar.act_dynamic is False
_, quantized_model_path = ar.quantize_and_save()
model = transformers.AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True)
assert model is not None, "Model loading failed after quantization with FP8_STATIC scheme"
def test_fp8_static_rtn(self, tiny_opt_model_path):
ar = AutoRound(tiny_opt_model_path, scheme="FP8_STATIC", nsamples=1, iters=0, disable_opt_rtn=True)
ar.post_init()
assert ar.bits == 8
assert ar.act_bits == 8
assert ar.data_type == "fp"
assert ar.act_data_type == "fp"
assert ar.group_size == -1
assert ar.act_dynamic is False
_, quantized_model_path = ar.quantize_and_save(self.save_folder)
model = transformers.AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True)
assert model is not None, "Model loading failed after quantization with FP8_STATIC scheme"
class TestQ2KMixedRecipe:
"""Unit tests for the gguf:q2_k_mixed _handle_special_schemes recipe.
Uses a small dummy model so no real checkpoint is required.
"""
class _DummyMLP(nn.Module):
def __init__(self):
super().__init__()
self.up_proj = nn.Linear(8, 8)
self.down_proj = nn.Linear(8, 8)
self.gate_proj = nn.Linear(8, 8)
class _DummyMoE(nn.Module):
def __init__(self):
super().__init__()
# singular form used by Qwen2MoE / Qwen3.6 / Qwen3_5MoE
self.shared_expert = TestQ2KMixedRecipe._DummyMLP()
self.shared_expert_gate = nn.Linear(8, 1)
# plural form used by DeepSeek / Mistral-Large
self.shared_experts = TestQ2KMixedRecipe._DummyMLP()
# regular routed experts (should NOT be promoted)
self.experts = nn.ModuleList([nn.Linear(8, 8) for _ in range(2)])
# fused expert weights stay 3D and should remain q2k in GGUF mapping
self.gate_up_proj = nn.Linear(8, 16, bias=False)
self.gate_up_proj.weight = nn.Parameter(torch.zeros(2, 16, 8))
self.down_proj = nn.Linear(8, 8, bias=False)
self.down_proj.weight = nn.Parameter(torch.zeros(2, 8, 8))
class _DummyModel(nn.Module):
def __init__(self):
super().__init__()
self.layer0 = nn.Module()
self.layer0.mlp = TestQ2KMixedRecipe._DummyMoE()
# lm_head is Linear but should get Q8_0, not Q4_K_S
self.lm_head = nn.Linear(8, 8)
# embedding should get Q8_0
self.embed_tokens = nn.Embedding(16, 8)
def setup_method(self):
self.model = self._DummyModel()
self.cfg = _handle_special_schemes("gguf:q2_k_mixed", {}, self.model)
def test_singular_shared_expert_linear_gets_q4ks(self):
"""2D shared expert linears should use Q4_K_S."""
for sub in ("up_proj", "down_proj", "gate_proj"):
key = f"layer0.mlp.shared_expert.{sub}"
assert key in self.cfg, f"{key} missing from layer_config"
assert self.cfg[key] == "GGUF:Q4_K_S", f"{key} expected Q4_K_S, got {self.cfg[key]}"
def test_shared_expert_gate_gets_q4ks(self):
"""2D shared expert gate should use Q4_K_S."""
key = "layer0.mlp.shared_expert_gate"
assert key in self.cfg, f"{key} missing from layer_config"
assert self.cfg[key] == "GGUF:Q4_K_S", f"{key} expected Q4_K_S, got {self.cfg[key]}"
def test_plural_shared_experts_linear_gets_q4ks(self):
"""2D plural shared experts should also use Q4_K_S."""
for sub in ("up_proj", "down_proj", "gate_proj"):
key = f"layer0.mlp.shared_experts.{sub}"
assert key in self.cfg, f"{key} missing from layer_config"
assert self.cfg[key] == "GGUF:Q4_K_S", f"{key} expected Q4_K_S, got {self.cfg[key]}"
def test_routed_experts_linears_not_in_recipe(self):
"""Routed expert linears are excluded from the recipe — falls through to dtype selector."""
for i in range(2):
key = f"layer0.mlp.experts.{i}"
assert key not in self.cfg, f"{key} should not be in the recipe"
def test_fused_three_dim_expert_weights_start_as_q2ks_recipe(self):
"""_handle_special_schemes sets 3D expert weights directly to Q2_K_S."""
assert self.cfg.get("layer0.mlp.gate_up_proj") == "GGUF:Q2_K_S"
assert self.cfg.get("layer0.mlp.down_proj") == "GGUF:Q2_K_S"
def test_lm_head_gets_q8_0(self):
"""lm_head should be Q8_0."""
assert self.cfg.get("lm_head") == "GGUF:Q8_0"
def test_embedding_gets_q8_0(self):
"""embed_tokens embedding should be Q8_0."""
assert self.cfg.get("embed_tokens") == "GGUF:Q8_0"