Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

15.0 KB
Raw Blame
import importlib.util
import os
import shutil
import sys
from test.helpers import (
    check_version,
    eval_generated_prompt,
    evaluate_accuracy,
    generate_prompt,
    get_model_path,
    save_tiny_model,
)

import pytest
import torch
import transformers
from packaging import version
from transformers import AutoModelForCausalLM, AutoTokenizer

from auto_round import AutoRound
from auto_round.modeling.fused_moe.fusion_spec import get_moe_fusion_spec

from ...envs import require_gguf

AUTO_ROUND_PATH = __file__.split("/")
AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])


class TestAutoRound:

    @pytest.fixture(autouse=True, scope="class")
    def setup_and_teardown_class(self):
        # ===== SETUP (setup_class) =====
        print("[Setup] Running before any test in class")

        # Yield to hand control to the test methods
        yield

        # ===== TEARDOWN (teardown_class) =====
        print("[Teardown] Running after all tests in class")
        shutil.rmtree("runs", ignore_errors=True)

    @pytest.fixture(autouse=True)
    def _save_dir(self, tmp_path):
        self.save_dir = str(tmp_path / "saved")
        yield
        shutil.rmtree(self.save_dir, ignore_errors=True)

    def _export_qwen35_moe_gguf(self, tiny_qwen35_moe_model_path):
        import gguf

        from auto_round.modeling.fused_moe.qwen3_5_moe import SequentialQwen3_5MoeExperts

        autoround = AutoRound(
            tiny_qwen35_moe_model_path,
            iters=0,
            nsamples=1,
            seqlen=8,
            disable_opt_rtn=True,
        )
        quantized_model, save_folder = autoround.quantize_and_save(output_dir=self.save_dir, format="gguf:q4_0")
        expert_count = quantized_model.config.get_text_config().num_experts
        live_expert_containers = [
            module for module in quantized_model.modules() if isinstance(module, SequentialQwen3_5MoeExperts)
        ]
        assert live_expert_containers, "Qwen3.5 quantization must retain live routed expert containers"
        expected_projections = {
            "gate_up_proj": ("gate_proj", "up_proj"),
            "down_proj": ("down_proj",),
        }
        for experts in live_expert_containers:
            fusion_spec = get_moe_fusion_spec(experts)
            assert fusion_spec is not None, "Qwen3.5 live experts must carry a registered MoEFusionSpec"
            assert fusion_spec.num_experts == expert_count
            assert {
                projection.checkpoint_projection: projection.source_projections
                for projection in fusion_spec.projections
            } == expected_projections
            assert len(experts) == expert_count
            assert all(
                all(hasattr(expert, projection) for projection in ("gate_proj", "up_proj", "down_proj"))
                for expert in experts
            )

        text_gguf_files = [
            os.path.join(save_folder, filename)
            for filename in os.listdir(save_folder)
            if filename.endswith(".gguf") and "mmproj" not in filename
        ]
        assert len(text_gguf_files) == 1
        gguf_path = text_gguf_files[0]

        reader = gguf.GGUFReader(gguf_path)
        for projection in ("gate", "up", "down"):
            tensors = [tensor for tensor in reader.tensors if tensor.name.endswith(f".ffn_{projection}_exps.weight")]
            assert tensors, f"missing fused expert {projection} tensor"
            assert all(len(tensor.shape) == 3 for tensor in tensors)
            assert all(int(tensor.shape[-1]) == expert_count for tensor in tensors)
            assert all(tensor.tensor_type == gguf.GGMLQuantizationType.Q4_0 for tensor in tensors)
        return gguf_path

    @pytest.mark.skip_ci(
        reason="Architecture: Qwen3.5 fused-MoE GGUF export writes 3D expert tensors and requires a large fixture"
    )
    @pytest.mark.skipif(not check_version("transformers>=5.2.0"), reason="requires transformers >= 5.2.0")
    @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
    @require_gguf
    def test_qwen35_moe_gguf(self, tiny_qwen35_moe_model_path, monkeypatch):
        monkeypatch.setenv("AR_DISABLE_GGUF_MTP_EXPORT", "1")
        self._export_qwen35_moe_gguf(tiny_qwen35_moe_model_path)

    @pytest.mark.skip(
        reason="llama-cpp-python in CI intermittently crashes/fails loading the new 3D fused-MoE GGUF "
        "tensors for Qwen3.5; skip until the dependency is confirmed/updated to support this format."
    )
    @pytest.mark.skipif(not check_version("transformers>=5.2.0"), reason="requires transformers >= 5.2.0")
    @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
    @pytest.mark.skipif(importlib.util.find_spec("llama_cpp") is None, reason="requires llama_cpp")
    @require_gguf
    def test_qwen35_moe_llama_cpp_load(self, tiny_qwen35_moe_model_path):
        import llama_cpp

        gguf_path = self._export_qwen35_moe_gguf(tiny_qwen35_moe_model_path)
        model = llama_cpp.Llama(model_path=gguf_path, n_ctx=32, logits_all=False)
        assert model is not None

    @require_gguf
    def test_gguf_format(self, tiny_qwen_model_path, dataloader):
        bits, group_size, sym = 4, 32, False
        autoround = AutoRound(
            tiny_qwen_model_path,
            bits=bits,
            group_size=group_size,
            sym=sym,
            iters=2,
            seqlen=2,
            nsamples=2,
            dataset=dataloader,
        )
        autoround.quantize()
        quantized_model_path = self.save_dir
        autoround.save_quantized(output_dir=quantized_model_path, format="gguf:q4_1")

    @pytest.mark.skip_ci(reason="Accuracy: Time-consuming accuracy evaluation; covered by nightly")
    @require_gguf
    def test_q4_0_accuracy(self):
        model_name = get_model_path("Qwen/Qwen2.5-0.5B-Instruct")
        bits, group_size, sym = 4, 32, True
        autoround = AutoRound(
            model_name, bits=bits, group_size=group_size, sym=sym, iters=0, data_type="int", disable_opt_rtn=True
        )
        autoround.quantize()
        quantized_model_path = self.save_dir

        autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0")

        gguf_file = os.listdir(quantized_model_path)[0]

        model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
        eval_generated_prompt(model, autoround.tokenizer)

        evaluate_accuracy(model, autoround.tokenizer, threshold=0.54, batch_size=16, task="piqa")

    @pytest.mark.skip_ci(reason="Matrix: Not necessary to test all options in CI")
    @require_gguf
    def test_q2_k_export(self, dataloader):
        bits, group_size, sym = 2, 16, False
        model_path = get_model_path("Qwen/Qwen2.5-0.5B-Instruct")
        tiny_model_path = "./tmp/tmp_tiny_qwen_model_path"
        save_tiny_model(model_path, tiny_model_path, num_layers=2)
        autoround = AutoRound(
            model_path,
            bits=bits,
            group_size=group_size,
            sym=sym,
            iters=1,
            seqlen=1,
            dataset=dataloader,
            data_type="int_asym_dq",
        )
        autoround.quantize()
        quantized_model_path = self.save_dir

        autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="gguf:q2_k_s")
        gguf_file = os.listdir(quantized_model_path)[0]
        model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
        result = generate_prompt(model, autoround.tokenizer)
        print(result)
        shutil.rmtree(tiny_model_path, ignore_errors=True)

    @pytest.mark.skip_ci(reason="Matrix: Not necessary to test all options in CI")
    @require_gguf
    def test_all_format(self):
        for model_name in ["Qwen/Qwen3-8B", "meta-llama/Llama-3.2-3B"]:
            for gguf_format in ["gguf:q5_0", "gguf:q5_1", "gguf:q3_k_m", "gguf:q5_k_m", "gguf:q6_k", "gguf:q8_0"]:
                model_path = get_model_path(model_name)
                tiny_model_path = "tmp_tiny_model"
                tiny_model_path = save_tiny_model(model_path, tiny_model_path, num_layers=2)
                ar = AutoRound(tiny_model_path, scheme=gguf_format, iters=0, nsamples=1, seqlen=16)
                ar.quantize_and_save(output_dir=self.save_dir, format=gguf_format)

                ar = AutoRound(tiny_model_path, scheme=gguf_format, iters=1, nsamples=1, seqlen=16)
                ar.quantize_and_save(output_dir=self.save_dir, format=gguf_format)

                shutil.rmtree(tiny_model_path, ignore_errors=True)

    @pytest.mark.skip_ci(reason="Architecture: Not necessary to test special models in CI")
    @require_gguf
    def test_special_model(self):
        from test.helpers import save_tiny_model

        model_name = get_model_path("ibm-granite/granite-4.0-h-tiny")
        tiny_model_path = save_tiny_model(model_name, "tiny_granite_model_path", num_layers=2)
        from auto_round import AutoRound

        autoround = AutoRound(
            tiny_model_path,
            iters=0,
            nsamples=8,
            disable_opt_rtn=True,
        )
        quantized_model_path = self.save_dir
        _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")
        file_name = os.listdir(quantized_model_path)[0]
        file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2
        assert abs(file_size - 307) < 5.0
        shutil.rmtree(tiny_model_path, ignore_errors=True)

    @pytest.mark.skip_ci(
        reason="Architecture: Gemma VLM GGUF export validates mmproj output and requires a large multimodal fixture"
    )
    @require_gguf
    def test_vlm_gguf(self):
        from test.helpers import save_tiny_model

        from huggingface_hub import hf_hub_download
        from huggingface_hub.errors import GatedRepoError, HfHubHTTPError

        model_name = "google/gemma-3-4b-it"
        tiny_model_path = save_tiny_model(
            model_name, "tiny_model_path", num_layers=3, is_mllm=True, use_fast=False, use_config=True
        )
        # Needs tokenizer.model for gguf
        # New transformers won't download it even with use_fast=False
        tokenizer_model_path = os.path.join(tiny_model_path, "tokenizer.model")
        if not os.path.isfile(tokenizer_model_path):
            try:
                hf_hub_download(repo_id=model_name, filename="tokenizer.model", local_dir=tiny_model_path)
            except (GatedRepoError, HfHubHTTPError) as error:
                pytest.skip(f"tokenizer.model unavailable for {model_name}: {error}")

        from auto_round import AutoRound

        autoround = AutoRound(
            tiny_model_path,
            device="auto",
            nsamples=32,
            iters=0,
            disable_opt_rtn=True,
            quant_nontext_module=True,
        )
        quantized_model_path = self.save_dir
        _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")
        assert "mmproj-model.gguf" in os.listdir(quantized_model_path)
        for file in os.listdir(quantized_model_path):
            print(f"{file}: {os.path.getsize(os.path.join(quantized_model_path, file)) / 1024**2} MB")
            file_size = os.path.getsize(os.path.join(quantized_model_path, file)) / 1024**2
            if "mmproj-model.gguf" in file:
                assert abs(file_size - 75) < 5.0
            else:
                assert abs(file_size - 690) < 10.0

        shutil.rmtree(tiny_model_path, ignore_errors=True)

    @pytest.mark.skip_ci(reason="Matrix: Not necessary to test all options in CI")
    def test_q2k_mixed(self):
        model_path = get_model_path("miromind-ai/MiroThinker-v1.5-30B")
        saved_tiny_model_path = save_tiny_model(
            model_path,
            "./tmp/tiny_qwen_model_path",
            num_layers=3,
            is_mllm=False,
        )
        autoround = AutoRound(
            saved_tiny_model_path,
            iters=0,
            nsamples=1,
            seqlen=16,
            disable_opt_rtn=True,
        )
        quantized_model_path = self.save_dir
        _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q2_k_mixed")
        gguf_file = os.listdir(quantized_model_path)[0]
        file_size = os.path.getsize(os.path.join(quantized_model_path, gguf_file)) / 1024**2
        assert abs(file_size - 1236) < 5.0
        from gguf.gguf_reader import GGUFReader

        gguf_model = GGUFReader(os.path.join(quantized_model_path, gguf_file))
        assert gguf_model.get_tensor(2).name == "blk.0.attn_v.weight"
        assert gguf_model.get_tensor(2).tensor_type.name == "Q4_K"
        assert gguf_model.get_tensor(9).name == "blk.0.ffn_up_exps.weight"
        assert gguf_model.get_tensor(9).tensor_type.name == "Q2_K"

        shutil.rmtree(saved_tiny_model_path, ignore_errors=True)

    @pytest.mark.skip_ci(
        reason="Matrix: The Q2_K_S first-eighth FFN-down dtype rule is covered by a dedicated GGUF dtype unit test"
    )
    @require_gguf
    def test_q2_k_s_ffn_down_q4k(self):
        """Verify blk.0.ffn_down.weight is Q4_K in gguf:q2_k_s format.
        Blocks where i_layer < n_layer/8 should use Q4_K instead of Q2_K for ffn_down."""
        from gguf.gguf_reader import GGUFReader

        model_path = get_model_path("Qwen/Qwen3-1.7B")
        tiny_model_path = "./tmp/tiny_qwen3_1b"
        tiny_model_path = save_tiny_model(model_path, tiny_model_path, num_layers=8)
        autoround = AutoRound(
            tiny_model_path,
            iters=0,
            nsamples=1,
            seqlen=16,
            disable_opt_rtn=True,
        )
        quantized_model_path = self.save_dir
        _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q2_k_s")
        gguf_file = os.listdir(quantized_model_path)[0]
        gguf_model = GGUFReader(os.path.join(quantized_model_path, gguf_file))
        ffn_down_type = None
        for tensor in gguf_model.tensors:
            if tensor.name == "blk.0.ffn_down.weight":
                ffn_down_type = tensor.tensor_type.name
                break
        assert ffn_down_type is not None, "blk.0.ffn_down.weight not found in GGUF file"
        assert ffn_down_type == "Q4_K", f"Expected Q4_K for blk.0.ffn_down.weight but got {ffn_down_type}"
        shutil.rmtree(tiny_model_path, ignore_errors=True)

    @pytest.mark.skip_ci(reason="Architecture: Only tiny model is suggested for CI")
    def test_gguf_baseline(self):
        model_name = get_model_path("Qwen/Qwen2.5-1.5B-Instruct")
        autoround = AutoRound(
            model_name,
            bits=3,
            group_size=16,
            sym=True,
            iters=0,
            nsamples=8,
            seqlen=2,
            data_type="rtn_int_sym_dq",
            super_group_size=16,
            super_bits=6,
            disable_opt_rtn=True,
        )
        quantized_model_path = self.save_dir
        _, quantized_model_path = autoround.quantize_and_save(
            output_dir=quantized_model_path, inplace=False, format="fake"
        )
        eval_generated_prompt(quantized_model_path)