Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

4.6 KB
Raw Blame
from test.helpers import get_model_path, get_tiny_model, transformers_version

import pytest
import torch
import transformers
from packaging import version

from auto_round import AutoRound
from auto_round.utils.weight_handler import (
    ModuleWeightType,
    check_and_mark_quantized_module,
    convert_module_to_hp_if_necessary,
)


class TestCompressedTensor:
    nvfp4_model_path = "kaitchup/Qwen3-0.6B-NVFP4"
    mxfp4_model_path = "QuixiAI/Llama-3.2-1B-MXFP4"
    fp8_block_model_path = "RedHatAI/Qwen3-0.6B-FP8-BLOCK"
    w4a16_model_path = "RedHatAI/Qwen3-0.6B-quantized.w4a16"

    def test_fp8_block(self):
        model = get_tiny_model(get_model_path(self.fp8_block_model_path))
        assert (
            model.model.layers[0].mlp.up_proj.weight.dtype == torch.float8_e4m3fn
        ), "Original weight is not in FP8 format"
        assert hasattr(
            model.model.layers[0].mlp.up_proj, "quantization_scheme"
        ), "Model does not contain CompressedLinear layers"
        detected_types = check_and_mark_quantized_module(model)
        assert ModuleWeightType.FP8 in detected_types
        model = convert_module_to_hp_if_necessary(model)
        assert (
            model.model.layers[0].mlp.up_proj.weight.dtype == torch.bfloat16
        ), "CompressedLinear layer was not converted to Linear"

    @pytest.mark.skip(
        reason="NVFP4 models are currently not supported due to issues with the compressed_tensors library. See https://github.com/vllm-project/compressed-tensors/issues/642"
    )
    def test_nvfp4(self):
        model = get_tiny_model(get_model_path(self.nvfp4_model_path))
        assert (
            model.model.layers[0].mlp.up_proj.weight_packed.dtype == torch.uint8
        ), "Original weight is not in FP8 format"
        assert hasattr(
            model.model.layers[0].mlp.up_proj, "quantization_scheme"
        ), "Model does not contain CompressedLinear layers"
        detected_types = check_and_mark_quantized_module(model)
        assert ModuleWeightType.NVFP4 in detected_types
        model = convert_module_to_hp_if_necessary(model)
        assert (
            model.model.layers[0].mlp.up_proj.weight.dtype == torch.bfloat16
        ), "CompressedLinear layer was not converted to Linear"

    @pytest.mark.skipif(
        transformers_version >= version.parse("5.0.0"),
        reason="Compressed-tensor is not compatible with transformers 5.0.0 and above. See https://github.com/vllm-project/compressed-tensors/issues/651",
    )
    def test_mxfp4(self):
        model = get_tiny_model(get_model_path(self.mxfp4_model_path))
        assert (
            model.model.layers[0].mlp.up_proj.weight_packed.dtype == torch.uint8
        ), "Original weight is not in FP8 format"
        assert hasattr(
            model.model.layers[0].mlp.up_proj, "quantization_scheme"
        ), "Model does not contain CompressedLinear layers"
        detected_types = check_and_mark_quantized_module(model)
        assert ModuleWeightType.MXFP4 in detected_types
        model = convert_module_to_hp_if_necessary(model)
        assert (
            model.model.layers[0].mlp.up_proj.weight.dtype == torch.bfloat16
        ), "CompressedLinear layer was not converted to Linear"

    def test_w4a16(self):
        model = get_tiny_model(get_model_path(self.w4a16_model_path))
        assert (
            model.model.layers[0].mlp.up_proj.weight_packed.dtype == torch.int32
        ), "Original weight is not in INT4 format"
        assert hasattr(
            model.model.layers[0].mlp.up_proj, "quantization_scheme"
        ), "Model does not contain CompressedLinear layers"
        detected_types = check_and_mark_quantized_module(model)
        assert ModuleWeightType.WOQ in detected_types
        model = convert_module_to_hp_if_necessary(model)
        assert (
            model.model.layers[0].mlp.up_proj.weight.dtype == torch.bfloat16
        ), "CompressedLinear layer was not converted to Linear"

    def test_w4a16_to_mxfp4(self, tmp_path):
        model = get_tiny_model(get_model_path(self.w4a16_model_path))
        model.config.name_or_path = None  # Clear the name_or_path to avoid MTP copying issues
        tokenizer = transformers.AutoTokenizer.from_pretrained(get_model_path(self.w4a16_model_path))
        ar = AutoRound(
            model,
            tokenizer=tokenizer,
            scheme="MXFP4",
            iters=2,
            nsamples=2,
        )
        _, quantized_model_path = ar.quantize_and_save(tmp_path, format="llm_compressor")
        model = transformers.AutoModelForCausalLM.from_pretrained(quantized_model_path)
        assert model, "Failed to load the quantized model"