Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

12.7 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for asymmetric (sym=False) quantization: RTN and tuning paths, across formats and quant params."""

import shutil
from test.helpers import model_infer

import pytest
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

from auto_round import AutoRound
from auto_round.utils.device_manager import get_available_device_types

# "cpu" is always available; extend with whatever accelerators AutoRound detects (cuda/xpu/hpu/...).
_AVAILABLE_DEVICES = ["cpu"] + [d for d in get_available_device_types() if d != "cpu"]
_CUDA_AVAILABLE = "cuda" in _AVAILABLE_DEVICES
requires_cuda = pytest.mark.skipif(not _CUDA_AVAILABLE, reason="requires a CUDA device")

# (bits, group_size) combinations exercised by the quant-param sweep below.
# 8-bit asym is absent on purpose: this suite exports to native int8-packed
# formats, which refuse 8-bit asym at construction (vLLM serves W8 asym only
# via compressed-tensors on the llm_compressor format); see
# test_w8_asym_refused_at_construction and test_w8_asym_llm_compressor_format_allowed.
_QUANT_PARAMS = [(4, 32), (4, 64), (4, 128), (2, 128)]

_FORMATS = [
    "auto_round",
    "auto_round:auto_gptq",
    # GPTQModel's ExLlama post-init can segfault on some CUDA/Python
    # combinations.  Its CUDA backend has dedicated coverage; keep this
    # common test active on CPU/XPU and skip only the unstable CUDA variant.
    pytest.param(
        "auto_round:gptqmodel",
        marks=pytest.mark.skipif(torch.cuda.is_available(), reason="GPTQModel ExLlama CUDA path is covered separately"),
    ),
]


def _device_params(devices):
    """Skip only XPU parameterizations until the asymmetric XPU backend is available."""
    return [
        (
            pytest.param(
                device,
                marks=pytest.mark.skip(reason="asymmetric quantization backend is not available on XPU"),
            )
            if device == "xpu"
            else device
        )
        for device in devices
    ]


class TestAutoRoundAsym:
    @pytest.fixture(autouse=True)
    def _save_dir(self, tmp_path):
        self.save_dir = str(tmp_path / "saved")
        yield
        shutil.rmtree(self.save_dir, ignore_errors=True)

    @pytest.fixture(autouse=True, scope="class")
    def setup_and_teardown_class(self):
        yield
        shutil.rmtree("runs", ignore_errors=True)

    # ------------------------------------------------------------------
    # RTN path (iters=0): cheap, so runs unconditionally in CI on every device.
    # ------------------------------------------------------------------
    @pytest.mark.parametrize("device", _device_params(_AVAILABLE_DEVICES))
    @pytest.mark.parametrize("format", _FORMATS)
    def test_asym_format_rtn(self, tiny_opt_model_path, format, device):
        """RTN-quantized asym model can be saved in each export format and reloaded for inference."""
        if str(device).startswith("cuda") and format in {
            "auto_round",
            "auto_round:auto_gptq",
            "auto_round:gptqmodel",
        }:
            pytest.skip("GPTQModel ExLlamaV2 CUDA path is covered by dedicated backend tests")
        bits, group_size, sym = 4, 128, False
        ar = AutoRound(
            tiny_opt_model_path,
            bits=bits,
            group_size=group_size,
            sym=sym,
            iters=0,
            seqlen=2,
            nsamples=1,
            disable_opt_rtn=True,
        )
        _, quantized_model_path = ar.quantize_and_save(format=format, output_dir=self.save_dir)

        model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map=device)
        tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
        model_infer(model, tokenizer)

    @pytest.mark.parametrize("device", _device_params(_AVAILABLE_DEVICES))
    @pytest.mark.parametrize("bits,group_size", _QUANT_PARAMS)
    def test_asym_quant_params_rtn(self, tiny_opt_model_path, bits, group_size, device):
        """RTN asym quantization works across bit-widths and group sizes, and the result is loadable."""
        ar = AutoRound(tiny_opt_model_path, bits=bits, group_size=group_size, sym=False, iters=0, seqlen=2, nsamples=1)
        _, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=self.save_dir)

        model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map=device)
        tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
        model_infer(model, tokenizer)

    def test_w8_asym_refused_at_construction(self, tiny_opt_model_path):
        """8-bit asym raises at AutoRound construction for the default native
        export format, for both iters=0 and iters=1 (the llm_compressor format
        keeps it; see test_w8_asym_llm_compressor_format_allowed)."""
        for iters in (0, 1):
            with pytest.raises(ValueError, match="8-bit asymmetric"):
                AutoRound(
                    tiny_opt_model_path,
                    bits=8,
                    group_size=128,
                    sym=False,
                    iters=iters,
                    seqlen=2,
                    nsamples=1,
                )

    def test_w8_asym_llm_compressor_format_allowed(self, tiny_opt_model_path):
        """8-bit asym is servable via compressed-tensors, so the llm_compressor
        format must accept it end-to-end (native formats still refuse)."""
        ar = AutoRound(
            tiny_opt_model_path,
            bits=8,
            group_size=128,
            sym=False,
            iters=0,
            disable_opt_rtn=True,
            format="auto_round:llm_compressor",
            seqlen=2,
            nsamples=1,
        )
        _, out = ar.quantize_and_save(format="auto_round:llm_compressor", output_dir=self.save_dir)
        assert out

    def test_w8_asym_env_overrides_native_refusal(self, tiny_opt_model_path, monkeypatch):
        """AR_ALLOW_W8_ASYM=1 skips the native-format refusal; the user opts
        into artifacts that stock vLLM GPTQ serving may reject."""
        monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")
        ar = AutoRound(
            tiny_opt_model_path,
            bits=8,
            group_size=128,
            sym=False,
            iters=0,
            disable_opt_rtn=True,
            seqlen=2,
            nsamples=1,
        )
        assert type(ar).__name__ == "ModelFreeCompressor"

    def test_w8_asym_scheme_object_spellings_regular_flow(self, tiny_opt_model_path, monkeypatch):
        """A QuantizationScheme OBJECT with sym=False must take the same
        format/env policy as the kwargs spelling on the regular flow."""
        from auto_round.schemes import QuantizationScheme

        obj = QuantizationScheme(bits=8, sym=False, data_type="int", group_size=128, act_bits=16)
        monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")
        ar = AutoRound(
            tiny_opt_model_path,
            scheme=obj,
            iters=0,
            disable_opt_rtn=True,
            disable_model_free=True,
            amp=False,
            seqlen=2,
            nsamples=1,
        )
        assert type(ar).__name__ == "CompressionOrchestrator"
        monkeypatch.delenv("AR_ALLOW_W8_ASYM")
        ar2 = AutoRound(
            tiny_opt_model_path,
            scheme=obj,
            iters=0,
            disable_opt_rtn=True,
            disable_model_free=True,
            format="auto_round:llm_compressor",
            amp=False,
            seqlen=2,
            nsamples=1,
        )
        assert type(ar2).__name__ == "CompressionOrchestrator"  # llmc needs no env

    def test_w8_asym_llm_compressor_recheck_at_save(self, tiny_opt_model_path, tmp_path):
        """A W8-asym scheme that slipped past construction without format
        knowledge (direct ModelFreeCompressor use) re-raises at save time for
        non-servable formats; llm_compressor passes the re-check."""
        from auto_round.compressors.model_free import ModelFreeCompressor
        from auto_round.schemes import QuantizationScheme

        obj = QuantizationScheme(bits=8, sym=False, data_type="int", group_size=128, act_bits=16)
        mf = ModelFreeCompressor(tiny_opt_model_path, scheme=obj, output_dir=str(tmp_path / "mf_w8_save"))
        with pytest.raises(ValueError, match="8-bit asymmetric"):
            mf.quantize_and_save(format="auto_round")
        # the llmc spelling passes the same re-check
        mf2 = ModelFreeCompressor(
            tiny_opt_model_path, scheme=obj, format="llm_compressor", output_dir=str(tmp_path / "mf_w8_save2")
        )
        assert mf2.format == "llm_compressor"

    def test_w8_asym_env_survives_model_free_fallback(self, tiny_opt_model_path, tmp_path):
        """The W8-asym opt-in is a process-wide environment variable, so the
        model-free fallback (which re-constructs AutoRound internally) reads
        the same opt-in without any kwarg forwarding."""
        from auto_round.compressors.model_free import ModelFreeCompressor

        mf = ModelFreeCompressor(
            tiny_opt_model_path,
            scheme="W8A16",
            sym=False,
            bits=8,
            group_size=128,
            output_dir=str(tmp_path / "mf_w8"),
        )
        assert "allow_w8_asym" not in mf._fallback_init_kwargs

    # ------------------------------------------------------------------
    # Tuning path (iters=1): exercises the real sign-gradient tuning loop.
    # This is genuinely slow (real gradient tuning), so it's cheaper in wall-clock
    # time to run once on cuda than on every device (bits/format sweeps below are
    # heavier still and stay skip_ci -- manual/nightly coverage only).
    # ------------------------------------------------------------------
    @requires_cuda
    @pytest.mark.parametrize("group_size", [32])
    def test_asym_group_size_tuning(self, tiny_opt_model_path, group_size, dataloader):
        """Tuned (iters=1) asym quantization works across group sizes."""
        device = "cuda"
        ar = AutoRound(
            tiny_opt_model_path,
            bits=4,
            group_size=group_size,
            sym=False,
            iters=1,
            seqlen=2,
            nsamples=1,
            dataset=dataloader,
        )
        _, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=self.save_dir)

        model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map=device)
        tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
        model_infer(model, tokenizer)

    @pytest.mark.skip_ci(reason="Coverage: Not necessary since it's covered by backend tests")
    @pytest.mark.parametrize("device", _device_params(_AVAILABLE_DEVICES))
    @pytest.mark.parametrize("bits", [2, 3])
    def test_asym_bits_tuning(self, tiny_opt_model_path, bits, device):
        """Tuned (iters=1) asym quantization works across bit-widths."""
        ar = AutoRound(tiny_opt_model_path, bits=bits, group_size=128, sym=False, iters=1, seqlen=2, nsamples=1)
        _, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=self.save_dir)

        model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map=device)
        tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
        model_infer(model, tokenizer)

    @pytest.mark.skip_ci(reason="Coverage: Not necessary since it's covered by backend tests")
    @pytest.mark.parametrize("device", _device_params(_AVAILABLE_DEVICES))
    @pytest.mark.parametrize("format", _FORMATS)
    def test_asym_format_tuning(self, tiny_opt_model_path, format, device):
        """Tuned (iters=1) asym model can be saved in each export format and reloaded for inference."""
        bits, group_size, sym = 4, 128, False
        ar = AutoRound(tiny_opt_model_path, bits=bits, group_size=group_size, sym=sym, iters=1, seqlen=2, nsamples=1)
        _, quantized_model_path = ar.quantize_and_save(format=format, output_dir=self.save_dir)

        if format == "auto_round:auto_gptq":
            # Cannot load correctly, skip auto_gptq since it's deprecated.
            return

        model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map=device)
        tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
        model_infer(model, tokenizer)