Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

3.0 KB
Raw Blame
import os
import shutil
from test.helpers import get_model_path, get_tiny_model
from types import SimpleNamespace

import pytest
from transformers import AutoTokenizer

from auto_round import AutoRound
from auto_round.algorithms.quantization.base import BaseQuantizer
from auto_round.algorithms.quantization.rtn.config import RTNConfig
from auto_round.compressors.utils import block_forward

from ...envs import require_gguf

pytestmark = pytest.mark.enable_torch_compile


class TestTorchCompile:
    save_dir = "./saved"

    @pytest.fixture(autouse=True, scope="class")
    def setup_and_teardown_class(self):
        print("[Setup] Running before any test in class")
        yield
        print("[Teardown] Running after all tests in class")
        shutil.rmtree("./saved", ignore_errors=True)
        shutil.rmtree("runs", ignore_errors=True)

    @pytest.mark.skip_ci(reason="Backend/JIT: torch.compile warmup with iters=2 is slow")
    @require_gguf
    def test_gguf_q2ks_torch_compile(self, dataloader):
        """Test GGUF Q2_K_S quantization with torch.compile enabled.

        Verifies that:
        1. Quantization completes successfully with enable_torch_compile=True.
        2. The GGUF file is exported correctly.
        3. The exported model can be loaded and run inference.
        """
        model_path = get_model_path("Qwen/Qwen2.5-1.5B-Instruct")
        model = get_tiny_model(model_path)
        tokenizer = AutoTokenizer.from_pretrained(model_path)

        autoround = AutoRound(
            model,
            tokenizer,
            scheme="gguf:q2_k_s",
            iters=2,
            seqlen=16,
            nsamples=2,
            enable_alg_ext=True,
            enable_torch_compile=True,
        )
        autoround.quantize()
        autoround.save_quantized(output_dir=self.save_dir)

        shutil.rmtree(self.save_dir, ignore_errors=True)

    @require_gguf
    def test_gguf_q2ks_torch_compile_iters0(self, tiny_qwen_model_path):
        """Test GGUF Q2_K_S with torch.compile and iters=0 (RTN mode).

        Verifies that torch.compile does not break the fast RTN-based quantization
        path when exporting to GGUF Q2_K_S format.
        """
        autoround = AutoRound(
            tiny_qwen_model_path,
            iters=0,
            nsamples=2,
            seqlen=16,
            enable_torch_compile=True,
        )
        _, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="gguf:q2_k_s")

        saved_files = [f for f in os.listdir(quantized_model_path) if f.endswith(".gguf")]
        assert len(saved_files) > 0, "No GGUF file was generated"

        shutil.rmtree(self.save_dir, ignore_errors=True)

    # def test_opt_rtn_uses_plain_block_forward(self):
    #     config = RTNConfig(bits=4, data_type="int", act_bits=16, disable_opt_rtn=False)
    #     quantizer = BaseQuantizer(config)
    #     quantizer.compress_context = SimpleNamespace(enable_torch_compile=True, device="cpu")
    #
    #     assert quantizer._resolve_block_forward() is block_forward