PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
disable torch compile for gguf with alg_ext (#1582)
Wenhua Chengfcb0d2d1942c
diff --git a/auto_round/alg_ext.py b/auto_round/alg_ext.pyindex 8a45b443e..07e3b5b44 100644--- a/auto_round/alg_ext.py+++ b/auto_round/alg_ext.py@@ -156,7 +156,7 @@ def qdq_mxfp(tensor, max_val, max_norm, emax, ebits, mbits):scale_emax = 2 ** (8 - 1) - 1shared_exp = (shared_exp - emax).clamp(min=-scale_emax, max=scale_emax)- scale = torch.pow(2, shared_exp)+ scale = torch.pow(2.0, shared_exp)tensor = tensor / scaletensor = torch.clamp(tensor, min=-max_norm, max=max_norm)tensor = quant_element(tensor, ebits, mbits, max_norm)@@ -271,7 +271,7 @@ def quant_mx(scale_emax = 2 ** (8 - 1) - 1shared_exp = (shared_exp - emax).clamp(min=-scale_emax, max=scale_emax)- scale = torch.pow(2, shared_exp)+ scale = torch.pow(2.0, shared_exp)tensor = tensor / scale + vtensor = torch.clamp(tensor, min=-max_norm, max=max_norm)tensor = quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding)@@ -564,6 +564,7 @@ def make_qp_quants(nmax, data, quant_weights, v=0):return sumlx / suml2, L+# @torch._disable_dynamo()def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None, v=0):"""Adapted from Llamacpp. Performs iterative weighted least squares quantization search.diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 864cc069d..5f6f73c6f 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -544,7 +544,7 @@ class BaseCompressor(object):self.wrapper_block = wrapper_blockif self.enable_alg_ext:try:- logger.warning_once("using algorithm extension for quantization.")+ logger.info("using algorithm extension for quantization.")from auto_round.alg_ext import wrapper_autoroundwrapper_autoround(self)@@ -825,6 +825,9 @@ class BaseCompressor(object):if self.enable_torch_compile and is_nv_fp(self.act_data_type):self.enable_torch_compile = Falselogger.warning("reset enable_torch_compile to `False` as nvfp4 is enabled")+ if self.enable_torch_compile and self.super_group_size is not None and self.enable_alg_ext:+ self.enable_torch_compile = False+ logger.warning("reset enable_torch_compile to `False` as super_group_size is set for algorithm extension")def _dq_check(self) -> None:"""Reset the default value of super_bits and super_group_size"""diff --git a/test/test_cuda/quantization/test_torch_compile.py b/test/test_cuda/quantization/test_torch_compile.pynew file mode 100644index 000000000..620ef75b0--- /dev/null+++ b/test/test_cuda/quantization/test_torch_compile.py@@ -0,0 +1,72 @@+import os+import shutil++import pytest+import torch+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound++from ...envs import require_gguf+from ...helpers import get_model_path, get_tiny_model+++class TestTorchCompile:+ save_dir = "./saved"++ @pytest.fixture(autouse=True, scope="class")+ def setup_and_teardown_class(self):+ print("[Setup] Running before any test in class")+ yield+ print("[Teardown] Running after all tests in class")+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ @require_gguf+ def test_gguf_q2ks_torch_compile(self, dataloader):+ """Test GGUF Q2_K_S quantization with torch.compile enabled.++ Verifies that:+ 1. Quantization completes successfully with enable_torch_compile=True.+ 2. The GGUF file is exported correctly.+ 3. The exported model can be loaded and run inference.+ """+ model_path = get_model_path("Qwen/Qwen2.5-1.5B-Instruct")+ model = get_tiny_model(model_path)+ tokenizer = AutoTokenizer.from_pretrained(model_path)++ autoround = AutoRound(+ model,+ tokenizer,+ scheme="gguf:q2_k_s",+ iters=2,+ seqlen=16,+ nsamples=2,+ enable_alg_ext=True,+ enable_torch_compile=True,+ )+ autoround.quantize()+ autoround.save_quantized(output_dir=self.save_dir)++ shutil.rmtree(self.save_dir, ignore_errors=True)++ @require_gguf+ def test_gguf_q2ks_torch_compile_iters0(self, tiny_qwen_model_path):+ """Test GGUF Q2_K_S with torch.compile and iters=0 (RTN mode).++ Verifies that torch.compile does not break the fast RTN-based quantization+ path when exporting to GGUF Q2_K_S format.+ """+ autoround = AutoRound(+ tiny_qwen_model_path,+ iters=0,+ nsamples=2,+ seqlen=16,+ enable_torch_compile=True,+ )+ autoround.quantize_and_save(output_dir=self.save_dir, format="gguf:q2_k_s")++ saved_files = [f for f in os.listdir(self.save_dir) if f.endswith(".gguf")]+ assert len(saved_files) > 0, "No GGUF file was generated"++ shutil.rmtree(self.save_dir, ignore_errors=True)