Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

disable torch compile for gguf with alg_ext (#1582)

Wenhua Chengfcb0d2d1942c
diff --git a/auto_round/alg_ext.py b/auto_round/alg_ext.py
index 8a45b443e..07e3b5b44 100644
--- a/auto_round/alg_ext.py
+++ b/auto_round/alg_ext.py
@@ -156,7 +156,7 @@ def qdq_mxfp(tensor, max_val, max_norm, emax, ebits, mbits):
scale_emax = 2 ** (8 - 1) - 1
shared_exp = (shared_exp - emax).clamp(min=-scale_emax, max=scale_emax)
- scale = torch.pow(2, shared_exp)
+ scale = torch.pow(2.0, shared_exp)
tensor = tensor / scale
tensor = torch.clamp(tensor, min=-max_norm, max=max_norm)
tensor = quant_element(tensor, ebits, mbits, max_norm)
@@ -271,7 +271,7 @@ def quant_mx(
scale_emax = 2 ** (8 - 1) - 1
shared_exp = (shared_exp - emax).clamp(min=-scale_emax, max=scale_emax)
- scale = torch.pow(2, shared_exp)
+ scale = torch.pow(2.0, shared_exp)
tensor = tensor / scale + v
tensor = torch.clamp(tensor, min=-max_norm, max=max_norm)
tensor = quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding)
@@ -564,6 +564,7 @@ def make_qp_quants(nmax, data, quant_weights, v=0):
return sumlx / suml2, L
+# @torch._disable_dynamo()
def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None, v=0):
"""Adapted from Llamacpp. Performs iterative weighted least squares quantization search.
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 864cc069d..5f6f73c6f 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -544,7 +544,7 @@ class BaseCompressor(object):
self.wrapper_block = wrapper_block
if self.enable_alg_ext:
try:
- logger.warning_once("using algorithm extension for quantization.")
+ logger.info("using algorithm extension for quantization.")
from auto_round.alg_ext import wrapper_autoround
wrapper_autoround(self)
@@ -825,6 +825,9 @@ class BaseCompressor(object):
if self.enable_torch_compile and is_nv_fp(self.act_data_type):
self.enable_torch_compile = False
logger.warning("reset enable_torch_compile to `False` as nvfp4 is enabled")
+ if self.enable_torch_compile and self.super_group_size is not None and self.enable_alg_ext:
+ self.enable_torch_compile = False
+ logger.warning("reset enable_torch_compile to `False` as super_group_size is set for algorithm extension")
def _dq_check(self) -> None:
"""Reset the default value of super_bits and super_group_size"""
diff --git a/test/test_cuda/quantization/test_torch_compile.py b/test/test_cuda/quantization/test_torch_compile.py
new file mode 100644
index 000000000..620ef75b0
--- /dev/null
+++ b/test/test_cuda/quantization/test_torch_compile.py
@@ -0,0 +1,72 @@
+import os
+import shutil
+
+import pytest
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+
+from ...envs import require_gguf
+from ...helpers import get_model_path, get_tiny_model
+
+
+class TestTorchCompile:
+ save_dir = "./saved"
+
+ @pytest.fixture(autouse=True, scope="class")
+ def setup_and_teardown_class(self):
+ print("[Setup] Running before any test in class")
+ yield
+ print("[Teardown] Running after all tests in class")
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ @require_gguf
+ def test_gguf_q2ks_torch_compile(self, dataloader):
+ """Test GGUF Q2_K_S quantization with torch.compile enabled.
+
+ Verifies that:
+ 1. Quantization completes successfully with enable_torch_compile=True.
+ 2. The GGUF file is exported correctly.
+ 3. The exported model can be loaded and run inference.
+ """
+ model_path = get_model_path("Qwen/Qwen2.5-1.5B-Instruct")
+ model = get_tiny_model(model_path)
+ tokenizer = AutoTokenizer.from_pretrained(model_path)
+
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ scheme="gguf:q2_k_s",
+ iters=2,
+ seqlen=16,
+ nsamples=2,
+ enable_alg_ext=True,
+ enable_torch_compile=True,
+ )
+ autoround.quantize()
+ autoround.save_quantized(output_dir=self.save_dir)
+
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ @require_gguf
+ def test_gguf_q2ks_torch_compile_iters0(self, tiny_qwen_model_path):
+ """Test GGUF Q2_K_S with torch.compile and iters=0 (RTN mode).
+
+ Verifies that torch.compile does not break the fast RTN-based quantization
+ path when exporting to GGUF Q2_K_S format.
+ """
+ autoround = AutoRound(
+ tiny_qwen_model_path,
+ iters=0,
+ nsamples=2,
+ seqlen=16,
+ enable_torch_compile=True,
+ )
+ autoround.quantize_and_save(output_dir=self.save_dir, format="gguf:q2_k_s")
+
+ saved_files = [f for f in os.listdir(self.save_dir) if f.endswith(".gguf")]
+ assert len(saved_files) > 0, "No GGUF file was generated"
+
+ shutil.rmtree(self.save_dir, ignore_errors=True)