Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix accuracy regression and check it in CUDA CI (#1785)

Xin He36767481bb59
diff --git a/auto_round/algorithms/quantization/rtn/quantizer.py b/auto_round/algorithms/quantization/rtn/quantizer.py
index 864ca40fe..260743518 100644
--- a/auto_round/algorithms/quantization/rtn/quantizer.py
+++ b/auto_round/algorithms/quantization/rtn/quantizer.py
@@ -155,7 +155,7 @@ class RTNQuantizer(BaseQuantizers):
enable_round_tuning=False,
enable_torch_compile=self.compress_context.enable_torch_compile,
disable_opt_rtn=disable_opt_rtn,
- enable_rtn=True,
+ iters=0,
)
m = m.unwrapper({})
except torch.OutOfMemoryError:
@@ -171,7 +171,7 @@ class RTNQuantizer(BaseQuantizers):
enable_norm_bias_tuning=False,
enable_round_tuning=False,
enable_torch_compile=self.compress_context.enable_torch_compile,
- enable_rtn=True,
+ iters=0,
)
m = m.unwrapper({})
except Exception as e:
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 0503c8235..99134c6f9 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -3616,6 +3616,8 @@ class BaseCompressor(object):
folders.append(save_folder)
if return_folders:
+ if len(folders) == 1:
+ folders = folders[0]
return compressed_model, folders
else:
return compressed_model
diff --git a/test/test_cuda/models/test_fp8_model.py b/test/test_cuda/models/test_fp8_model.py
index 11fa10650..7eceacfbb 100644
--- a/test/test_cuda/models/test_fp8_model.py
+++ b/test/test_cuda/models/test_fp8_model.py
@@ -17,6 +17,8 @@ from auto_round.utils.weight_handler import (
from ...helpers import evaluate_accuracy, generate_prompt, get_model_path, get_tiny_model, transformers_version
+DEVICE_CAPABILITY = torch.cuda.get_device_capability()
+
class TestAutoRound:
@@ -59,7 +61,7 @@ class TestAutoRound:
model_name = get_model_path("Qwen/Qwen3-0.6B-FP8")
ar = AutoRound(model=model_name, iters=0, disable_opt_rtn=True)
_, folder = ar.quantize_and_save(output_dir=self.save_dir)
- with patch("torch.cuda.get_device_capability", return_value=(8, 0)): # revert mock_fp8_capable_device
+ with patch("torch.cuda.get_device_capability", return_value=DEVICE_CAPABILITY): # revert DEVICE_CAPABILITY
evaluate_accuracy(folder, threshold=0.25)
@pytest.mark.skip_ci(reason="Triton issue; time-consuming")
@@ -67,24 +69,23 @@ class TestAutoRound:
model_name = get_model_path("Qwen/Qwen3-0.6B-FP8")
ar = AutoRound(model=model_name, iters=1)
_, folder = ar.quantize_and_save(output_dir=self.save_dir)
- with patch("torch.cuda.get_device_capability", return_value=(8, 0)): # revert mock_fp8_capable_device
+ with patch("torch.cuda.get_device_capability", return_value=DEVICE_CAPABILITY): # revert DEVICE_CAPABILITY
evaluate_accuracy(folder, threshold=0.25)
- @pytest.mark.skip_ci(reason="Triton issue; time-consuming")
- def test_medium_model_rtn(self, mock_fp8_capable_device):
+ def test_small_model_opt_rtn(self, mock_fp8_capable_device):
model_name = get_model_path("Qwen/Qwen3-0.6B-FP8")
ar = AutoRound(model=model_name, iters=0)
_, folder = ar.quantize_and_save(output_dir=self.save_dir)
- with patch("torch.cuda.get_device_capability", return_value=(8, 0)): # revert mock_fp8_capable_device
+ with patch("torch.cuda.get_device_capability", return_value=DEVICE_CAPABILITY): # revert DEVICE_CAPABILITY
evaluate_accuracy(folder, threshold=0.33)
@pytest.mark.skip_ci(reason="Triton issue; time-consuming")
- def test_medium_model_rtn_with_lm_head(self, mock_fp8_capable_device):
+ def test_small_model_rtn_with_lm_head(self, mock_fp8_capable_device):
model_name = get_model_path("Qwen/Qwen3-0.6B-FP8")
layer_config = {"lm_head": {"bits": 4}}
ar = AutoRound(model=model_name, iters=0, layer_config=layer_config)
_, folder = ar.quantize_and_save(output_dir=self.save_dir)
- with patch("torch.cuda.get_device_capability", return_value=(8, 0)): # revert mock_fp8_capable_device
+ with patch("torch.cuda.get_device_capability", return_value=DEVICE_CAPABILITY): # revert DEVICE_CAPABILITY
evaluate_accuracy(folder, threshold=0.33)
def test_fp8_model_gguf_q4(self, mock_fp8_capable_device, tiny_fp8_qwen_model_path):
diff --git a/test/test_cuda/models/test_get_block_name.py b/test/test_cuda/models/test_get_block_name.py
index 843e139ae..0933573a0 100644
--- a/test/test_cuda/models/test_get_block_name.py
+++ b/test/test_cuda/models/test_get_block_name.py
@@ -161,8 +161,9 @@ class TestAutoRound:
)
assert not is_pure_text_model(model)
- def test_gemma3(self, tmp_path):
+ def test_gemma3(self):
model_name = get_model_path("google/gemma-3-12b-it")
+ tmp_path = "./tmp/tiny_gemma3" # fixture:clean_tmp_model_folder will clean it
save_tiny_model(
model_name, tmp_path, from_config=True
) # make sure the model is downloaded before the test, to avoid timeout in the test