PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support gptqmodel 7.0.0 and fix bug in CI (#1772)
Xin He871f6f3c4b12
diff --git a/auto_round/compressors_new/calib.py b/auto_round/compressors_new/calib.pyindex dea8233df..67c77c058 100644--- a/auto_round/compressors_new/calib.py+++ b/auto_round/compressors_new/calib.py@@ -943,7 +943,11 @@ class CalibCompressor(BaseCompressor):materialize_model_(m)convert_module_to_hp_if_necessary(m, self.model_context.amp_dtype, self.compress_context.device)- if is_auto_device_mapping(self.compress_context.device_map) and len(self.compress_context.device_list) > 1:+ if (+ is_auto_device_mapping(self.compress_context.device_map)+ and len(self.compress_context.device_list) > 1+ and not self.model_context.is_diffusion+ ):from auto_round.utils.device import set_auto_device_map_for_block_with_tuningcard_0_in_high_risk, loss_device = set_auto_device_map_for_block_with_tuning(@@ -1457,6 +1461,7 @@ class CalibratedRTNCompressor(CalibCompressor):if (is_auto_device_mapping(self.compress_context.device_map)and len(self.compress_context.device_list) > 1+ and not self.model_context.is_diffusion):from auto_round.utils.device import set_auto_device_map_for_block_with_tuningdiff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex c46b8c947..0b5bc359e 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -272,15 +272,20 @@ def save_quantized_as_autogptq(modules_in_block_to_quantize = []# for backward compatibilityfor block_names in all_blocks:- first_block = get_module(model, block_names[0])- for n, m in first_block.named_modules():- if m.global_name not in layer_config:- continue- if not check_to_quantized(layer_config[m.global_name]):- all_to_quantized = False- else:- modules_in_block_to_quantize.append(n)- modules_in_block_to_quantize = [modules_in_block_to_quantize]+ quantized_in_group = set()+ not_quantized_in_group = set()+ for block_name in block_names:+ block = get_module(model, block_name)+ for n, m in block.named_modules():+ if m.global_name not in layer_config:+ continue+ if not check_to_quantized(layer_config[m.global_name]):+ not_quantized_in_group.add(n)+ else:+ quantized_in_group.add(n)+ if not_quantized_in_group:+ all_to_quantized = False+ modules_in_block_to_quantize.append(sorted(quantized_in_group))if all_to_quantized:modules_in_block_to_quantize = Nonediff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex d643ff659..b37bcfa33 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -14,9 +14,11 @@import functoolsimport platformfrom dataclasses import dataclass, field+from importlib import import_modulefrom typing import TYPE_CHECKING, Any, Optionalimport torch+from packaging.version import Versionfrom transformers.utils.versions import require_versionimport auto_round_extension.cuda.gptqmodel_marlin@@ -881,21 +883,18 @@ def dynamic_import_inference_linear(backend, config, packing_format=None):return QuantLinear- # Handle gptqmodel GPTQ backends- if "gptqmodel" in backend and "awq" not in backend:- return get_gptqmodel_infer_linear(backend, bits, group_size, sym)+ # Handle gptqmodel GPTQ/AWQ backends+ if "gptqmodel" in backend:+ return get_gptqmodel_infer_linear(backend, is_awq="awq" in backend)if "gptq" in backend and "gptqmodel" not in backend:return get_autogptq_infer_linear(backend, bits, group_size, sym)if "awq" in backend:- if "gptqmodel" in backend:- return get_gptqmodel_awq_infer_linear(backend)- else:- # Fallback to autoawq for backward compatibility- from awq.modules.linear import WQLinear_GEMM # pylint: disable=E0401+ # Fallback to autoawq for backward compatibility+ from awq.modules.linear import WQLinear_GEMM # pylint: disable=E0401- return WQLinear_GEMM+ return WQLinear_GEMMif backend == "auto_round:tritonv2":from auto_round_extension.triton.qlinear_tritonv2 import QuantLinear@@ -938,46 +937,43 @@ def safe_import_gptqmodel():torch.set_default_dtype(dtype)-def get_gptqmodel_awq_infer_linear(backend):- """Returns the appropriate gptqmodel AWQ QuantLinear class for inference."""-+def get_gptqmodel_infer_linear(backend, is_awq=False):gptqmodel = safe_import_gptqmodel()-- # Select AWQ kernel based on the BackendInfos key- if "marlin" in backend:- from gptqmodel.nn_modules.qlinear.marlin_awq import AwqMarlinQuantLinear # pylint: disable=E0401-- return AwqMarlinQuantLinear- elif "exllamav2" in backend:- from gptqmodel.nn_modules.qlinear.exllamav2_awq import AwqExllamaV2QuantLinear # pylint: disable=E0401-- return AwqExllamaV2QuantLinear- elif "gemm" in backend:- from gptqmodel.nn_modules.qlinear.gemm_awq import AwqGEMMQuantLinear # pylint: disable=E0401-- return AwqGEMMQuantLinear- elif "torch" in backend:- from gptqmodel.nn_modules.qlinear.torch_awq import AwqTorchQuantLinear # pylint: disable=E0401-- return AwqTorchQuantLinear+ new_version = Version(gptqmodel.__version__) >= Version("7.0.0")+ if is_awq:+ backend_specs = (+ ("marlin", "gptqmodel.nn_modules.qlinear.marlin_awq", "AwqMarlinLinear", "AwqMarlinQuantLinear"),+ (+ "exllamav2",+ "gptqmodel.nn_modules.qlinear.exllamav2_awq",+ "AwqExllamaV2Linear",+ "AwqExllamaV2QuantLinear",+ ),+ ("gemm", "gptqmodel.nn_modules.qlinear.gemm_awq", "AwqGEMMLinear", "AwqGEMMQuantLinear"),+ ("torch", "gptqmodel.nn_modules.qlinear.torch_awq", "AwqTorchLinear", "AwqTorchQuantLinear"),+ )else:- raise ValueError(f"Unsupported {backend}")---def get_gptqmodel_infer_linear(backend, bits=4, group_size=128, sym=False):- gptqmodel = safe_import_gptqmodel()+ backend_specs = (+ ("marlin", auto_round_extension.cuda.gptqmodel_marlin.get_marlin_layer),+ ("exllamav2", "gptqmodel.nn_modules.qlinear.exllamav2", "ExllamaV2Linear", "ExllamaV2QuantLinear"),+ ("tritonv2", "gptqmodel.nn_modules.qlinear.tritonv2", "TritonV2Linear", "TritonV2QuantLinear"),+ ("torch", "gptqmodel.nn_modules.qlinear.torch", "TorchLinear", "TorchQuantLinear"),+ )- if "marlin" in backend:- return auto_round_extension.cuda.gptqmodel_marlin.get_marlin_layer()- # return gptqmodel.nn_modules.qlinear.marlin.MarlinQuantLinear- elif "exllamav2" in backend:- return gptqmodel.nn_modules.qlinear.exllamav2.ExllamaV2QuantLinear- elif "tritonv2" in backend:- return gptqmodel.nn_modules.qlinear.tritonv2.TritonV2QuantLinear- elif "torch" in backend:- return gptqmodel.nn_modules.qlinear.torch.TorchQuantLinear- else:- raise ValueError(f"Unsupported {backend}")+ for spec in backend_specs:+ if spec[0] not in backend:+ continue+ if is_awq:+ _, module_path, new_name, legacy_name = spec+ module = import_module(module_path)+ return getattr(module, new_name) if new_version else getattr(module, legacy_name)+ if spec[0] == "marlin":+ return spec[1]()+ _, module_path, new_name, legacy_name = spec+ module = import_module(module_path)+ return getattr(module, new_name) if new_version else getattr(module, legacy_name)++ raise ValueError(f"Unsupported {backend}")def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 70bb311f7..a69211348 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -17,6 +17,7 @@ from typing import Unionimport torchimport torch.nn as nn+from packaging.version import Versionfrom tqdm import tqdmfrom transformers.pytorch_utils import Conv1D@@ -559,7 +560,14 @@ def infer_target_device(device_map: Union[dict, int, str, None] = None) -> str:def convert_gptq_v1_to_v2_format(model: nn.Module):"""Convert gptq v1 to v2 format to ensure compatible with gptqmodel:exllamav2 backend."""- from gptqmodel.nn_modules.qlinear.exllamav2 import ExllamaV2QuantLinear # pylint: disable=E0401+ import gptqmodel # pylint: disable=E0401++ if Version(gptqmodel.__version__) >= Version("7.0.0"):+ from gptqmodel.nn_modules.qlinear.exllamav2 import ExllamaV2Linear # pylint: disable=E0401++ ExllamaV2QuantLinear = ExllamaV2Linear # pylint: disable=E0401+ else:+ from gptqmodel.nn_modules.qlinear.exllamav2 import ExllamaV2QuantLinear # pylint: disable=E0401for n, m in model.named_modules():if isinstance(m, ExllamaV2QuantLinear):diff --git a/auto_round_extension/cuda/gptqmodel_marlin.py b/auto_round_extension/cuda/gptqmodel_marlin.pyindex 144209b08..a804572f5 100644--- a/auto_round_extension/cuda/gptqmodel_marlin.py+++ b/auto_round_extension/cuda/gptqmodel_marlin.py@@ -40,6 +40,10 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demandNEW_VERSION_6_0 = Falseif Version(gptqmodel.__version__) >= Version("6.0.0"):NEW_VERSION_6_0 = True+ NEW_VERSION_7_0 = False+ if Version(gptqmodel.__version__) >= Version("7.0.0"):+ NEW_VERSION_7_0 = True+from gptqmodel.models._const import DEVICE, PLATFORM # pylint: disable=E0401from gptqmodel.nn_modules.qlinear import BaseQuantLinear # pylint: disable=E0401from gptqmodel.utils.backend import BACKEND # pylint: disable=E0401@@ -51,10 +55,13 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demandNEW_VERSION = Falsemarlin_import_exception = None- try:- import gptqmodel_marlin_kernels # pylint: disable=E0401- except ImportError as e:- marlin_import_exception = e+ if NEW_VERSION_7_0:+ import gptqmodel.utils.marlin as gptqmodel_marlin_kernels # pylint: disable=E0401+ else:+ try:+ import gptqmodel_marlin_kernels # pylint: disable=E0401+ except ImportError as e:+ marlin_import_exception = efrom auto_round.utils import logger@@ -187,7 +194,7 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demandg_idx,g_idx_sort_indices,workspace,- wtype.id,+ wtype if NEW_VERSION_7_0 else wtype.id,reshaped_x.shape[0],output_size_per_partition,input_size_per_partition,diff --git a/test/README.md b/test/README.mdindex c17170de5..ee0af61f1 100644--- a/test/README.md+++ b/test/README.md@@ -80,7 +80,6 @@ save_tiny_model(model_path, save_path) # Save tiny model to disk**Model Inference:**```pythonmodel_infer(model, tokenizer, input_text) # Run inference and return output-is_model_outputs_similar(out1, out2) # Compare two model outputs```**Data Utilities:**diff --git a/test/helpers.py b/test/helpers.pyindex b568c137a..cfbc7ceee 100644--- a/test/helpers.py+++ b/test/helpers.py@@ -110,9 +110,6 @@ def get_model_path(model_name: str) -> str:local_path = f"/models/{model_name.split('/')[-1]}"local_path_1 = f"/dataset/{model_name.split('/')[-1]}"- if "DeepSeek-V2-Lite" in model_name and os.path.exists("/data0/deepseek-ai/DeepSeek-V2-Lite"):- return "/data0/deepseek-ai/DeepSeek-V2-Lite"-if os.path.exists(ut_path):return ut_pathelif os.path.exists(local_path):@@ -300,31 +297,27 @@ def get_tiny_model(else:trust_remote_code = kwargs.get("trust_remote_code", True)config = transformers.AutoConfig.from_pretrained(model_name_or_path, trust_remote_code=trust_remote_code)- use_config_only = not (not trust_remote_code and getattr(config, "auto_map", None))- if use_config_only:- # Special cases, for transformers == 5.4.0- if config.model_type == "qwen3_omni_moe":- config.initializer_range = 0.02 # Default initializer range for weight initialization- _reduce_config_layers(config, num_layers, num_experts)-- # Pick the right model class- base_lib = transformers- architectures = getattr(config, "architectures", [None])[0]- if (- is_mllm- and architectures.endswith("Model")- and hasattr(base_lib, n := architectures.replace("Model", "ForConditionalGeneration"))- ):- model_cls = getattr(base_lib, n)- elif hasattr(base_lib, architectures):- model_cls = getattr(base_lib, architectures)- else:- model_cls = (- transformers.AutoModelForCausalLM- ) # default to causal LM if we can't find a better match- model = model_cls._from_config(config)- model = model.eval()- return model+ # Special cases, for transformers == 5.4.0+ if config.model_type == "qwen3_omni_moe":+ config.initializer_range = 0.02 # Default initializer range for weight initialization+ _reduce_config_layers(config, num_layers, num_experts)++ # Pick the right model class+ base_lib = transformers+ architectures = getattr(config, "architectures", [None])[0]+ if (+ is_mllm+ and architectures.endswith("Model")+ and hasattr(base_lib, n := architectures.replace("Model", "ForConditionalGeneration"))+ ):+ model_cls = getattr(base_lib, n)+ elif hasattr(base_lib, architectures):+ model_cls = getattr(base_lib, architectures)+ else:+ model_cls = transformers.AutoModelForCausalLM # default to causal LM if we can't find a better match+ model = model_cls._from_config(config)+ model = model.eval()+ return model# ---- original path: load pretrained weights then slice ----def slice_layers(module):@@ -539,76 +532,6 @@ def get_output(model_name_or_path):return outputs.detach().cpu()[email protected]_mode()-def is_model_outputs_similar(model_path_1, model_path_2, metric="cosine_similarity", threshold=0.98, k=5, verbose=True):- """- Compare outputs from two models using specified metric and return pass/fail.-- Args:- model_path_1: Path to first model- model_path_2: Path to second model- metric: Metric to use - "mse", "cosine_similarity"/"cos_sim", or "topk"- threshold: Threshold value for pass/fail- k: K value for top-k metric (only used when metric="topk")- verbose: Whether to print detailed results-- Returns:- bool: True if metric passes threshold, False otherwise- """- if verbose:- print(f"\n{'='*70}")- print("Comparing Model Outputs")- print(f"{'='*70}")- print(f"Model 1: {model_path_1}")- print(f"Model 2: {model_path_2}")- print(f"Metric: {metric} | Threshold: {threshold}" + (f" | K: {k}" if "top" in metric.lower() else ""))- print(f"{'='*70}\n")-- output_1 = get_output(model_path_1)- output_2 = get_output(model_path_2)- metric = metric.lower().replace("-", "_")-- # Calculate metric and check threshold- if metric == "mse":- value = torch.mean((output_1.float() - output_2.float()) ** 2).item()- passed = value <= threshold- if verbose:- print(f"MSE: {value:.6f} | Threshold: <= {threshold} | {'✓ PASS' if passed else '✗ FAIL'}\n")-- elif metric in ["cosine_similarity", "cos_sim", "cosine"]:- out1 = output_1.float().flatten()- out2 = output_2.float().flatten()- value = torch.nn.functional.cosine_similarity(out1.unsqueeze(0), out2.unsqueeze(0)).item()- passed = value >= threshold- if verbose:- print(f"Cosine Similarity: {value:.6f} | Threshold: >= {threshold} | {'✓ PASS' if passed else '✗ FAIL'}\n")-- elif metric in ["topk", "top_k"]:- _, topk_1 = torch.topk(output_1, k=min(k, output_1.size(-1)), dim=-1)- _, topk_2 = torch.topk(output_2, k=min(k, output_2.size(-1)), dim=-1)-- total_agreement = 0- total_positions = topk_1.numel() // topk_1.size(-1)-- for i in range(topk_1.size(0)):- for j in range(topk_1.size(1)):- set1 = set(topk_1[i, j].tolist())- set2 = set(topk_2[i, j].tolist())- total_agreement += len(set1 & set2) / k-- value = total_agreement / total_positions- passed = value >= threshold- if verbose:- print(- f"Top-{k} Agreement: {value:.4%} | Threshold: >= {threshold:.4%} | {'✓ PASS' if passed else '✗ FAIL'}\n"- )-- else:- raise ValueError(f"Unknown metric: {metric}. Choose from: 'mse', 'cosine_similarity', 'topk'")-- return passed--def is_cuda_support_fp8(major=9, minor=0):"""Check if the current CUDA device capability is >= (major, minor).diff --git a/test/test_cpu/quantization/test_mxfp_nvfp.py b/test/test_cpu/quantization/test_mxfp_nvfp.pyindex 2b091a068..21cb10204 100644--- a/test/test_cpu/quantization/test_mxfp_nvfp.py+++ b/test/test_cpu/quantization/test_mxfp_nvfp.py@@ -11,7 +11,7 @@ from auto_round import AutoRoundfrom auto_round.export.export_to_autoround import export_to_nvfp_mx as autoround_nvfp_mx_exportfrom ...envs import require_compressed_tensors-from ...helpers import forbid_threaded_packing, is_model_outputs_similar, transformers_version+from ...helpers import forbid_threaded_packing, transformers_versiondef _get_folder_size(path: str) -> float:@@ -91,7 +91,6 @@ class TestAutoRoundFP:and lm_head.weight_packed.dtype is torch.uint8and lm_head.weight_scale.dtype is torch.float8_e4m3fn), "Illegal NVFP4 packing for lm_head layer"- assert is_model_outputs_similar(model_name, quantized_model_path)def test_mxfp4_moe_ar(self, tiny_deepseek_v2_model_path, dataloader):model_name = tiny_deepseek_v2_model_path@@ -349,7 +348,6 @@ class TestAutoRoundFP:_, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, inplace=True, format="auto_round")- assert is_model_outputs_similar(model_name, quantized_model_path)@pytest.mark.parametrize("scheme, static_kv_dtype, static_attention_dtype",diff --git a/test/test_cuda/models/test_fp8_model.py b/test/test_cuda/models/test_fp8_model.pyindex 0bbbba1f8..11fa10650 100644--- a/test/test_cuda/models/test_fp8_model.py+++ b/test/test_cuda/models/test_fp8_model.py@@ -1,5 +1,6 @@import osimport shutil+from unittest.mock import patchimport pytestimport torch@@ -56,23 +57,26 @@ class TestAutoRound:@pytest.mark.skip_ci(reason="Triton issue; time-consuming")def test_small_model_rtn(self, mock_fp8_capable_device):model_name = get_model_path("Qwen/Qwen3-0.6B-FP8")- ar = AutoRound(model=model_name, iters=0)+ ar = AutoRound(model=model_name, iters=0, disable_opt_rtn=True)_, folder = ar.quantize_and_save(output_dir=self.save_dir)- evaluate_accuracy(folder, threshold=0.25)+ with patch("torch.cuda.get_device_capability", return_value=(8, 0)): # revert mock_fp8_capable_device+ evaluate_accuracy(folder, threshold=0.25)@pytest.mark.skip_ci(reason="Triton issue; time-consuming")def test_small_model_iters1(self, mock_fp8_capable_device):model_name = get_model_path("Qwen/Qwen3-0.6B-FP8")ar = AutoRound(model=model_name, iters=1)_, folder = ar.quantize_and_save(output_dir=self.save_dir)- evaluate_accuracy(folder, threshold=0.25)+ with patch("torch.cuda.get_device_capability", return_value=(8, 0)): # revert mock_fp8_capable_device+ evaluate_accuracy(folder, threshold=0.25)@pytest.mark.skip_ci(reason="Triton issue; time-consuming")def test_medium_model_rtn(self, mock_fp8_capable_device):model_name = get_model_path("Qwen/Qwen3-0.6B-FP8")ar = AutoRound(model=model_name, iters=0)_, folder = ar.quantize_and_save(output_dir=self.save_dir)- evaluate_accuracy(folder, threshold=0.33)+ with patch("torch.cuda.get_device_capability", return_value=(8, 0)): # revert mock_fp8_capable_device+ evaluate_accuracy(folder, threshold=0.33)@pytest.mark.skip_ci(reason="Triton issue; time-consuming")def test_medium_model_rtn_with_lm_head(self, mock_fp8_capable_device):@@ -80,7 +84,8 @@ class TestAutoRound:layer_config = {"lm_head": {"bits": 4}}ar = AutoRound(model=model_name, iters=0, layer_config=layer_config)_, folder = ar.quantize_and_save(output_dir=self.save_dir)- evaluate_accuracy(folder, threshold=0.33)+ with patch("torch.cuda.get_device_capability", return_value=(8, 0)): # revert mock_fp8_capable_device+ evaluate_accuracy(folder, threshold=0.33)def test_fp8_model_gguf_q4(self, mock_fp8_capable_device, tiny_fp8_qwen_model_path):from llama_cpp import Llamadiff --git a/test/test_cuda/models/test_get_block_name.py b/test/test_cuda/models/test_get_block_name.pyindex 4f26987ec..843e139ae 100644--- a/test/test_cuda/models/test_get_block_name.py+++ b/test/test_cuda/models/test_get_block_name.py@@ -17,7 +17,7 @@ from transformers import (from auto_round import AutoRoundfrom auto_round.utils import get_block_names, is_pure_text_model-from ...helpers import get_model_path, transformers_version+from ...helpers import get_model_path, save_tiny_model, transformers_version@pytest.mark.skip_ci(reason="Only tiny model is suggested")@@ -161,15 +161,18 @@ class TestAutoRound:)assert not is_pure_text_model(model)- def test_gemma3(self):+ def test_gemma3(self, tmp_path):model_name = get_model_path("google/gemma-3-12b-it")- model = Gemma3ForConditionalGeneration.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ save_tiny_model(+ model_name, tmp_path, from_config=True+ ) # make sure the model is downloaded before the test, to avoid timeout in the test+ model = Gemma3ForConditionalGeneration.from_pretrained(tmp_path, torch_dtype="auto", trust_remote_code=True)block_names = get_block_names(model)self.check_block_names(block_names, ["model.language_model.layers"], [48])block_names = get_block_names(model, quant_vision=True)self.check_block_names(- block_names, ["model.vision_tower.vision_model.encoder.layers", "model.language_model.layers"], [27, 48]+ block_names, ["model.vision_tower.encoder.layers", "model.language_model.layers"], [27, 48])assert not is_pure_text_model(model)