PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix Qwen3-VL model auto_awq export, add auto_awq vllm ut (#1384)
Weiwei40bd641035d0
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex d1328cd22..32d99fd8a 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -35,6 +35,7 @@ from auto_round.export.export_to_awq.utils import WQLinear_GEMMfrom auto_round.export.utils import filter_quantization_config, release_layer_safely, save_modelfrom auto_round.logger import loggerfrom auto_round.utils import (+ INNER_SUPPORTED_LAYER_TYPES,SUPPORTED_LAYER_TYPES,check_to_quantized,copy_python_files_from_model_cache,@@ -44,6 +45,66 @@ from auto_round.utils import (set_module,unsupported_meta_device,)+from auto_round.utils.model import get_layer_names_in_block+++def _is_supported_layer(module: torch.nn.Module) -> bool:+ """Check if module is a supported quantizable layer type."""+ return type(module) in SUPPORTED_LAYER_TYPES or module.__class__.__name__ in INNER_SUPPORTED_LAYER_TYPES+++def _collect_modules_to_not_convert(+ model: torch.nn.Module,+ layer_config: dict,+ regex_config: dict,+ to_quant_block_names: list = None,+) -> list:+ """Collect all module names that should not be converted (not quantized).++ Args:+ model: The model to scan+ layer_config: Configuration dict for layers+ regex_config: Regex-based configuration for layers+ to_quant_block_names: List of block names to quantize (for MLLM models)++ Returns:+ List of module names to not convert+ """+ modules_to_not_convert = set()+ all_blocks = get_block_names(model, quant_vision=True)+ # 1. add non-quantized block directly+ if to_quant_block_names:+ all_block_names = extract_block_names_to_str(all_blocks).split(",")+ to_quant_set = set(+ to_quant_block_names.split(",") if isinstance(to_quant_block_names, str) else to_quant_block_names+ )+ non_quant_blocks = set(all_block_names) - to_quant_set+ modules_to_not_convert.update(non_quant_blocks)++ layers_in_blocks = set(get_layer_names_in_block(model, quant_block_list=all_blocks))++ # 2. Collect non-quantized layers from layer_config+ layers_from_block_patterns = set()+ for layer_name, layer_cfg in layer_config.items():+ if not check_to_quantized(layer_cfg) and not any(name in layer_name for name in modules_to_not_convert):+ layers_from_block_patterns.add(layer_name)+ modules_to_not_convert.update(layers_from_block_patterns)++ # 3. Scan full model for supported layers not in layer_config and not in blocks+ for module_name, module in model.named_modules():+ if _is_supported_layer(module):+ # If this layer is not in layer_config, it wasn't quantized+ if module_name not in layer_config and module_name not in layers_in_blocks:+ # Standalone layer outside blocks+ modules_to_not_convert.add(module_name)++ # 4. Add high-precision layers from regex_config (bits > 8)+ for regex_name, regex_cfg in regex_config.items():+ bits = regex_cfg.get("bits")+ if bits and int(bits) > 8:+ modules_to_not_convert.add(regex_name)++ return list(modules_to_not_convert)def pack_layer(name, model, backend, device=None):@@ -94,7 +155,6 @@ def save_quantized_as_autoawq(to_quant_block_names = serialization_dict.get("to_quant_block_names", None)processor = kwargs.get("processor", None)image_processor = kwargs.get("image_processor", None)- modules_to_not_convert = []if output_dir is not None and os.path.exists(output_dir):logger.warning(f"{output_dir} already exists, this may cause model conflict")@@ -102,14 +162,9 @@ def save_quantized_as_autoawq(logger.info("Saving quantized model to auto_awq format")if tokenizer is not None and hasattr(tokenizer, "save_pretrained") and output_dir is not None:tokenizer.save_pretrained(output_dir)+if processor is not None and output_dir is not None:processor.save_pretrained(output_dir)- # mllm models- all_blocks = get_block_names(model, quant_vision=True)- all_block_names = extract_block_names_to_str(all_blocks)- all_block_names = all_block_names.split(",")- to_quant_block_names = to_quant_block_names.split(",")- modules_to_not_convert = list(set(all_block_names) - set(to_quant_block_names))if image_processor is not None and output_dir is not None:image_processor.save_pretrained(output_dir)@@ -148,19 +203,19 @@ def save_quantized_as_autoawq(if output_dir is None:return compressed_model- for key in layer_config.keys():- if not check_to_quantized(layer_config[key]) and not any(name in key for name in modules_to_not_convert):- modules_to_not_convert.append(key)- for key, cfg in regex_config.items():- bits = cfg.get("bits")- if bits > 8: # save fp_layer regexs- modules_to_not_convert.append(key)+ # Collect all modules that should not be converted (not quantized)+ modules_to_not_convert = _collect_modules_to_not_convert(+ compressed_model,+ layer_config,+ regex_config,+ to_quant_block_names=to_quant_block_names,+ )quantization_config["provider"] = "auto-round"quantization_config["quant_method"] = "awq"quantization_config["zero_point"] = not quantization_config["sym"]quantization_config["version"] = "gemm"- quantization_config["modules_to_not_convert"] = list(dict.fromkeys(modules_to_not_convert))+ quantization_config["modules_to_not_convert"] = modules_to_not_convert##check module quantized in block, this may have bug for mixed precision quantizationfilter_quantization_config(quantization_config)if hasattr(compressed_model, "config"):diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.pyindex 93cfdaac5..35f9adb77 100644--- a/test/test_cpu/export/test_export.py+++ b/test/test_cpu/export/test_export.py@@ -1,3 +1,4 @@+import jsonimport osimport shutil@@ -461,6 +462,34 @@ class TestAutoRound:)get_formats("auto_round:auto_awq", ar)+ def test_autoawq_qwen3_vl_infer(self, dataloader):+ model_path = get_model_path("Qwen/Qwen3-VL-2B-Instruct")+ autoround = AutoRound(+ model=model_path,+ scheme="W4A16",+ iters=0,+ seqlen=2,+ batch_size=1,+ dataset=dataloader,+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_awq")++ # Check items of modules_to_not_convert in quantization config+ quantization_config_path = f"{quantized_model_path}/quantization_config.json"+ with open(quantization_config_path, "r") as f:+ quantization_config = json.load(f)+ modules_to_not_convert = quantization_config.get("modules_to_not_convert", [])+ assert (+ "model.visual.merger.linear_fc2" in modules_to_not_convert+ ), f"'model.visual.merger.linear_fc2' should be in modules_to_not_convert. Got: {modules_to_not_convert}"+ assert (+ "model.visual.merger.linear_fc1" in modules_to_not_convert+ ), f"'model.visual.merger.linear_fc1' should be in modules_to_not_convert. Got: {modules_to_not_convert}"+ assert (+ "model.visual.blocks" in modules_to_not_convert+ ), f"'model.visual.blocks' should be in modules_to_not_convert. Got: {modules_to_not_convert}"+def test_llmc_dynamic_wint8aint8_export(self):from safetensors import safe_opendiff --git a/test/test_cuda/export/test_export.py b/test/test_cuda/export/test_export.pyindex 25f29a4c8..4eb582c32 100644--- a/test/test_cuda/export/test_export.py+++ b/test/test_cuda/export/test_export.py@@ -198,6 +198,55 @@ class TestAutoRound:)shutil.rmtree("./saved", ignore_errors=True)+ def test_autoawq_qwen3_vl_vllm_infer(self, dataloader):+ pytest.importorskip("vllm")+ from vllm import LLM, SamplingParams+ from vllm.platforms import current_platform++ if not current_platform.is_cuda():+ pytest.skip("only supports CUDA backend for this test")++ model_path = get_model_path("Qwen/Qwen3-VL-2B-Instruct")+ autoround = AutoRound(+ model=model_path,+ scheme="W4A16",+ iters=0,+ seqlen=2,+ batch_size=1,+ dataset=dataloader,+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_awq")++ # Check items of modules_to_not_convert in quantization config+ import json++ quantization_config_path = f"{quantized_model_path}/quantization_config.json"+ with open(quantization_config_path, "r") as f:+ quantization_config = json.load(f)+ modules_to_not_convert = quantization_config.get("modules_to_not_convert", [])+ assert (+ "model.visual.merger.linear_fc2" in modules_to_not_convert+ ), f"'model.visual.merger.linear_fc2' should be in modules_to_not_convert. Got: {modules_to_not_convert}"+ assert (+ "model.visual.blocks" in modules_to_not_convert+ ), f"'model.visual.blocks' should be in modules_to_not_convert. Got: {modules_to_not_convert}"++ sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=32)+ llm = LLM(+ model=quantized_model_path,+ quantization="awq",+ trust_remote_code=True,+ tensor_parallel_size=1,+ allow_deprecated_quantization=True,+ dtype="half",+ )+ outputs = llm.generate(["Tell me a short joke."], sampling_params)+ generated_text = outputs[0].outputs[0].text+ print(generated_text)+ assert "!!!" not in generated_text+ shutil.rmtree("./saved", ignore_errors=True)+@require_optimum@require_awq@require_package_version_ut("transformers", "<4.57.0")@@ -363,3 +412,35 @@ class TestAutoRound:res = tokenizer.decode(model.generate(**inputs, max_new_tokens=5)[0])print(res)shutil.rmtree(quantized_model_path, ignore_errors=True)++ def test_autogtq_qwen3_vl_vllm_infer(self, dataloader):+ pytest.importorskip("vllm")+ from vllm import LLM, SamplingParams+ from vllm.platforms import current_platform++ if not current_platform.is_cuda():+ pytest.skip("only supports CUDA backend for this test")++ model_path = get_model_path("Qwen/Qwen3-VL-2B-Instruct")+ autoround = AutoRound(+ model=model_path,+ scheme="W4A16",+ iters=0,+ seqlen=2,+ batch_size=1,+ dataset=dataloader,+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_gptq")+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=20)+ llm = LLM(+ model=quantized_model_path,+ trust_remote_code=True,+ tensor_parallel_size=1,+ allow_deprecated_quantization=True,+ )+ outputs = llm.generate(["Tell me a short joke."], sampling_params)+ generated_text = outputs[0].outputs[0].text+ print(generated_text)+ assert "!!!" not in generated_text+ shutil.rmtree("./saved", ignore_errors=True)