Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix Qwen3-VL model auto_awq export, add auto_awq vllm ut (#1384)

Weiwei40bd641035d0
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index d1328cd22..32d99fd8a 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -35,6 +35,7 @@ from auto_round.export.export_to_awq.utils import WQLinear_GEMM
from auto_round.export.utils import filter_quantization_config, release_layer_safely, save_model
from auto_round.logger import logger
from auto_round.utils import (
+ INNER_SUPPORTED_LAYER_TYPES,
SUPPORTED_LAYER_TYPES,
check_to_quantized,
copy_python_files_from_model_cache,
@@ -44,6 +45,66 @@ from auto_round.utils import (
set_module,
unsupported_meta_device,
)
+from auto_round.utils.model import get_layer_names_in_block
+
+
+def _is_supported_layer(module: torch.nn.Module) -> bool:
+ """Check if module is a supported quantizable layer type."""
+ return type(module) in SUPPORTED_LAYER_TYPES or module.__class__.__name__ in INNER_SUPPORTED_LAYER_TYPES
+
+
+def _collect_modules_to_not_convert(
+ model: torch.nn.Module,
+ layer_config: dict,
+ regex_config: dict,
+ to_quant_block_names: list = None,
+) -> list:
+ """Collect all module names that should not be converted (not quantized).
+
+ Args:
+ model: The model to scan
+ layer_config: Configuration dict for layers
+ regex_config: Regex-based configuration for layers
+ to_quant_block_names: List of block names to quantize (for MLLM models)
+
+ Returns:
+ List of module names to not convert
+ """
+ modules_to_not_convert = set()
+ all_blocks = get_block_names(model, quant_vision=True)
+ # 1. add non-quantized block directly
+ if to_quant_block_names:
+ all_block_names = extract_block_names_to_str(all_blocks).split(",")
+ to_quant_set = set(
+ to_quant_block_names.split(",") if isinstance(to_quant_block_names, str) else to_quant_block_names
+ )
+ non_quant_blocks = set(all_block_names) - to_quant_set
+ modules_to_not_convert.update(non_quant_blocks)
+
+ layers_in_blocks = set(get_layer_names_in_block(model, quant_block_list=all_blocks))
+
+ # 2. Collect non-quantized layers from layer_config
+ layers_from_block_patterns = set()
+ for layer_name, layer_cfg in layer_config.items():
+ if not check_to_quantized(layer_cfg) and not any(name in layer_name for name in modules_to_not_convert):
+ layers_from_block_patterns.add(layer_name)
+ modules_to_not_convert.update(layers_from_block_patterns)
+
+ # 3. Scan full model for supported layers not in layer_config and not in blocks
+ for module_name, module in model.named_modules():
+ if _is_supported_layer(module):
+ # If this layer is not in layer_config, it wasn't quantized
+ if module_name not in layer_config and module_name not in layers_in_blocks:
+ # Standalone layer outside blocks
+ modules_to_not_convert.add(module_name)
+
+ # 4. Add high-precision layers from regex_config (bits > 8)
+ for regex_name, regex_cfg in regex_config.items():
+ bits = regex_cfg.get("bits")
+ if bits and int(bits) > 8:
+ modules_to_not_convert.add(regex_name)
+
+ return list(modules_to_not_convert)
def pack_layer(name, model, backend, device=None):
@@ -94,7 +155,6 @@ def save_quantized_as_autoawq(
to_quant_block_names = serialization_dict.get("to_quant_block_names", None)
processor = kwargs.get("processor", None)
image_processor = kwargs.get("image_processor", None)
- modules_to_not_convert = []
if output_dir is not None and os.path.exists(output_dir):
logger.warning(f"{output_dir} already exists, this may cause model conflict")
@@ -102,14 +162,9 @@ def save_quantized_as_autoawq(
logger.info("Saving quantized model to auto_awq format")
if tokenizer is not None and hasattr(tokenizer, "save_pretrained") and output_dir is not None:
tokenizer.save_pretrained(output_dir)
+
if processor is not None and output_dir is not None:
processor.save_pretrained(output_dir)
- # mllm models
- all_blocks = get_block_names(model, quant_vision=True)
- all_block_names = extract_block_names_to_str(all_blocks)
- all_block_names = all_block_names.split(",")
- to_quant_block_names = to_quant_block_names.split(",")
- modules_to_not_convert = list(set(all_block_names) - set(to_quant_block_names))
if image_processor is not None and output_dir is not None:
image_processor.save_pretrained(output_dir)
@@ -148,19 +203,19 @@ def save_quantized_as_autoawq(
if output_dir is None:
return compressed_model
- for key in layer_config.keys():
- if not check_to_quantized(layer_config[key]) and not any(name in key for name in modules_to_not_convert):
- modules_to_not_convert.append(key)
- for key, cfg in regex_config.items():
- bits = cfg.get("bits")
- if bits > 8: # save fp_layer regexs
- modules_to_not_convert.append(key)
+ # Collect all modules that should not be converted (not quantized)
+ modules_to_not_convert = _collect_modules_to_not_convert(
+ compressed_model,
+ layer_config,
+ regex_config,
+ to_quant_block_names=to_quant_block_names,
+ )
quantization_config["provider"] = "auto-round"
quantization_config["quant_method"] = "awq"
quantization_config["zero_point"] = not quantization_config["sym"]
quantization_config["version"] = "gemm"
- quantization_config["modules_to_not_convert"] = list(dict.fromkeys(modules_to_not_convert))
+ quantization_config["modules_to_not_convert"] = modules_to_not_convert
##check module quantized in block, this may have bug for mixed precision quantization
filter_quantization_config(quantization_config)
if hasattr(compressed_model, "config"):
diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.py
index 93cfdaac5..35f9adb77 100644
--- a/test/test_cpu/export/test_export.py
+++ b/test/test_cpu/export/test_export.py
@@ -1,3 +1,4 @@
+import json
import os
import shutil
@@ -461,6 +462,34 @@ class TestAutoRound:
)
get_formats("auto_round:auto_awq", ar)
+ def test_autoawq_qwen3_vl_infer(self, dataloader):
+ model_path = get_model_path("Qwen/Qwen3-VL-2B-Instruct")
+ autoround = AutoRound(
+ model=model_path,
+ scheme="W4A16",
+ iters=0,
+ seqlen=2,
+ batch_size=1,
+ dataset=dataloader,
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_awq")
+
+ # Check items of modules_to_not_convert in quantization config
+ quantization_config_path = f"{quantized_model_path}/quantization_config.json"
+ with open(quantization_config_path, "r") as f:
+ quantization_config = json.load(f)
+ modules_to_not_convert = quantization_config.get("modules_to_not_convert", [])
+ assert (
+ "model.visual.merger.linear_fc2" in modules_to_not_convert
+ ), f"'model.visual.merger.linear_fc2' should be in modules_to_not_convert. Got: {modules_to_not_convert}"
+ assert (
+ "model.visual.merger.linear_fc1" in modules_to_not_convert
+ ), f"'model.visual.merger.linear_fc1' should be in modules_to_not_convert. Got: {modules_to_not_convert}"
+ assert (
+ "model.visual.blocks" in modules_to_not_convert
+ ), f"'model.visual.blocks' should be in modules_to_not_convert. Got: {modules_to_not_convert}"
+
def test_llmc_dynamic_wint8aint8_export(self):
from safetensors import safe_open
diff --git a/test/test_cuda/export/test_export.py b/test/test_cuda/export/test_export.py
index 25f29a4c8..4eb582c32 100644
--- a/test/test_cuda/export/test_export.py
+++ b/test/test_cuda/export/test_export.py
@@ -198,6 +198,55 @@ class TestAutoRound:
)
shutil.rmtree("./saved", ignore_errors=True)
+ def test_autoawq_qwen3_vl_vllm_infer(self, dataloader):
+ pytest.importorskip("vllm")
+ from vllm import LLM, SamplingParams
+ from vllm.platforms import current_platform
+
+ if not current_platform.is_cuda():
+ pytest.skip("only supports CUDA backend for this test")
+
+ model_path = get_model_path("Qwen/Qwen3-VL-2B-Instruct")
+ autoround = AutoRound(
+ model=model_path,
+ scheme="W4A16",
+ iters=0,
+ seqlen=2,
+ batch_size=1,
+ dataset=dataloader,
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_awq")
+
+ # Check items of modules_to_not_convert in quantization config
+ import json
+
+ quantization_config_path = f"{quantized_model_path}/quantization_config.json"
+ with open(quantization_config_path, "r") as f:
+ quantization_config = json.load(f)
+ modules_to_not_convert = quantization_config.get("modules_to_not_convert", [])
+ assert (
+ "model.visual.merger.linear_fc2" in modules_to_not_convert
+ ), f"'model.visual.merger.linear_fc2' should be in modules_to_not_convert. Got: {modules_to_not_convert}"
+ assert (
+ "model.visual.blocks" in modules_to_not_convert
+ ), f"'model.visual.blocks' should be in modules_to_not_convert. Got: {modules_to_not_convert}"
+
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=32)
+ llm = LLM(
+ model=quantized_model_path,
+ quantization="awq",
+ trust_remote_code=True,
+ tensor_parallel_size=1,
+ allow_deprecated_quantization=True,
+ dtype="half",
+ )
+ outputs = llm.generate(["Tell me a short joke."], sampling_params)
+ generated_text = outputs[0].outputs[0].text
+ print(generated_text)
+ assert "!!!" not in generated_text
+ shutil.rmtree("./saved", ignore_errors=True)
+
@require_optimum
@require_awq
@require_package_version_ut("transformers", "<4.57.0")
@@ -363,3 +412,35 @@ class TestAutoRound:
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=5)[0])
print(res)
shutil.rmtree(quantized_model_path, ignore_errors=True)
+
+ def test_autogtq_qwen3_vl_vllm_infer(self, dataloader):
+ pytest.importorskip("vllm")
+ from vllm import LLM, SamplingParams
+ from vllm.platforms import current_platform
+
+ if not current_platform.is_cuda():
+ pytest.skip("only supports CUDA backend for this test")
+
+ model_path = get_model_path("Qwen/Qwen3-VL-2B-Instruct")
+ autoround = AutoRound(
+ model=model_path,
+ scheme="W4A16",
+ iters=0,
+ seqlen=2,
+ batch_size=1,
+ dataset=dataloader,
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_gptq")
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=20)
+ llm = LLM(
+ model=quantized_model_path,
+ trust_remote_code=True,
+ tensor_parallel_size=1,
+ allow_deprecated_quantization=True,
+ )
+ outputs = llm.generate(["Tell me a short joke."], sampling_params)
+ generated_text = outputs[0].outputs[0].text
+ print(generated_text)
+ assert "!!!" not in generated_text
+ shutil.rmtree("./saved", ignore_errors=True)