PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix_quantization_config_for_inference (#542)
WeiweiZhang1765764065887
diff --git a/README.md b/README.mdindex 404930ee6..db8ae6906 100644--- a/README.md+++ b/README.md@@ -467,7 +467,7 @@ model architecture or kernel limitations.|--------------------------------|---------------------|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|| allenai/Molmo | pile | X | [Molmo-7B-D-0924-int4-sym](https://huggingface.co/OPEA/Molmo-7B-D-0924-int4-sym-inc), [Molmo-72B-0924-int4-sym-gptq](https://huggingface.co/OPEA/Molmo-72B-0924-int4-sym-gptq-inc), [Molmo-72B-0924-int4-sym](https://huggingface.co/OPEA/Molmo-72B-0924-int4-sym-inc) || deepseek-ai/deepseek-vl2 | pile/llava | √ | [deepseek-vl2-int4-sym-gptq](https://huggingface.co/OPEA/deepseek-vl2-int4-sym-gptq-inc) |-| google/gemma-3 | pile/llava | √ | [gemma-3-12b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-12b-it-AutoRound-gguf-q4-0), [gemma-3-27b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-27b-it-AutoRound-gguf-q4-0), [gemma-3-12b-it-int4-AutoRound-cpu](https://huggingface.co/OPEA/gemma-3-12b-it-int4-AutoRound-cpu), [gemma-3-27b-it-int4-AutoRound-cpu](https://huggingface.co/OPEA/gemma-3-27b-it-int4-AutoRound-cpu) |+| google/gemma-3 | pile/llava | √ | [gemma-3-12b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-12b-it-AutoRound-gguf-q4-0), [gemma-3-27b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-27b-it-AutoRound-gguf-q4-0), [gemma-3-12b-it-int4-AutoRound](https://huggingface.co/OPEA/gemma-3-12b-it-int4-AutoRound), [gemma-3-27b-it-int4-AutoRound](https://huggingface.co/OPEA/gemma-3-27b-it-int4-AutoRound) || HuggingFaceTB/SmolVLM | pile/llava | √ | [SmolVLM-Instruct-int4-sym](https://huggingface.co/OPEA/SmolVLM-Instruct-int4-sym-inc) || ibm-granite/granite-vision-3.2 | pile/llava | - | || liuhaotian/Llava-v1.5 | pile/llava | X | [llava-v1.5-7b-int4-sym](https://huggingface.co/OPEA/llava-v1.5-7b-int4-sym-inc) |@@ -515,3 +515,4 @@ If you find AutoRound useful for your research, please cite our paper:+diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex beffe62a4..bf48eb05d 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -260,6 +260,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exlayer_config = kwargs["layer_config"]quantization_config = kwargs["serialization_dict"]+ quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)quantization_config["quant_method"] = "auto-round"if quantization_config["bits"] == 3:backend = "auto_round:auto_gptq"@@ -363,3 +364,4 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seriif hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)+diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 205dd357c..85e4c06cf 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -227,7 +227,9 @@ def get_layer_config(model, quantization_config):# Determine the quantization block listquant_block_list = getattr(quantization_config, "quant_block_list", None)if quant_block_list is None:- to_quant_block_names = getattr(quantization_config, "to_quant_block_names", None)+ to_quant_block_names = getattr(quantization_config, "block_name_to_quantize", None) # Prioritize this parameter+ if to_quant_block_names is None:+ to_quant_block_names = getattr(quantization_config, "to_quant_block_names", None)if isinstance(to_quant_block_names, (list, tuple)):quant_block_list = to_quant_block_nameselif isinstance(to_quant_block_names, str):@@ -564,3 +566,4 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):logger.warning(extra_info)return model, used_backends+diff --git a/auto_round/mllm/README.md b/auto_round/mllm/README.mdindex 2f961df1c..58e511a34 100644--- a/auto_round/mllm/README.md+++ b/auto_round/mllm/README.md@@ -106,7 +106,7 @@ model architecture or kernel limitations.|--------------------------------|---------------------|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|| allenai/Molmo | pile | X | [Molmo-7B-D-0924-int4-sym](https://huggingface.co/OPEA/Molmo-7B-D-0924-int4-sym-inc), [Molmo-72B-0924-int4-sym-gptq](https://huggingface.co/OPEA/Molmo-72B-0924-int4-sym-gptq-inc), [Molmo-72B-0924-int4-sym](https://huggingface.co/OPEA/Molmo-72B-0924-int4-sym-inc) || deepseek-ai/deepseek-vl2 | pile/llava | √ | [deepseek-vl2-int4-sym-gptq](https://huggingface.co/OPEA/deepseek-vl2-int4-sym-gptq-inc) |-| google/gemma-3 | pile/llava | √ | [gemma-3-12b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-12b-it-AutoRound-gguf-q4-0), [gemma-3-27b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-27b-it-AutoRound-gguf-q4-0), [gemma-3-12b-it-int4-AutoRound-cpu](https://huggingface.co/OPEA/gemma-3-12b-it-int4-AutoRound-cpu), [gemma-3-27b-it-int4-AutoRound-cpu](https://huggingface.co/OPEA/gemma-3-27b-it-int4-AutoRound-cpu) |+| google/gemma-3 | pile/llava | √ | [gemma-3-12b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-12b-it-AutoRound-gguf-q4-0), [gemma-3-27b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-27b-it-AutoRound-gguf-q4-0), [gemma-3-12b-it-int4-AutoRound](https://huggingface.co/OPEA/gemma-3-12b-it-int4-AutoRound), [gemma-3-27b-it-int4-AutoRound](https://huggingface.co/OPEA/gemma-3-27b-it-int4-AutoRound) || HuggingFaceTB/SmolVLM | pile/llava | √ | [SmolVLM-Instruct-int4-sym](https://huggingface.co/OPEA/SmolVLM-Instruct-int4-sym-inc) || ibm-granite/granite-vision-3.2 | pile/llava | - | || liuhaotian/Llava-v1.5 | pile/llava | X | [llava-v1.5-7b-int4-sym](https://huggingface.co/OPEA/llava-v1.5-7b-int4-sym-inc) |@@ -185,3 +185,4 @@ For more details on quantization, inference, evaluation, and environment, see th+diff --git a/test/test_block_names.py b/test/test_block_names.pyindex a24e45e82..8e86e62f2 100644--- a/test/test_block_names.py+++ b/test/test_block_names.py@@ -171,7 +171,7 @@ class TestQuantizationBlocks(unittest.TestCase):inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))quant_config = model.config.quantization_config- assert quant_config.to_quant_block_names is not None+ assert quant_config.block_name_to_quantize is not Noneshutil.rmtree("./saved", ignore_errors=True)@@ -228,3 +228,4 @@ if __name__ == "__main__":+