Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix_quantization_config_for_inference (#542)

WeiweiZhang1765764065887
diff --git a/README.md b/README.md
index 404930ee6..db8ae6906 100644
--- a/README.md
+++ b/README.md
@@ -467,7 +467,7 @@ model architecture or kernel limitations.
|--------------------------------|---------------------|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| allenai/Molmo | pile | X | [Molmo-7B-D-0924-int4-sym](https://huggingface.co/OPEA/Molmo-7B-D-0924-int4-sym-inc), [Molmo-72B-0924-int4-sym-gptq](https://huggingface.co/OPEA/Molmo-72B-0924-int4-sym-gptq-inc), [Molmo-72B-0924-int4-sym](https://huggingface.co/OPEA/Molmo-72B-0924-int4-sym-inc) |
| deepseek-ai/deepseek-vl2 | pile/llava | √ | [deepseek-vl2-int4-sym-gptq](https://huggingface.co/OPEA/deepseek-vl2-int4-sym-gptq-inc) |
-| google/gemma-3 | pile/llava | √ | [gemma-3-12b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-12b-it-AutoRound-gguf-q4-0), [gemma-3-27b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-27b-it-AutoRound-gguf-q4-0), [gemma-3-12b-it-int4-AutoRound-cpu](https://huggingface.co/OPEA/gemma-3-12b-it-int4-AutoRound-cpu), [gemma-3-27b-it-int4-AutoRound-cpu](https://huggingface.co/OPEA/gemma-3-27b-it-int4-AutoRound-cpu) |
+| google/gemma-3 | pile/llava | √ | [gemma-3-12b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-12b-it-AutoRound-gguf-q4-0), [gemma-3-27b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-27b-it-AutoRound-gguf-q4-0), [gemma-3-12b-it-int4-AutoRound](https://huggingface.co/OPEA/gemma-3-12b-it-int4-AutoRound), [gemma-3-27b-it-int4-AutoRound](https://huggingface.co/OPEA/gemma-3-27b-it-int4-AutoRound) |
| HuggingFaceTB/SmolVLM | pile/llava | √ | [SmolVLM-Instruct-int4-sym](https://huggingface.co/OPEA/SmolVLM-Instruct-int4-sym-inc) |
| ibm-granite/granite-vision-3.2 | pile/llava | - | |
| liuhaotian/Llava-v1.5 | pile/llava | X | [llava-v1.5-7b-int4-sym](https://huggingface.co/OPEA/llava-v1.5-7b-int4-sym-inc) |
@@ -515,3 +515,4 @@ If you find AutoRound useful for your research, please cite our paper:
+
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index beffe62a4..bf48eb05d 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -260,6 +260,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
layer_config = kwargs["layer_config"]
quantization_config = kwargs["serialization_dict"]
+ quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)
quantization_config["quant_method"] = "auto-round"
if quantization_config["bits"] == 3:
backend = "auto_round:auto_gptq"
@@ -363,3 +364,4 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
+
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 205dd357c..85e4c06cf 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -227,7 +227,9 @@ def get_layer_config(model, quantization_config):
# Determine the quantization block list
quant_block_list = getattr(quantization_config, "quant_block_list", None)
if quant_block_list is None:
- to_quant_block_names = getattr(quantization_config, "to_quant_block_names", None)
+ to_quant_block_names = getattr(quantization_config, "block_name_to_quantize", None) # Prioritize this parameter
+ if to_quant_block_names is None:
+ to_quant_block_names = getattr(quantization_config, "to_quant_block_names", None)
if isinstance(to_quant_block_names, (list, tuple)):
quant_block_list = to_quant_block_names
elif isinstance(to_quant_block_names, str):
@@ -564,3 +566,4 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):
logger.warning(extra_info)
return model, used_backends
+
diff --git a/auto_round/mllm/README.md b/auto_round/mllm/README.md
index 2f961df1c..58e511a34 100644
--- a/auto_round/mllm/README.md
+++ b/auto_round/mllm/README.md
@@ -106,7 +106,7 @@ model architecture or kernel limitations.
|--------------------------------|---------------------|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| allenai/Molmo | pile | X | [Molmo-7B-D-0924-int4-sym](https://huggingface.co/OPEA/Molmo-7B-D-0924-int4-sym-inc), [Molmo-72B-0924-int4-sym-gptq](https://huggingface.co/OPEA/Molmo-72B-0924-int4-sym-gptq-inc), [Molmo-72B-0924-int4-sym](https://huggingface.co/OPEA/Molmo-72B-0924-int4-sym-inc) |
| deepseek-ai/deepseek-vl2 | pile/llava | √ | [deepseek-vl2-int4-sym-gptq](https://huggingface.co/OPEA/deepseek-vl2-int4-sym-gptq-inc) |
-| google/gemma-3 | pile/llava | √ | [gemma-3-12b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-12b-it-AutoRound-gguf-q4-0), [gemma-3-27b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-27b-it-AutoRound-gguf-q4-0), [gemma-3-12b-it-int4-AutoRound-cpu](https://huggingface.co/OPEA/gemma-3-12b-it-int4-AutoRound-cpu), [gemma-3-27b-it-int4-AutoRound-cpu](https://huggingface.co/OPEA/gemma-3-27b-it-int4-AutoRound-cpu) |
+| google/gemma-3 | pile/llava | √ | [gemma-3-12b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-12b-it-AutoRound-gguf-q4-0), [gemma-3-27b-it-AutoRound-gguf-q4-0](https://huggingface.co/OPEA/gemma-3-27b-it-AutoRound-gguf-q4-0), [gemma-3-12b-it-int4-AutoRound](https://huggingface.co/OPEA/gemma-3-12b-it-int4-AutoRound), [gemma-3-27b-it-int4-AutoRound](https://huggingface.co/OPEA/gemma-3-27b-it-int4-AutoRound) |
| HuggingFaceTB/SmolVLM | pile/llava | √ | [SmolVLM-Instruct-int4-sym](https://huggingface.co/OPEA/SmolVLM-Instruct-int4-sym-inc) |
| ibm-granite/granite-vision-3.2 | pile/llava | - | |
| liuhaotian/Llava-v1.5 | pile/llava | X | [llava-v1.5-7b-int4-sym](https://huggingface.co/OPEA/llava-v1.5-7b-int4-sym-inc) |
@@ -185,3 +185,4 @@ For more details on quantization, inference, evaluation, and environment, see th
+
diff --git a/test/test_block_names.py b/test/test_block_names.py
index a24e45e82..8e86e62f2 100644
--- a/test/test_block_names.py
+++ b/test/test_block_names.py
@@ -171,7 +171,7 @@ class TestQuantizationBlocks(unittest.TestCase):
inputs = tokenizer(text, return_tensors="pt").to(model.device)
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
quant_config = model.config.quantization_config
- assert quant_config.to_quant_block_names is not None
+ assert quant_config.block_name_to_quantize is not None
shutil.rmtree("./saved", ignore_errors=True)
@@ -228,3 +228,4 @@ if __name__ == "__main__":
+