Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support to save image_processor (#694)

Heng Guo4079a81061a9
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index 4ccfae2d3..10d312ab4 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -135,12 +135,15 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
quant_block_list = kwargs.get("quant_block_list", get_block_names(model))
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
+ image_processor = kwargs.get("image_processor", None)
if output_dir is not None and os.path.exists(output_dir):
logger.warning(f"{output_dir} already exists, this may cause model conflict")
if output_dir is not None and tokenizer is not None:
tokenizer.save_pretrained(output_dir)
if output_dir is not None and processor is not None:
processor.save_pretrained(output_dir)
+ if output_dir is not None and image_processor is not None:
+ image_processor.save_pretrained(output_dir)
##check module quantized in block, this may have bug for mixed precision quantization
quantization_config = kwargs["serialization_dict"]
all_blocks = quant_block_list
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 7fa583848..89d25d9cb 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -289,6 +289,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
+ image_processor = kwargs.get("image_processor", None)
extra_config = {}
block_name_to_quantize = quantization_config["block_name_to_quantize"]
if isinstance(block_name_to_quantize, str):
@@ -354,6 +355,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
if processor is not None:
processor.save_pretrained(output_dir)
+ if image_processor is not None:
+ image_processor.save_pretrained(output_dir)
if quantization_config.get("act_bits", 16) <= 8:
dtype = torch.bfloat16
elif "awq" in quantization_config.get("packing_format", "auto_round:auto_gptq"):
diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8_woq.py
index 6589f243f..e175341e6 100644
--- a/auto_round/export/export_to_autoround/export_to_fp8_woq.py
+++ b/auto_round/export/export_to_autoround/export_to_fp8_woq.py
@@ -174,6 +174,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
+ image_processor = kwargs.get("image_processor", None)
extra_config = {}
block_name_to_quantize = quantization_config["block_name_to_quantize"]
if isinstance(block_name_to_quantize, str):
@@ -244,6 +245,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",
if processor is not None:
processor.save_pretrained(output_dir)
+ if image_processor is not None:
+ image_processor.save_pretrained(output_dir)
if quantization_config.get("act_bits", 16) <= 8:
dtype = torch.bfloat16
elif "awq" in quantization_config.get("packing_format", "auto_round:auto_gptq"):
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index 115d211b7..03a92a922 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -86,6 +86,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
to_quant_block_names = kwargs.get("to_quant_block_names", None)
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
+ image_processor = kwargs.get("image_processor", None)
modules_to_not_convert = []
if output_dir is not None and os.path.exists(output_dir):
@@ -102,6 +103,8 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
all_block_names = all_block_names.split(",")
to_quant_block_names = to_quant_block_names.split(",")
modules_to_not_convert = list(set(all_block_names) - set(to_quant_block_names))
+ if image_processor is not None and output_dir is not None:
+ image_processor.save_pretrained(output_dir)
if inplace:
compressed_model = model.to("cpu")
diff --git a/auto_round/mllm/README.md b/auto_round/mllm/README.md
index 7dfb37c3d..58d56edd2 100644
--- a/auto_round/mllm/README.md
+++ b/auto_round/mllm/README.md
@@ -121,47 +121,40 @@ model architecture or kernel limitations.
| mistralai/Mistral-Small-3.1 | pile/llava | X | [Mistral-Small-3.1-24B-Instruct-2503-int4-AutoRound-gptq-sym](https://huggingface.co/OPEA/Mistral-Small-3.1-24B-Instruct-2503-int4-AutoRound-gptq-sym), [Mistral-Small-3.1-24B-Instruct-2503-int4-AutoRound-awq-sym](https://huggingface.co/OPEA/Mistral-Small-3.1-24B-Instruct-2503-int4-AutoRound-awq-sym) |
| moonshotai/Kimi-VL | pile/llava | √ | |
| Qwen/Qwen2-VL | pile/llava | - | [Qwen2-VL-7B-Instruct-int4-sym](https://huggingface.co/OPEA/Qwen2-VL-7B-Instruct-int4-sym-inc), [Qwen2-VL-72B-Instruct-int4-sym](https://huggingface.co/OPEA/Qwen2-VL-72B-Instruct-int4-sym-inc), [Qwen2-VL-72B-Instruct-int2-sym](https://huggingface.co/OPEA/Qwen2-VL-72B-Instruct-int2-sym-inc) |
+| Qwen/Qwen2.5-VL | pile/llava | √ | |
| rhymes-ai/Aria | pile/llava | √ | |
| THUDM/CogVLM2 | pile/llava | √ | [cogvlm2-llama3-chat-19B-int4-sym](https://huggingface.co/OPEA/cogvlm2-llama3-chat-19B-int4-sym-inc), [cogvlm2-llama3-chat-19B-qvision-int4-sym](https://huggingface.co/OPEA/cogvlm2-llama3-chat-19B-qvision-int4-sym-inc) |
| THUDM/glm-4v | pile | X | [glm-4v-9b-int4-sym](https://huggingface.co/OPEA/glm-4v-9b-int4-sym-inc) |
√ means support, - means support to export but cannot infer, X means not support.
-### New Models Support
+<details>
+<summary style="font-size:17px;">Calibration Dataset</summary>
-#### Template
+For mllm, we used **text-only** calibration dataset (NeelNanda/pile-10k) as our default. If the model type does not
+support plain text calibration(e.g. Llama-3.2-vision), it will also automatically switch to llava dataset and adjust the
+hyperparameters.
-For autoround MLLMs, using Template to customize different operations for different models. User can add a custom chat
-template through json file as below.
-
-```json
-{
- "model_type": "qwen2_vl",
- "format_user": "<|im_start|>user\n{{content}}<|im_end|>\n",
- "format_assistant": "<|im_start|>assistant\n{{content}}<|im_end|>\n",
- "format_system": "<|im_start|>system\n{{content}}<|im_end|>\n",
- "format_observation": "<|im_start|>tool\n{{content}}<|im_end|>\n<|im_start|>assistant\n",
- "format_separator": "\n",
- "default_system": "You are a helpful assistant.",
- "replace_tokens": [
- "<image>",
- "<|vision_start|><|image_pad|><|vision_end|>"
- ],
- "extra_encode": "True",
- "processor": "qwen2_vl"
-}
-```
+Through argument --dataset(text file), user can use other datasets such as "liuhaotian/llava_conv_58k" "
+liuhaotian/llava_instruct_80k", "liuhaotian/llava_instruct_150k" or a file path to use local file.
-The special token ```{{content}}``` is a placeholder to tell the preprocessor where to fill in the corresponding
-dialogue content.
+</details>
+
+
+
+<details>
+<summary style="font-size:17px;">Nontext Module Quantization</summary>
+### New Models Support
-```format_*```: Add specific token to chat content depends on different role names.
+#### Template
-For example, the input conversations:<br>
-```[{'role': 'user', 'value': '<image>\nWhat are the colors of the bus in the image?'}, {'role': 'assistant', 'value': 'The bus in the image is white and red.'}]```
+For autoround MLLMs, using Template to customize different operations for different models. User can use template to support new model which not in support list.
+```python
+from auto_round.mllm.template import _register_template
-Using the above template, the input will be converted to the specified format required by Qwen2-vl as below: <br>
-```'<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>\nWhat are the colors of the bus in the image?<|im_end|>\n<|im_start|>assistant\nThe bus in the image is white and red.<|im_end|>\n<|im_start|>user\nWhat feature can be seen on the back of the bus?<|im_end|>\n<|im_start|>assistant\nThe back of the bus features an advertisement.<|im_end|>\n<|im_start|>user\nIs the bus driving down the street or pulled off to the side?<|im_end|>\n<|im_start|>assistant\nThe bus is driving down the street, which is crowded with people and other vehicles.<|im_end|>\n'```.
+model_type = model.config.model_type
+_register_template(model_type=model_type, default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])
+```
#### Processor
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index 08cd1f1bb..f47b60ecc 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -188,6 +188,7 @@ class AutoRoundMLLM(AutoRound):
processor=processor,
image_processor=image_processor,
use_rtn=iters == 0,
+ quiet=not self.quant_nontext_module,
)
dataset = self.template.default_dataset if dataset is None else dataset
diff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.py
index 0844c4fea..ec74d64e5 100644
--- a/auto_round/mllm/template.py
+++ b/auto_round/mllm/template.py
@@ -117,6 +117,7 @@ def _register_template(
_register_template("qwen2_vl", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["qwen2_vl"])
+_register_template("qwen2_5_vl", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["qwen2_vl"])
_register_template("mllama", default_dataset="liuhaotian/llava", processor=PROCESSORS["hf"])
_register_template("deepseek_vl_v2", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["deepseek_v2"])
_register_template("mistral3", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])
@@ -157,7 +158,7 @@ _load_preset_template()
def get_template(
- template_or_path: str, model=None, tokenizer=None, processor=None, image_processor=None, use_rtn=False
+ template_or_path: str, model=None, tokenizer=None, processor=None, image_processor=None, use_rtn=False, quiet=False
):
"""Get template by template name or from a json file.
@@ -174,7 +175,8 @@ def get_template(
if template_or_path in TEMPLATES:
template = TEMPLATES[template_or_path]
else:
- logger.warning(f"Unable to recognize {template_or_path}, using default template instead.")
+ if not quiet:
+ logger.warning(f"Unable to recognize {template_or_path}, using default template instead.")
template = TEMPLATES["default"]
template.model_type = template_or_path
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 161ffe158..c381854a9 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -20,6 +20,7 @@ SUPPORT_ONLY_TEXT_MODELS = [
"cogvlm2",
"llava",
"qwen2_vl",
+ "qwen2_5_vl",
"deepseek_vl_v2",
"chatglm",
"idefics3",
diff --git a/test/test_cpu/test_mllm.py b/test/test_cpu/test_mllm.py
index afeb8a570..cd9fb25b4 100644
--- a/test/test_cpu/test_mllm.py
+++ b/test/test_cpu/test_mllm.py
@@ -216,6 +216,61 @@ class TestAutoRoundMLLM(unittest.TestCase):
)
print(output_text[0])
+ def test_qwen2_5(self):
+ from auto_round.utils import mllm_load_model
+
+ model_name = "Qwen/Qwen2.5-VL-3B-Instruct"
+ model, processor, tokenizer, image_processor = mllm_load_model(model_name)
+ autoround = AutoRoundMLLM(
+ model,
+ tokenizer,
+ iters=1,
+ nsamples=1,
+ seqlen=32,
+ quant_nontext_module=True,
+ processor=processor,
+ image_processor=image_processor,
+ )
+ autoround.quantize_and_save("./saved/", format="auto_round")
+
+ import requests
+ from PIL import Image
+ from transformers import AutoProcessor, AutoTokenizer, Qwen2_5_VLForConditionalGeneration
+
+ model = Qwen2_5_VLForConditionalGeneration.from_pretrained("./saved", torch_dtype="auto", device_map="auto")
+ image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
+ processor = AutoProcessor.from_pretrained("./saved")
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "image",
+ "image": image_url,
+ },
+ {"type": "text", "text": "Describe this image."},
+ ],
+ }
+ ]
+
+ text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+ image_inputs = Image.open(requests.get(image_url, stream=True).raw)
+ inputs = processor(
+ text=[text],
+ images=image_inputs,
+ padding=True,
+ return_tensors="pt",
+ )
+ inputs = inputs.to(model.device)
+
+ # Inference: Generation of the output
+ generated_ids = model.generate(**inputs, max_new_tokens=5)
+ generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]
+ output_text = processor.batch_decode(
+ generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+ )
+ print(output_text)
+
if __name__ == "__main__":
unittest.main()