PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support to save image_processor (#694)
Heng Guo4079a81061a9
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 4ccfae2d3..10d312ab4 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -135,12 +135,15 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllquant_block_list = kwargs.get("quant_block_list", get_block_names(model))tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)+ image_processor = kwargs.get("image_processor", None)if output_dir is not None and os.path.exists(output_dir):logger.warning(f"{output_dir} already exists, this may cause model conflict")if output_dir is not None and tokenizer is not None:tokenizer.save_pretrained(output_dir)if output_dir is not None and processor is not None:processor.save_pretrained(output_dir)+ if output_dir is not None and image_processor is not None:+ image_processor.save_pretrained(output_dir)##check module quantized in block, this may have bug for mixed precision quantizationquantization_config = kwargs["serialization_dict"]all_blocks = quant_block_listdiff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 7fa583848..89d25d9cb 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -289,6 +289,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:extokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)+ image_processor = kwargs.get("image_processor", None)extra_config = {}block_name_to_quantize = quantization_config["block_name_to_quantize"]if isinstance(block_name_to_quantize, str):@@ -354,6 +355,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exif processor is not None:processor.save_pretrained(output_dir)+ if image_processor is not None:+ image_processor.save_pretrained(output_dir)if quantization_config.get("act_bits", 16) <= 8:dtype = torch.bfloat16elif "awq" in quantization_config.get("packing_format", "auto_round:auto_gptq"):diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8_woq.pyindex 6589f243f..e175341e6 100644--- a/auto_round/export/export_to_autoround/export_to_fp8_woq.py+++ b/auto_round/export/export_to_autoround/export_to_fp8_woq.py@@ -174,6 +174,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)+ image_processor = kwargs.get("image_processor", None)extra_config = {}block_name_to_quantize = quantization_config["block_name_to_quantize"]if isinstance(block_name_to_quantize, str):@@ -244,6 +245,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",if processor is not None:processor.save_pretrained(output_dir)+ if image_processor is not None:+ image_processor.save_pretrained(output_dir)if quantization_config.get("act_bits", 16) <= 8:dtype = torch.bfloat16elif "awq" in quantization_config.get("packing_format", "auto_round:auto_gptq"):diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 115d211b7..03a92a922 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -86,6 +86,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):to_quant_block_names = kwargs.get("to_quant_block_names", None)tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)+ image_processor = kwargs.get("image_processor", None)modules_to_not_convert = []if output_dir is not None and os.path.exists(output_dir):@@ -102,6 +103,8 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):all_block_names = all_block_names.split(",")to_quant_block_names = to_quant_block_names.split(",")modules_to_not_convert = list(set(all_block_names) - set(to_quant_block_names))+ if image_processor is not None and output_dir is not None:+ image_processor.save_pretrained(output_dir)if inplace:compressed_model = model.to("cpu")diff --git a/auto_round/mllm/README.md b/auto_round/mllm/README.mdindex 7dfb37c3d..58d56edd2 100644--- a/auto_round/mllm/README.md+++ b/auto_round/mllm/README.md@@ -121,47 +121,40 @@ model architecture or kernel limitations.| mistralai/Mistral-Small-3.1 | pile/llava | X | [Mistral-Small-3.1-24B-Instruct-2503-int4-AutoRound-gptq-sym](https://huggingface.co/OPEA/Mistral-Small-3.1-24B-Instruct-2503-int4-AutoRound-gptq-sym), [Mistral-Small-3.1-24B-Instruct-2503-int4-AutoRound-awq-sym](https://huggingface.co/OPEA/Mistral-Small-3.1-24B-Instruct-2503-int4-AutoRound-awq-sym) || moonshotai/Kimi-VL | pile/llava | √ | || Qwen/Qwen2-VL | pile/llava | - | [Qwen2-VL-7B-Instruct-int4-sym](https://huggingface.co/OPEA/Qwen2-VL-7B-Instruct-int4-sym-inc), [Qwen2-VL-72B-Instruct-int4-sym](https://huggingface.co/OPEA/Qwen2-VL-72B-Instruct-int4-sym-inc), [Qwen2-VL-72B-Instruct-int2-sym](https://huggingface.co/OPEA/Qwen2-VL-72B-Instruct-int2-sym-inc) |+| Qwen/Qwen2.5-VL | pile/llava | √ | || rhymes-ai/Aria | pile/llava | √ | || THUDM/CogVLM2 | pile/llava | √ | [cogvlm2-llama3-chat-19B-int4-sym](https://huggingface.co/OPEA/cogvlm2-llama3-chat-19B-int4-sym-inc), [cogvlm2-llama3-chat-19B-qvision-int4-sym](https://huggingface.co/OPEA/cogvlm2-llama3-chat-19B-qvision-int4-sym-inc) || THUDM/glm-4v | pile | X | [glm-4v-9b-int4-sym](https://huggingface.co/OPEA/glm-4v-9b-int4-sym-inc) |√ means support, - means support to export but cannot infer, X means not support.-### New Models Support+<details>+<summary style="font-size:17px;">Calibration Dataset</summary>-#### Template+For mllm, we used **text-only** calibration dataset (NeelNanda/pile-10k) as our default. If the model type does not+support plain text calibration(e.g. Llama-3.2-vision), it will also automatically switch to llava dataset and adjust the+hyperparameters.-For autoround MLLMs, using Template to customize different operations for different models. User can add a custom chat-template through json file as below.--```json-{- "model_type": "qwen2_vl",- "format_user": "<|im_start|>user\n{{content}}<|im_end|>\n",- "format_assistant": "<|im_start|>assistant\n{{content}}<|im_end|>\n",- "format_system": "<|im_start|>system\n{{content}}<|im_end|>\n",- "format_observation": "<|im_start|>tool\n{{content}}<|im_end|>\n<|im_start|>assistant\n",- "format_separator": "\n",- "default_system": "You are a helpful assistant.",- "replace_tokens": [- "<image>",- "<|vision_start|><|image_pad|><|vision_end|>"- ],- "extra_encode": "True",- "processor": "qwen2_vl"-}-```+Through argument --dataset(text file), user can use other datasets such as "liuhaotian/llava_conv_58k" "+liuhaotian/llava_instruct_80k", "liuhaotian/llava_instruct_150k" or a file path to use local file.-The special token ```{{content}}``` is a placeholder to tell the preprocessor where to fill in the corresponding-dialogue content.+</details>++++<details>+<summary style="font-size:17px;">Nontext Module Quantization</summary>+### New Models Support-```format_*```: Add specific token to chat content depends on different role names.+#### Template-For example, the input conversations:<br>-```[{'role': 'user', 'value': '<image>\nWhat are the colors of the bus in the image?'}, {'role': 'assistant', 'value': 'The bus in the image is white and red.'}]```+For autoround MLLMs, using Template to customize different operations for different models. User can use template to support new model which not in support list.+```python+from auto_round.mllm.template import _register_template-Using the above template, the input will be converted to the specified format required by Qwen2-vl as below: <br>-```'<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>\nWhat are the colors of the bus in the image?<|im_end|>\n<|im_start|>assistant\nThe bus in the image is white and red.<|im_end|>\n<|im_start|>user\nWhat feature can be seen on the back of the bus?<|im_end|>\n<|im_start|>assistant\nThe back of the bus features an advertisement.<|im_end|>\n<|im_start|>user\nIs the bus driving down the street or pulled off to the side?<|im_end|>\n<|im_start|>assistant\nThe bus is driving down the street, which is crowded with people and other vehicles.<|im_end|>\n'```.+model_type = model.config.model_type+_register_template(model_type=model_type, default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])+```#### Processordiff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex 08cd1f1bb..f47b60ecc 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -188,6 +188,7 @@ class AutoRoundMLLM(AutoRound):processor=processor,image_processor=image_processor,use_rtn=iters == 0,+ quiet=not self.quant_nontext_module,)dataset = self.template.default_dataset if dataset is None else datasetdiff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.pyindex 0844c4fea..ec74d64e5 100644--- a/auto_round/mllm/template.py+++ b/auto_round/mllm/template.py@@ -117,6 +117,7 @@ def _register_template(_register_template("qwen2_vl", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["qwen2_vl"])+_register_template("qwen2_5_vl", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["qwen2_vl"])_register_template("mllama", default_dataset="liuhaotian/llava", processor=PROCESSORS["hf"])_register_template("deepseek_vl_v2", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["deepseek_v2"])_register_template("mistral3", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])@@ -157,7 +158,7 @@ _load_preset_template()def get_template(- template_or_path: str, model=None, tokenizer=None, processor=None, image_processor=None, use_rtn=False+ template_or_path: str, model=None, tokenizer=None, processor=None, image_processor=None, use_rtn=False, quiet=False):"""Get template by template name or from a json file.@@ -174,7 +175,8 @@ def get_template(if template_or_path in TEMPLATES:template = TEMPLATES[template_or_path]else:- logger.warning(f"Unable to recognize {template_or_path}, using default template instead.")+ if not quiet:+ logger.warning(f"Unable to recognize {template_or_path}, using default template instead.")template = TEMPLATES["default"]template.model_type = template_or_pathdiff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 161ffe158..c381854a9 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -20,6 +20,7 @@ SUPPORT_ONLY_TEXT_MODELS = ["cogvlm2","llava","qwen2_vl",+ "qwen2_5_vl","deepseek_vl_v2","chatglm","idefics3",diff --git a/test/test_cpu/test_mllm.py b/test/test_cpu/test_mllm.pyindex afeb8a570..cd9fb25b4 100644--- a/test/test_cpu/test_mllm.py+++ b/test/test_cpu/test_mllm.py@@ -216,6 +216,61 @@ class TestAutoRoundMLLM(unittest.TestCase):)print(output_text[0])+ def test_qwen2_5(self):+ from auto_round.utils import mllm_load_model++ model_name = "Qwen/Qwen2.5-VL-3B-Instruct"+ model, processor, tokenizer, image_processor = mllm_load_model(model_name)+ autoround = AutoRoundMLLM(+ model,+ tokenizer,+ iters=1,+ nsamples=1,+ seqlen=32,+ quant_nontext_module=True,+ processor=processor,+ image_processor=image_processor,+ )+ autoround.quantize_and_save("./saved/", format="auto_round")++ import requests+ from PIL import Image+ from transformers import AutoProcessor, AutoTokenizer, Qwen2_5_VLForConditionalGeneration++ model = Qwen2_5_VLForConditionalGeneration.from_pretrained("./saved", torch_dtype="auto", device_map="auto")+ image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"+ processor = AutoProcessor.from_pretrained("./saved")+ messages = [+ {+ "role": "user",+ "content": [+ {+ "type": "image",+ "image": image_url,+ },+ {"type": "text", "text": "Describe this image."},+ ],+ }+ ]++ text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)+ image_inputs = Image.open(requests.get(image_url, stream=True).raw)+ inputs = processor(+ text=[text],+ images=image_inputs,+ padding=True,+ return_tensors="pt",+ )+ inputs = inputs.to(model.device)++ # Inference: Generation of the output+ generated_ids = model.generate(**inputs, max_new_tokens=5)+ generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]+ output_text = processor.batch_decode(+ generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False+ )+ print(output_text)+if __name__ == "__main__":unittest.main()