PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
qwen2_bugfix, add adamround vision UT (#281)
WeiweiZhang17cfff9670b18
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 3bde5b6f1..256aba477 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -720,8 +720,8 @@ class AutoRound(object):self.inputs[name][key].extend(list(torch.split(alibi.to("cpu"), 1, dim=0)))else:self.inputs[name][key] = list(torch.split(alibi.to("cpu"), 1, dim=0))- elif "position_ids" in key or 'cache_position' in key:- if self.train_bs == 1 and self.not_share_rotary_pos_emb_flag:+ elif "position_ids" in key or 'cache_position' in key or 'position_embeddings' in key:+ if self.train_bs == 1 and self.not_share_position_ids_flag:if key not in self.inputs[name].keys():self.inputs[name][key] = [to_device(kwargs[key], device=torch.device("cpu"))]else:@@ -1104,7 +1104,7 @@ class AutoRound(object):input_others[key] = input_others[key].to(tmp_dtype)elif isinstance(input_others[key], list):for i in range(len(input_others[key])):- input_others[key][i].to(tmp_dtype)+ to_dtype(input_others[key][i], tmp_dtype)pbar = tqdm(range(0, len(block_names), nblocks))for i in pbar:if nblocks == 1:@@ -1621,3 +1621,4 @@ class AutoAdamRound(AutoOPTRound):)+diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 7f8a68007..e39ffc695 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -113,13 +113,16 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll"""Export the model to autogptq format to easily leverage cuda kernel."""model = kwargs["model"]- tokenizer = kwargs["tokenizer"]supported_types = kwargs["supported_types"]safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]quant_block_list = kwargs["quant_block_list"]logger.info("Saving quantized model to autogptq format, this may take a while...")+ tokenizer = kwargs.get("tokenizer", None)+ processor = kwargs.get("processor", None)if tokenizer is not None:tokenizer.save_pretrained(output_dir)+ if processor is not None:+ processor.save_pretrained(output_dir)##check module quantized in block, this may have bug for mixed precision quantizationif bool(quant_block_list):all_blocks = quant_block_list@@ -200,3 +203,4 @@ def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safif hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)+diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 0ab2f505e..e9cace5b2 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -198,6 +198,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exlayer_config = kwargs["layer_config"]quantization_config = kwargs["serialization_dict"]quantization_config["quant_method"] = "intel/auto-round"+ tokenizer = kwargs.get("tokenizer", None)+ processor = kwargs.get("processor", None)if "awq" not in backend:quantization_config["backend"] = backendextra_config = {}@@ -235,12 +237,14 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exmodel.config.quantization_config = quantization_configif output_dir is None:return model- tokenizer = kwargs["tokenizer"]+if output_dir is None:model.tokenizer = tokenizerreturn modelif tokenizer is not None:tokenizer.save_pretrained(output_dir)+ if processor is not None:+ processor.save_pretrained(output_dir)modules_to_not_convert = []if "awq" not in backend:save(model, output_dir, safe_serialization=safe_serialization)@@ -317,3 +321,4 @@ def save_awq(if hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(quantization_config, f, indent=2)+diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 83f6824ea..d191f3442 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -91,11 +91,14 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):enable_minmax_tuning = kwargs["enable_minmax_tuning"]enable_quanted_input = kwargs["enable_quanted_input"]scale_dtype = kwargs["scale_dtype"]- tokenizer = kwargs["tokenizer"]+ tokenizer = kwargs.get("tokenizer", None)+ processor = kwargs.get("processor", None)logger.info("Saving quantized model to auto_awq format")if tokenizer is not None:tokenizer.save_pretrained(output_dir)+ if processor is not None:+ processor.save_pretrained(output_dir)##check module quantized in block, this may have bug for mixed precision quantizationmodules_to_not_convert = []if inplace:@@ -250,3 +253,4 @@ def get_module_name(model, module_to_find):if module is module_to_find:return namereturn None+diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.pyindex f80d3076a..950d9d8e0 100644--- a/auto_round/export/export_to_itrex/export.py+++ b/auto_round/export/export_to_itrex/export.py@@ -121,7 +121,8 @@ def save_quantized_as_itrex_xpu(output_dir, inplace=True, **kwargs):enable_minmax_tuning = kwargs["enable_minmax_tuning"]enable_quanted_input = kwargs["enable_quanted_input"]scale_dtype = kwargs["scale_dtype"]- tokenizer = kwargs["tokenizer"]+ tokenizer = kwargs.get("tokenizer", None)+ processor = kwargs.get("processor", None)compressed_model = pack_model(inplace=inplace, **kwargs)if output_dir is None:@@ -149,6 +150,8 @@ def save_quantized_as_itrex_xpu(output_dir, inplace=True, **kwargs):compressed_model.save_pretrained(output_dir, safe_serialization=True)if tokenizer is not None:tokenizer.save_pretrained(output_dir)+ if processor is not None:+ processor.save_pretrained(output_dir)logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))except IOError as e: # pragma: no coverlogger.error("Fail to save configure file and weights due to {}.".format(e))@@ -252,3 +255,4 @@ def pack_model(return compressed_model+diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 10beff712..a3913ce18 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -366,14 +366,18 @@ def sampling_inputs(input_ids, input_others, indices, seqlen,current_input_others = {"positional_inputs": input_others["positional_inputs"]}for key in input_others.keys():if not share_attention_mask_flag and ("attention_mask" in key or "alibi" in key) \- or (not_share_position_ids_flag and ("position_ids" in key or "cache_position" in key)) \+ or (not_share_position_ids_flag and ("position_ids" in key or \+ "cache_position" in key or "position_embeddings" in key)) \or (not_share_rotary_pos_emb_flag and ("rotary_pos_emb" in key or 'cu_seqlens' in key)) \or "cross_attention_states" in key:current_input_others[key] = Noneif input_others[key] is not None:current_input_others[key] = [input_others[key][i] for i in indices]if not isinstance(current_input_others[key], torch.Tensor):- current_input_others[key] = torch.cat(current_input_others[key], dim=0)+ if len(current_input_others[key]) == 1:+ current_input_others[key] = current_input_others[key][0]+ else:+ current_input_others[key] = torch.cat(current_input_others[key], dim=0)else:current_input_others[key] = input_others[key]@@ -973,3 +977,4 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):return QuantLinear+diff --git a/examples/multimodal-modeling/Qwen-VL/README.md b/examples/multimodal-modeling/Common_model/README.mdsimilarity index 88%rename from examples/multimodal-modeling/Qwen-VL/README.mdrename to examples/multimodal-modeling/Common_model/README.mdindex d49b2c2ba..6cec35952 100644--- a/examples/multimodal-modeling/Qwen-VL/README.md+++ b/examples/multimodal-modeling/Common_model/README.md@@ -193,6 +193,37 @@ print(output_text)```+- Llama-3.2-11B-Vision-Instruct inference++```python+import requests+import torch+from PIL import Image+from transformers import MllamaForConditionalGeneration, AutoProcessor+from auto_round.auto_quantizer import AutoHfQuantizer+quantized_model_path="./tmp_autoround"+model = MllamaForConditionalGeneration.from_pretrained(+ quantized_model_path, torch_dtype=torch.bfloat16, device_map="auto", trust_remote_code=True)+processor = AutoProcessor.from_pretrained(quantized_model_path)++url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg"+image = Image.open(requests.get(url, stream=True).raw)++prompt = "<|image|><|begin_of_text|>If I had to write a haiku for this one"+inputs = processor(image, prompt, return_tensors="pt", truncation=True).to(model.device)++output = model.generate(**inputs, max_new_tokens=30)+print(processor.decode(output[0]))++# <|begin_of_text|><|image|><|begin_of_text|>If I had to write a haiku for this one, it would be:++# Rabbit in a coat+# Dressed up in style for the day+# Country charm abounds++# The image depicts a rabbit+```+## 4. ResultsUsing [COCO 2017](https://cocodataset.org/) and [LLaVA-Instruct-150K](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K) datasets for quantization calibration, and TextVQA dataset for evaluation. please follow the [recipe](./run_autoround.sh) and [evaluate script](./run_eval.sh). The results for Qwen-VL are as follows:| Metric | bf16 | INT4 |@@ -241,3 +272,4 @@ If you find SignRound useful for your research, please cite our paper:+diff --git a/examples/multimodal-modeling/Qwen-VL/eval_042/__init__.py b/examples/multimodal-modeling/Common_model/eval_042/__init__.pysimilarity index 100%rename from examples/multimodal-modeling/Qwen-VL/eval_042/__init__.pyrename to examples/multimodal-modeling/Common_model/eval_042/__init__.pydiff --git a/examples/multimodal-modeling/Qwen-VL/eval_042/evaluation.py b/examples/multimodal-modeling/Common_model/eval_042/evaluation.pysimilarity index 100%rename from examples/multimodal-modeling/Qwen-VL/eval_042/evaluation.pyrename to examples/multimodal-modeling/Common_model/eval_042/evaluation.pydiff --git a/examples/multimodal-modeling/Qwen-VL/main.py b/examples/multimodal-modeling/Common_model/main.pysimilarity index 98%rename from examples/multimodal-modeling/Qwen-VL/main.pyrename to examples/multimodal-modeling/Common_model/main.pyindex b6d0f7ac3..7ac7194a4 100644--- a/examples/multimodal-modeling/Qwen-VL/main.py+++ b/examples/multimodal-modeling/Common_model/main.py@@ -34,7 +34,7 @@ def DataFormating(raw_data, image_folder=None, model_type='qwen'):sentence['value'] = sentence['value'].strip()if 'qwen2' in model_type: # for Qwen2-vlreplace_token = '<|vision_start|><|image_pad|><|vision_end|>'- if 'mllama' in model_type:+ elif 'mllama' in model_type:replace_token = '<|image|>'else:replace_img = os.path.join(image_folder, os.path.basename(source["image"]))@@ -422,7 +422,7 @@ if __name__ == '__main__':model_type = config.model_typeif "mllama" in model_type:from transformers import MllamaForConditionalGeneration- model = MllamaForConditionalGeneration.from_pretrained(args.model_name,+ model = MllamaForConditionalGeneration.from_pretrained(args.model_name, attn_implementation="eager",trust_remote_code=not args.disable_trust_remote_code) # torch_dtype=torch.bfloat16processor = AutoProcessor.from_pretrained(args.model_name)tokenizer.processor = processor@@ -534,17 +534,17 @@ if __name__ == '__main__':for gpu_format in gpu_formats:if "round" in gpu_format:eval_folder = f'{export_dir}-round'- autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)+ autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace, processor=processor)elif "gptq" in gpu_format:eval_folder = f'{export_dir}-gpu'- autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)+ autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace, processor=processor)if 'xpu' in deployment_device:autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,compression_dtype=torch.int8, compression_dim=0, use_optimum_format=False,- device="xpu")+ device="xpu", processor=processor)if "cpu" in deployment_device:- autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace)+ autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace, processor=processor)if "fake" in deployment_device:model = model.to("cpu")model.save_pretrained(output_dir)@@ -580,3 +580,4 @@ if __name__ == '__main__':+diff --git a/examples/multimodal-modeling/Qwen-VL/mm_evaluation/__init__.py b/examples/multimodal-modeling/Common_model/mm_evaluation/__init__.pysimilarity index 100%rename from examples/multimodal-modeling/Qwen-VL/mm_evaluation/__init__.pyrename to examples/multimodal-modeling/Common_model/mm_evaluation/__init__.pydiff --git a/examples/multimodal-modeling/Qwen-VL/mm_evaluation/evaluate_multiple_choice.py b/examples/multimodal-modeling/Common_model/mm_evaluation/evaluate_multiple_choice.pysimilarity index 100%rename from examples/multimodal-modeling/Qwen-VL/mm_evaluation/evaluate_multiple_choice.pyrename to examples/multimodal-modeling/Common_model/mm_evaluation/evaluate_multiple_choice.pydiff --git a/examples/multimodal-modeling/Qwen-VL/mm_evaluation/evaluate_vqa.py b/examples/multimodal-modeling/Common_model/mm_evaluation/evaluate_vqa.pysimilarity index 100%rename from examples/multimodal-modeling/Qwen-VL/mm_evaluation/evaluate_vqa.pyrename to examples/multimodal-modeling/Common_model/mm_evaluation/evaluate_vqa.pydiff --git a/examples/multimodal-modeling/Qwen-VL/mm_evaluation/main.py b/examples/multimodal-modeling/Common_model/mm_evaluation/main.pysimilarity index 100%rename from examples/multimodal-modeling/Qwen-VL/mm_evaluation/main.pyrename to examples/multimodal-modeling/Common_model/mm_evaluation/main.pydiff --git a/examples/multimodal-modeling/Qwen-VL/mm_evaluation/vqa.py b/examples/multimodal-modeling/Common_model/mm_evaluation/vqa.pysimilarity index 100%rename from examples/multimodal-modeling/Qwen-VL/mm_evaluation/vqa.pyrename to examples/multimodal-modeling/Common_model/mm_evaluation/vqa.pydiff --git a/examples/multimodal-modeling/Qwen-VL/mm_evaluation/vqa_eval.py b/examples/multimodal-modeling/Common_model/mm_evaluation/vqa_eval.pysimilarity index 100%rename from examples/multimodal-modeling/Qwen-VL/mm_evaluation/vqa_eval.pyrename to examples/multimodal-modeling/Common_model/mm_evaluation/vqa_eval.pydiff --git a/examples/multimodal-modeling/Qwen-VL/run_autoround.sh b/examples/multimodal-modeling/Common_model/run_autoround.shsimilarity index 100%rename from examples/multimodal-modeling/Qwen-VL/run_autoround.shrename to examples/multimodal-modeling/Common_model/run_autoround.shdiff --git a/examples/multimodal-modeling/Qwen-VL/run_eval.sh b/examples/multimodal-modeling/Common_model/run_eval.shsimilarity index 100%rename from examples/multimodal-modeling/Qwen-VL/run_eval.shrename to examples/multimodal-modeling/Common_model/run_eval.shdiff --git a/test/test_autoopt.py b/test/test_autoopt.pyindex 2f500162a..988173c0a 100644--- a/test/test_autoopt.py+++ b/test/test_autoopt.py@@ -8,7 +8,7 @@ import torchimport transformersfrom transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoOPTRound+from auto_round import AutoOPTRound, AutoAdamRoundclass LLMDataLoader:@@ -44,9 +44,31 @@ class TestAutoRound(unittest.TestCase):iters=2,seqlen=10,dataset=self.llm_dataloader,+ quant_block_list=None)autoround.quantize()+++ def test_Adam(self):+ bits, group_size, sym = 4, 128, False+ from auto_round.utils import get_multimodal_block_names+ llm_block_names = get_multimodal_block_names(self.model, quant_vision=True)+ bits, group_size, sym, batch_size = 4, 128, False, 20+ adamround = AutoAdamRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ batch_size=batch_size,+ dataset=self.llm_dataloader,+ quant_block_list=llm_block_names+ )+ adamround.quantize()if __name__ == "__main__":unittest.main()+diff --git a/test/test_block_names.py b/test/test_block_names.pyindex 6e101f4a7..60fc8568d 100644--- a/test/test_block_names.py+++ b/test/test_block_names.py@@ -150,7 +150,7 @@ class TestQuantizationBlocks(unittest.TestCase):from auto_round.utils import get_block_names, validate_modulesllm_block_names = get_block_names(self.model)validate_modules(llm_block_names)- bits, group_size, sym, train_bs = 4, 128, False, 20+ bits, group_size, sym, batch_size = 4, 128, False, 20autoround = AutoRound(self.model,self.tokenizer,@@ -159,7 +159,7 @@ class TestQuantizationBlocks(unittest.TestCase):sym=sym,iters=2,seqlen=2,- train_bs=train_bs,+ batch_size=batch_size,dataset=self.llm_dataloader,quant_block_list=llm_block_names)@@ -190,3 +190,4 @@ if __name__ == "__main__":unittest.main()+