PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
enable llama3.2-vision model quantization (#269)
WeiweiZhang16b99d10a3c0e
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex ecd026435..44cbf92e6 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -704,6 +704,7 @@ class AutoRound(object):for key in kwargs.keys():if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) \+ or isinstance(kwargs[key], tuple) \or (key == "alibi") or (key == "attention_mask"):if "attention_mask" in key:if key not in self.inputs[name].keys():@@ -732,8 +733,8 @@ class AutoRound(object):self.inputs[name][key].append(to_device(kwargs[key], device=torch.device("cpu")))elif key not in self.inputs[name].keys():self.inputs[name][key] = list(torch.split(kwargs[key].to("cpu"), 1, dim=0)) \- if self.not_share_position_ids_flag \- else to_device(kwargs[key], device=torch.device("cpu"))+ if self.not_share_position_ids_flag \+ else to_device(kwargs[key], device=torch.device("cpu"))elif kwargs[key] is not None and self.not_share_position_ids_flag:self.inputs[name][key].extend(list(torch.split(kwargs[key].to("cpu"), 1, dim=0)))elif 'rotary_pos_emb' in key or 'cu_seqlens' in key:@@ -743,8 +744,14 @@ class AutoRound(object):else to_device(kwargs[key], device=torch.device("cpu"))elif kwargs[key] is not None and self.not_share_rotary_pos_emb_flag:self.inputs[name][key].append(to_device(kwargs[key], device=torch.device("cpu")))+ elif "cross_attention_states" in key:+ if key not in self.inputs[name].keys():+ self.inputs[name][key] = [to_device(kwargs[key], device=torch.device("cpu"))]+ else:+ self.inputs[name][key].extend(list(torch.split(kwargs[key].to("cpu"), 1, dim=0)))elif key not in self.inputs[name].keys():self.inputs[name][key] = to_device(kwargs[key], device=torch.device("cpu"))+if name == self.last_cache_name:raise NotImplementedErrorelse:@@ -1617,3 +1624,4 @@ class AutoAdamRound(AutoOPTRound):optimizer,**kwargs,)+diff --git a/auto_round/utils.py b/auto_round/utils.pyindex c0354318f..064bed6eb 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -367,7 +367,8 @@ def sampling_inputs(input_ids, input_others, indices, seqlen,for key in input_others.keys():if not share_attention_mask_flag and ("attention_mask" in key or "alibi" in key) \or (not_share_position_ids_flag and ("position_ids" in key or "cache_position" in key)) \- or (not_share_rotary_pos_emb_flag and ("rotary_pos_emb" in key or 'cu_seqlens' in key)):+ or (not_share_rotary_pos_emb_flag and ("rotary_pos_emb" in key or 'cu_seqlens' in key)) \+ or "cross_attention_states" in key:current_input_others[key] = Noneif input_others[key] is not None:current_input_others[key] = [input_others[key][i] for i in indices]@@ -970,3 +971,4 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):use_marlin=not disable_marlin,)return QuantLinear+diff --git a/examples/multimodal-modeling/Qwen-VL/README.md b/examples/multimodal-modeling/Qwen-VL/README.mdindex 9882f4fb1..d49b2c2ba 100644--- a/examples/multimodal-modeling/Qwen-VL/README.md+++ b/examples/multimodal-modeling/Qwen-VL/README.md@@ -1,8 +1,8 @@Step-by-Step============-This document presents step-by-step instructions for auto-round.-# Run Quantization on Qwen-VL Models++# Run Quantization on Qwen-VL/Qwen2-VL-Instruct ModelsIn this example, we introduce an straight-forward way to execute quantization on some popular multimodal models such as Qwen-VL and Qwen2-VL-Instruct.@@ -17,8 +17,9 @@ COCO: [train2017](http://images.cocodataset.org/zips/train2017.zip), and unzip tYou can also refer to the official Qwen-VL finetuning requirements to create a [custom dataset](https://github.com/QwenLM/Qwen-VL/blob/master/README.md#data-preparation)## Download the evaluation data-Please refer to [Qwen-VL evaluation](https://github.com/cognitedata/Qwen-VL-finetune/blob/master/eval_mm/EVALUATION.md)++Note: The quantized Qwen2-VL model evaluation is currently not supported.<details><summary>TextVQA Data Preparation</summary>@@ -71,15 +72,18 @@ cd ../..<br />## 2. Run Examples+Enter into the examples folder and install requirements+```bashpip install -r requirements.txt```-For Qwen2-VL quantization/Inference, please follow [Qwen2 official requirements](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct)++For Qwen2-VL quantization/Inference, please follow [Qwen2 official requirements](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct) and install the latest version of transformers by **building from source**.- **Default Settings:**```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Qwen/Qwen-VL --bits 4 --group_size 128+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Qwen/Qwen2-VL-7B-Instruct --bits 4 --group_size 128```- **Speedup the tuning:**@@ -88,17 +92,6 @@ reduce the seqlen to 512(potential large accuracy drop)or combine them-- **Enable quantized lm-head:**--Currently only support in Intel xpu and AutoRound format,however, we found the fake tuning could improve the accuracy is some scenarios. low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing nsamples,e.g. 128, to alleviate the issue.-```bash-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Qwen/Qwen-VL --bits 4 --group_size 128 --quant_lm_head-```--- **Utilizing the AdamW Optimizer:**--Include the flag `--adam`. Note that AdamW is less effective than sign gradient descent in many scenarios we tested.-- **Running on Intel Gaudi2**```bashbash run_autoround.sh@@ -247,3 +240,4 @@ If you find SignRound useful for your research, please cite our paper:+diff --git a/examples/multimodal-modeling/Qwen-VL/main.py b/examples/multimodal-modeling/Qwen-VL/main.pyindex fd7d0a19e..b6d0f7ac3 100644--- a/examples/multimodal-modeling/Qwen-VL/main.py+++ b/examples/multimodal-modeling/Qwen-VL/main.py@@ -1,10 +1,9 @@import argparseimport sys-sys.path.insert(0, '../../..')parser = argparse.ArgumentParser()-import torchimport os+import torchimport transformersos.environ["TOKENIZERS_PARALLELISM"] = "false"os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"@@ -12,8 +11,7 @@ torch.use_deterministic_algorithms(True, warn_only=True)from transformers import set_seedimport jsonfrom torch.utils.data import Dataset, DataLoader-from transformers import AutoModelForCausalLM, AutoTokenizer-import torch+from transformers import AutoTokenizer, AutoProcessor, AutoModelForCausalLMfrom auto_round.utils import convert_dtype_torch2strfrom typing import Dict, Optional, ListOLD_IMAGE_TOKEN = '<image>'@@ -36,6 +34,8 @@ def DataFormating(raw_data, image_folder=None, model_type='qwen'):sentence['value'] = sentence['value'].strip()if 'qwen2' in model_type: # for Qwen2-vlreplace_token = '<|vision_start|><|image_pad|><|vision_end|>'+ if 'mllama' in model_type:+ replace_token = '<|image|>'else:replace_img = os.path.join(image_folder, os.path.basename(source["image"]))replace_token = DEFAULT_IM_START_TOKEN + replace_img + DEFAULT_IM_END_TOKEN + '\n'@@ -43,17 +43,25 @@ def DataFormating(raw_data, image_folder=None, model_type='qwen'):return raw_data-def qwen2_preprocess(+def common_preprocess(sources,tokenizer: transformers.PreTrainedTokenizer,max_len: int,- system_message: str = "You are a helpful assistant."+ system_message: str = "You are a helpful assistant.",+ model_type='qwen2') -> Dict:- roles = {"user": "<|im_start|>user", "assistant": "<|im_start|>assistant"}- im_start = "<|im_start|>"- im_end = "<|im_end|>"+ if 'mllama' in model_type:+ roles = {"user": "<|start_header_id|>user<|end_header_id|>\n", "assistant": "<|start_header_id|>assistant<|end_header_id|>\n"}+ im_start = "<|start_header_id|>"+ im_end = "<|end_header_id|>\n"+ im_dot = '<|eot_id|>'+ text_start = '<|begin_of_text|>'+ else :+ roles = {"user": "<|im_start|>user", "assistant": "<|im_start|>assistant"}+ im_start = "<|im_start|>"+ im_end = "<|im_end|>"nl_tokens = '\n'- _system = 'system' + nl_tokens+ _system = 'system'# Apply prompt templatesinputs, targets = [], []@@ -62,12 +70,19 @@ def qwen2_preprocess(source = source[1:]text, target = "", None- system = im_start + _system + system_message + im_end + nl_tokens+ if 'mllama' in model_type:+ system = text_start + im_start + _system + im_end + nl_tokens + system_message + im_dot+ else:+ system = im_start + _system + nl_tokens + system_message + im_end + nl_tokenstext += systemfor j, sentence in enumerate(source):role = roles[sentence["from"]]- _text = role + nl_tokens + \- sentence["value"] + im_end + nl_tokens+ if 'mllama' in model_type:+ _text = role + nl_tokens + \+ sentence["value"] + im_dot+ else:+ _text = role + nl_tokens + \+ sentence["value"] + im_end + nl_tokenstext += _texttoken_length = len(tokenizer(text).input_ids)if token_length < max_len:@@ -156,33 +171,46 @@ class LazySupervisedDataset(Dataset):if i in self.cached_data_dict:return self.cached_data_dict[i]- if 'qwen2' not in model_type:+ if 'qwen' == model_type: # for Qwen-VLret = preprocess([self.raw_data[i]["conversations"]], self.tokenizer, self.max_len)ret = dict(input_ids=ret["input_ids"][0],labels=ret["labels"][0],attention_mask=ret["attention_mask"][0],)- else:- texts = qwen2_preprocess([self.raw_data[i]["conversations"]], self.tokenizer, self.max_len)- image_path = os.path.join(f"file://{self.image_folder}", os.path.basename(self.raw_data[i]["image"]))- image_inputs = fetch_image({'image':image_path})+ else: # Qwen2-VL and Llama-3.2+ texts = common_preprocess([self.raw_data[i]["conversations"]], self.tokenizer, self.max_len, model_type=model_type)+ if 'qwen2' in model_type:+ image_path = os.path.join(f"file://{self.image_folder}", os.path.basename(self.raw_data[i]["image"]))+ image = fetch_image({'image':image_path})+ else:+ image = Image.open(os.path.join(self.image_folder, os.path.basename(self.raw_data[i]["image"]))) #.convert('RGB')ret = self.tokenizer.processor(text=texts,- images=image_inputs,- videos=None,+ images=image,padding=True,+ truncation=True,return_tensors="pt",+ # videos=None,)- ret = dict(- input_ids=ret["input_ids"][0],- # labels=ret["labels"][0],- attention_mask=ret["attention_mask"][0],- image_grid_thw=ret["image_grid_thw"][0],- pixel_values=ret["pixel_values"],- )+ if 'qwen2' in model_type:+ ret = dict(+ input_ids=ret["input_ids"][0],+ # labels=ret["labels"][0],+ attention_mask=ret["attention_mask"][0],+ image_grid_thw=ret["image_grid_thw"][0],+ pixel_values=ret["pixel_values"],+ )+ else:+ ret = dict(+ input_ids=ret["input_ids"][0],+ attention_mask=ret["attention_mask"][0],+ aspect_ratio_ids=ret["aspect_ratio_ids"][0],+ aspect_ratio_mask=ret["aspect_ratio_mask"][0],+ cross_attention_mask=ret["cross_attention_mask"][0],+ pixel_values=ret["pixel_values"][0],+ )self.cached_data_dict[i] = ret-return ret@@ -340,7 +368,7 @@ if __name__ == '__main__':# ================= Evaluation Related =====================# parser.add_argument("--eval-path", type=str, default=None)- parser.add_argument("--eval-dataset", type=str, default="textvqa_val,scienceqa_test_img")+ parser.add_argument("--eval_dataset", type=str, default="textvqa_val,scienceqa_test_img")args = parser.parse_args()@@ -392,7 +420,14 @@ if __name__ == '__main__':questions = json.load(open(args.question_file, "r"))config = transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)model_type = config.model_type- if 'qwen2' not in model_type: # for Qwen-VL/Qwen-VL-Chat+ if "mllama" in model_type:+ from transformers import MllamaForConditionalGeneration+ model = MllamaForConditionalGeneration.from_pretrained(args.model_name,+ trust_remote_code=not args.disable_trust_remote_code) # torch_dtype=torch.bfloat16+ processor = AutoProcessor.from_pretrained(args.model_name)+ tokenizer.processor = processor+ default_collator = default_data_collator+ elif 'qwen2' not in model_type: # for Qwen-VL/Qwen-VL-Chattokenizer.pad_token_id = tokenizer.eod_idconfig.use_cache = Falseif dtype_str == "bf16":@@ -401,7 +436,6 @@ if __name__ == '__main__':model = AutoModelForCausalLM.from_pretrained(args.model_name, trust_remote_code=not args.disable_trust_remote_code, fp16=True).eval()else:model = AutoModelForCausalLM.from_pretrained(args.model_name, trust_remote_code=not args.disable_trust_remote_code).eval()- raw_data = DataFormating(questions, args.image_folder)default_collator = default_data_collator if tokenizer is None else DataCollatorWithPadding(tokenizer)else: # for Qwen2-VL-instructfrom transformers import Qwen2VLForConditionalGeneration, AutoProcessor@@ -410,7 +444,7 @@ if __name__ == '__main__':processor = AutoProcessor.from_pretrained(args.model_name)tokenizer.processor = processordefault_collator = default_data_collator-+raw_data = DataFormating(questions, args.image_folder, model_type=model_type)dataset = LazySupervisedDataset(raw_data, tokenizer,max_len=min(args.seqlen, tokenizer.model_max_length), image_folder=args.image_folder)@@ -518,7 +552,7 @@ if __name__ == '__main__':if eval_folder is None:eval_folder = output_dir- if not args.disable_eval and "fake" in deployment_device: ## TODO+ if not args.disable_eval and "fake" in deployment_device and model_type == "qwen": ## for Qwen-VLmodel = model.half()model = model.to(torch_device)datasets=args.eval_dataset.split(',')