PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
enable_qwen2-vl_quantization (#248)
WeiweiZhang13275df9309ed
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 6f84275c7..043893ce8 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -28,7 +28,10 @@ from tqdm import tqdmfrom .calib_dataset import get_dataloaderfrom .quantizer import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layer, \WrapperTransformerConv1d-from .special_model_handler import check_hidden_state_dim, check_share_attention_mask, check_not_share_position_ids+from .special_model_handler import (check_hidden_state_dim,+ check_share_attention_mask,+ check_not_share_position_ids,+ check_not_share_rotary_pos_emb)from .utils import (CpuInfo,block_forward,@@ -452,12 +455,16 @@ class AutoRound(object):self.seqlen,self.share_attention_mask_flag,self.not_share_position_ids_flag,+ self.not_share_rotary_pos_emb_flag,self.input_dim)tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(cache_device)- output.extend(list(torch.split(tmp_output, 1, dim=self.input_dim)))+ if self.train_bs == 1 and self.not_share_rotary_pos_emb_flag:+ output.append(tmp_output)+ else:+ output.extend(list(torch.split(tmp_output, 1, dim=self.input_dim)))torch.cuda.empty_cache()return output@@ -657,11 +664,19 @@ class AutoRound(object):self.input_dim = check_hidden_state_dim(self.model, positional_args)self.share_attention_mask_flag = check_share_attention_mask(self.model, hidden_states, **kwargs)self.not_share_position_ids_flag = check_not_share_position_ids(self.model, **kwargs)+ self.not_share_rotary_pos_emb_flag = check_not_share_rotary_pos_emb(self.model, **kwargs)if name in self.inputs:- self.inputs[name]["input_ids"].extend(list(torch.split(hidden_states.to("cpu"), 1, dim=self.input_dim)))+ if self.train_bs == 1 and self.not_share_rotary_pos_emb_flag:+ self.inputs[name]["input_ids"].append(hidden_states.to("cpu"))+ else:+ self.inputs[name]["input_ids"].extend(+ list(torch.split(hidden_states.to("cpu"), 1, dim=self.input_dim)))else:self.inputs[name] = {}- self.inputs[name]["input_ids"] = list(torch.split(hidden_states.to("cpu"), 1, dim=self.input_dim))+ if self.train_bs == 1 and self.not_share_rotary_pos_emb_flag:+ self.inputs[name]["input_ids"] = [hidden_states.to("cpu")]+ else:+ self.inputs[name]["input_ids"] = list(torch.split(hidden_states.to("cpu"), 1, dim=self.input_dim))if "positional_inputs" not in self.inputs[name]:self.inputs[name]["positional_inputs"] = []@@ -690,13 +705,25 @@ class AutoRound(object):self.inputs[name][key].extend(list(torch.split(alibi.to("cpu"), 1, dim=0)))else:self.inputs[name][key] = list(torch.split(alibi.to("cpu"), 1, dim=0))- elif "position_ids" in key:- if key not in self.inputs[name].keys():- self.inputs[name][key] = list(torch.split(kwargs[key].to("cpu"), 1, dim=0)) \- if self.not_share_position_ids_flag \- else to_device(kwargs[key], device=torch.device("cpu"))+ elif "position_ids" in key or 'cache_position' in key:+ if self.train_bs == 1 and self.not_share_rotary_pos_emb_flag:+ if key not in self.inputs[name].keys():+ self.inputs[name][key] = [to_device(kwargs[key], device=torch.device("cpu"))]+ else:+ self.inputs[name][key].append(to_device(kwargs[key], device=torch.device("cpu")))+ elif key not in self.inputs[name].keys():+ self.inputs[name][key] = list(torch.split(kwargs[key].to("cpu"), 1, dim=0)) \+ if self.not_share_position_ids_flag \+ else to_device(kwargs[key], device=torch.device("cpu"))elif kwargs[key] is not None and self.not_share_position_ids_flag:self.inputs[name][key].extend(list(torch.split(kwargs[key].to("cpu"), 1, dim=0)))+ elif 'rotary_pos_emb' in key or 'cu_seqlens' in key:+ if key not in self.inputs[name].keys():+ self.inputs[name][key] = [to_device(kwargs[key], device=torch.device("cpu"))] \+ if self.not_share_rotary_pos_emb_flag \+ else to_device(kwargs[key], device=torch.device("cpu"))+ elif kwargs[key] is not None and self.not_share_rotary_pos_emb_flag:+ self.inputs[name][key].append(to_device(kwargs[key], device=torch.device("cpu")))elif key not in self.inputs[name].keys():self.inputs[name][key] = to_device(kwargs[key], device=torch.device("cpu"))if name == self.last_cache_name:@@ -939,10 +966,11 @@ class AutoRound(object):seqlen=self.seqlen,share_attention_mask_flag=self.share_attention_mask_flag,not_share_position_ids_flag=self.not_share_position_ids_flag,+ not_share_rotary_pos_emb_flag=self.not_share_rotary_pos_emb_flag,input_dim=self.input_dim,)- current_output = [output[i] for i in indices]+ current_output = [output[x] for x in indices]current_output = torch.cat(current_output, dim=self.input_dim)current_output = to_device(current_output, device)@@ -1560,3 +1588,4 @@ class AutoAdamRound(AutoOPTRound):optimizer,**kwargs,)+diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex b8ca27d26..e46b4800e 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -232,6 +232,9 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exif output_dir is None:return modeltokenizer = kwargs["tokenizer"]+ if output_dir is None:+ model.tokenizer = tokenizer+ return modelif tokenizer is not None:tokenizer.save_pretrained(output_dir)modules_to_not_convert = []diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex ce79cfb2a..722d51a9e 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -16,8 +16,8 @@ import torchshare_attention_mask_tuple = ("baichuan",)special_states_dim_tuple = ("chatglm",)-not_share_position_ids_tuple = ("llava", "phi3_v",)-+not_share_position_ids_tuple = ("llava", "phi3_v", "qwen2_vl",)+not_share_rotary_pos_emb_tuple = ("qwen2_vl",)def check_share_attention_mask(model, hidden_states, attention_mask=None, **kwargs):"""Checks if the attention mask states of the hidden states are shared in the model.@@ -65,3 +65,11 @@ def check_not_share_position_ids(model, **kwargs):return bool(is_special)+def check_not_share_rotary_pos_emb(model, **kwargs):+ is_special = False+ for key in not_share_rotary_pos_emb_tuple:+ if hasattr(model, "config") and key in model.config.model_type:+ is_special = True+ break+ return bool(is_special)+diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 60ca6d99c..5fb597dd7 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -346,7 +346,9 @@ def collect_best_params(block):@torch.no_grad()def sampling_inputs(input_ids, input_others, indices, seqlen,- share_attention_mask_flag=False, not_share_position_ids_flag=False, input_dim=0):+ share_attention_mask_flag=False,+ not_share_position_ids_flag=False,+ not_share_rotary_pos_emb_flag=False, input_dim=0):"""Samples inputs based on the given indices and sequence length.Args:@@ -361,15 +363,16 @@ def sampling_inputs(input_ids, input_others, indices, seqlen,"""current_input_ids = [input_ids[i] for i in indices]current_input_ids = torch.cat(current_input_ids, dim=input_dim)-current_input_others = {"positional_inputs": input_others["positional_inputs"]}for key in input_others.keys():if not share_attention_mask_flag and ("attention_mask" in key or "alibi" in key) \- or (not_share_position_ids_flag and "position_ids" in key):+ or (not_share_position_ids_flag and ("position_ids" in key or "cache_position" in key)) \+ or (not_share_rotary_pos_emb_flag and ("rotary_pos_emb" in key or 'cu_seqlens' in key)):current_input_others[key] = Noneif input_others[key] is not None:current_input_others[key] = [input_others[key][i] for i in indices]- current_input_others[key] = torch.cat(current_input_others[key], dim=0)+ if not isinstance(current_input_others[key], torch.Tensor):+ current_input_others[key] = torch.cat(current_input_others[key], dim=0)else:current_input_others[key] = input_others[key]diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 48ed2770f..3d514e75e 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -94,10 +94,10 @@ if __name__ == '__main__':parser.add_argument("--enable_minmax_tuning", action='store_true',help="enable_minmax_tuning is deprecated")- parser.add_argument("--deployment_device", default=None, type=str,- help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'gpu' and 'xpu'."- "default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")-+ parser.add_argument("--deployment_device", default='auto_round', type=str,+ help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'xpu', 'gpu(auto_gptq)' and 'auto_round'."+ "default to 'auto_round', 'fake' indicating that it only performs fake quantization and won't be exported to any device.")+parser.add_argument("--format", default=None, type=str,help="The format in which to save the model. ""The options are 'auto_round', 'auto_gptq', 'auto_awq', 'itrex', 'itrex_xpu' and 'fake'."@@ -387,6 +387,7 @@ if __name__ == '__main__':tokenizer.save_pretrained(output_dir)if eval_folder is None:eval_folder = output_dir+if (not ('gpu' in deployment_device or len(gpu_formats) > 0)) and 'fake' not in deployment_device:print('does not support cpu, xpu model evaluation.')exit() ## does not support cpu,xpu model eval@@ -454,3 +455,4 @@ if __name__ == '__main__':from lm_eval.utils import make_tableprint(make_table(res))+diff --git a/examples/multimodal-modeling/Phi-3-vision/README.md b/examples/multimodal-modeling/Phi-3-vision/README.mdindex b0557bae2..1ca9e3340 100644--- a/examples/multimodal-modeling/Phi-3-vision/README.md+++ b/examples/multimodal-modeling/Phi-3-vision/README.md@@ -61,7 +61,7 @@ import requestsimport iofrom transformers import AutoModelForCausalLMfrom transformers import AutoProcessor-from auto_round.auto_quantizer import AutoHfQuantizer+from auto_round import AutoRoundConfigquantized_model_path = "./tmp_autoround"model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True, torch_dtype="auto", _attn_implementation='flash_attention_2') # use _attn_implementation='eager' to disable flash attention@@ -137,3 +137,4 @@ If you find SignRound useful for your research, please cite our paper:+diff --git a/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py b/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.pyindex dd440f106..5aa57ae7a 100644--- a/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py+++ b/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py@@ -597,11 +597,14 @@ if __name__ == "__main__":model_args = f"pretrained={args.model_name}"if hasattr(config, "quantization_config"):quantization_config = config.quantization_config- if "quant_method" in quantization_config and "auto-round" in quantization_config["quant_method"]:- from auto_round.auto_quantizer import AutoHfQuantizer- else:- from phi3_v import Phi3VGPTQForCausalLM- model_args += f",autogptq=True,gptq_use_triton=True"+ if "quant_method" in quantization_config and ("auto-round" in quantization_config["quant_method"] or+ ("gptq" in quantization_config["quant_method"] and args.device == "hpu")):+ try:+ from auto_round import AutoRoundConfig+ except:+ from auto_round.auto_quantizer import AutoHfQuantizer++ model_args += f",autogptq=True,gptq_use_triton=True"if args.trust_remote_code:model_args += f",trust_remote_code=True"model_args += ",dtype=bfloat16"@@ -621,3 +624,4 @@ if __name__ == "__main__":print("cost time: ", time.time() - s)+diff --git a/examples/multimodal-modeling/Qwen-VL/README.md b/examples/multimodal-modeling/Qwen-VL/README.mdindex 4980eb28a..9882f4fb1 100644--- a/examples/multimodal-modeling/Qwen-VL/README.md+++ b/examples/multimodal-modeling/Qwen-VL/README.md@@ -4,7 +4,7 @@ Step-by-StepThis document presents step-by-step instructions for auto-round.# Run Quantization on Qwen-VL Models-In this example, we introduce an straight-forward way to execute quantization on some popular multimodal models such as Qwen-VL.+In this example, we introduce an straight-forward way to execute quantization on some popular multimodal models such as Qwen-VL and Qwen2-VL-Instruct.## Download the calibration data@@ -75,6 +75,7 @@ Enter into the examples folder and install requirements```bashpip install -r requirements.txt```+For Qwen2-VL quantization/Inference, please follow [Qwen2 official requirements](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct)- **Default Settings:**```bash@@ -105,13 +106,15 @@ bash run_autoround.sh## 3. run inference+- Qwen-VL inference+```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom transformers.generation import GenerationConfigimport torchfrom transformers import set_seedset_seed(1234)- from auto_round.auto_quantizer import AutoHfQuantizer+ from auto_round import AutoRoundConfigquantized_model_path = "./tmp_autoround"tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, trust_remote_code=True)# use bf16@@ -141,6 +144,62 @@ bash run_autoround.sh```+- Qwen2-VL-7B-Instruct inference++```python+from transformers import Qwen2VLForConditionalGeneration, AutoProcessor+from qwen_vl_utils import process_vision_info+from auto_round import AutoRoundConfig+quantized_model_path="./tmp_autoround"+model = Qwen2VLForConditionalGeneration.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)+processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct")+messages = [{+ "role": "user",+ "content": [+ {+ "type": "image",+ "image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg",+ },+ {"type": "text", "text": "Describe this image."},]+}]+# Preparation for inference+text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)+image_inputs, video_inputs = process_vision_info(messages)+inputs = processor(+ text=[text],+ images=image_inputs,+ videos=video_inputs,+ padding=True,+ return_tensors="pt",+)+inputs = inputs.to("cuda")++# Inference: Generation of the output+generated_ids = model.generate(**inputs, max_new_tokens=50)+generated_ids_trimmed = [+ out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)+]+output_text = processor.batch_decode(+ generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False+)+print(output_text)+# The image depicts a serene beach scene at sunset. A woman is sitting on the sand, facing a large dog that appears to be a Labrador Retriever. The dog is wearing a harness and is extending its paw towards the woman's hand, possibly++# messages = [{+# "role": "user",+# "content": [+# {+# "type": "image",+# "image": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg",+# },+# {"type": "text", "text": "What does the label 15 represent? (1) lava (2) core (3) tunnel (4) ash cloud"},]+# }]++# The label 15 represents an ash cloud. In the context of a volcano, an ash cloud is formed when volcanic ash is ejected into the atmosphere during an eruption. Therefore, the correct answer is:\n\n(4) ash cloud++```++## 4. ResultsUsing [COCO 2017](https://cocodataset.org/) and [LLaVA-Instruct-150K](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K) datasets for quantization calibration, and TextVQA dataset for evaluation. please follow the [recipe](./run_autoround.sh) and [evaluate script](./run_eval.sh). The results for Qwen-VL are as follows:| Metric | bf16 | INT4 |@@ -163,7 +222,6 @@ Using [COCO 2017](https://cocodataset.org/) and [LLaVA-Instruct-150K](https://hu| scienceVQA | 0.6748 | 0.6574 |-## 5. EnvironmentPyTorch 1.8 or higher version is needed@@ -188,3 +246,4 @@ If you find SignRound useful for your research, please cite our paper:+diff --git a/examples/multimodal-modeling/Qwen-VL/main.py b/examples/multimodal-modeling/Qwen-VL/main.pyindex 0c5f35bf4..fd7d0a19e 100644--- a/examples/multimodal-modeling/Qwen-VL/main.py+++ b/examples/multimodal-modeling/Qwen-VL/main.py@@ -22,8 +22,9 @@ DEFAULT_IM_END_TOKEN = '</img>'from transformers.trainer_pt_utils import LabelSmootherIGNORE_TOKEN_ID = LabelSmoother.ignore_indeximport inspect+from PIL import Image-def DataFormating(raw_data, image_folder):+def DataFormating(raw_data, image_folder=None, model_type='qwen'):for source in raw_data:source_inputs = source['conversations']for sentence in source_inputs:@@ -31,23 +32,67 @@ def DataFormating(raw_data, image_folder):sentence['from'] = sentence['from'].replace('gpt', 'assistant')if OLD_IMAGE_TOKEN in sentence['value']:sentence['value'] = sentence['value'].replace(OLD_IMAGE_TOKEN, '').strip()- sentence['value'] = OLD_IMAGE_TOKEN + '\n' + sentence['value']+ sentence['value'] = OLD_IMAGE_TOKEN + sentence['value']sentence['value'] = sentence['value'].strip()- replace_img = os.path.join(image_folder, os.path.basename(source["image"]))- replace_token = DEFAULT_IM_START_TOKEN + replace_img + DEFAULT_IM_END_TOKEN+ if 'qwen2' in model_type: # for Qwen2-vl+ replace_token = '<|vision_start|><|image_pad|><|vision_end|>'+ else:+ replace_img = os.path.join(image_folder, os.path.basename(source["image"]))+ replace_token = DEFAULT_IM_START_TOKEN + replace_img + DEFAULT_IM_END_TOKEN + '\n'sentence["value"] = sentence["value"].replace(OLD_IMAGE_TOKEN, replace_token)return raw_data-def preprocess(++def qwen2_preprocess(sources,tokenizer: transformers.PreTrainedTokenizer,max_len: int,system_message: str = "You are a helpful assistant.") -> Dict:roles = {"user": "<|im_start|>user", "assistant": "<|im_start|>assistant"}+ im_start = "<|im_start|>"+ im_end = "<|im_end|>"+ nl_tokens = '\n'+ _system = 'system' + nl_tokens++ # Apply prompt templates+ inputs, targets = [], []+ for i, source in enumerate(sources):+ if roles[source[0]["from"]] != roles["user"]:+ source = source[1:]++ text, target = "", None+ system = im_start + _system + system_message + im_end + nl_tokens+ text += system+ for j, sentence in enumerate(source):+ role = roles[sentence["from"]]+ _text = role + nl_tokens + \+ sentence["value"] + im_end + nl_tokens+ text += _text+ token_length = len(tokenizer(text).input_ids)+ if token_length < max_len:+ text += tokenizer.pad_token * (max_len - token_length)+ else:+ text = tokenizer.decode(tokenizer(text).input_ids[:max_len])+ pass+ inputs.append(text)++ return inputs- im_start = tokenizer.im_start_id- im_end = tokenizer.im_end_id++def preprocess(+ sources,+ tokenizer: transformers.PreTrainedTokenizer,+ max_len: int,+ system_message: str = "You are a helpful assistant."+) -> Dict:+ roles = {"user": "<|im_start|>user", "assistant": "<|im_start|>assistant"}+ if 'qwen2' not in model_type:+ im_start = tokenizer.im_start_id+ im_end = tokenizer.im_end_id+ else:+ im_start = tokenizer('<|im_start|>')+ im_end = tokenizer('<|im_end|>')nl_tokens = tokenizer('\n').input_ids_system = tokenizer('system').input_ids + nl_tokens_user = tokenizer('user').input_ids + nl_tokens@@ -82,51 +127,25 @@ def preprocess(target += [IGNORE_TOKEN_ID] * (max_len - len(target))input_ids.append(input_id[:max_len])targets.append(target[:max_len])- if i >= 512:- breakinput_ids = torch.tensor(input_ids, dtype=torch.int)targets = torch.tensor(targets, dtype=torch.int)-+return dict(input_ids=input_ids,labels=targets,attention_mask=input_ids.ne(tokenizer.pad_token_id),)--class SupervisedDataset(Dataset):- """Dataset for supervised fine-tuning."""-- def __init__(self, raw_data, tokenizer: transformers.PreTrainedTokenizer, max_len: int):- super(SupervisedDataset, self).__init__()-- print("Formatting inputs...")- sources = [example["conversations"] for example in raw_data]- data_dict = preprocess(sources, tokenizer, max_len)-- self.input_ids = data_dict["input_ids"]- self.labels = data_dict["labels"]- self.attention_mask = data_dict["attention_mask"]-- def __len__(self):- return len(self.input_ids)- def __getitem__(self, i) -> Dict[str, torch.Tensor]:- return dict(- input_ids=self.input_ids[i],- labels=self.labels[i],- attention_mask=self.attention_mask[i],- )-class LazySupervisedDataset(Dataset):"""Dataset for supervised fine-tuning."""- def __init__(self, raw_data, tokenizer: transformers.PreTrainedTokenizer, max_len: int):+ def __init__(self, raw_data, tokenizer: transformers.PreTrainedTokenizer,+ max_len: int, image_folder=None, model_type='qwen_vl'):super(LazySupervisedDataset, self).__init__()self.tokenizer = tokenizerself.max_len = max_len-+ self.image_folder = image_folderprint("Formatting inputs...Skip in lazy mode")- self.tokenizer = tokenizerself.raw_data = raw_dataself.cached_data_dict = {}@@ -137,12 +156,31 @@ class LazySupervisedDataset(Dataset):if i in self.cached_data_dict:return self.cached_data_dict[i]- ret = preprocess([self.raw_data[i]["conversations"]], self.tokenizer, self.max_len)- ret = dict(- input_ids=ret["input_ids"][0],- labels=ret["labels"][0],- attention_mask=ret["attention_mask"][0],- )+ if 'qwen2' not in model_type:+ ret = preprocess([self.raw_data[i]["conversations"]], self.tokenizer, self.max_len)+ ret = dict(+ input_ids=ret["input_ids"][0],+ labels=ret["labels"][0],+ attention_mask=ret["attention_mask"][0],+ )+ else:+ texts = qwen2_preprocess([self.raw_data[i]["conversations"]], self.tokenizer, self.max_len)+ image_path = os.path.join(f"file://{self.image_folder}", os.path.basename(self.raw_data[i]["image"]))+ image_inputs = fetch_image({'image':image_path})+ ret = self.tokenizer.processor(+ text=texts,+ images=image_inputs,+ videos=None,+ padding=True,+ return_tensors="pt",+ )+ ret = dict(+ input_ids=ret["input_ids"][0],+ # labels=ret["labels"][0],+ attention_mask=ret["attention_mask"][0],+ image_grid_thw=ret["image_grid_thw"][0],+ pixel_values=ret["pixel_values"],+ )self.cached_data_dict[i] = retreturn ret@@ -160,7 +198,7 @@ def set_signature_columns_if_needed(model):# Labels may be named label or label_ids, the default data collator handles that.signature_columns += list(set(["label", "label_ids", 'labels']))return signature_columns-+def get_collator_with_removed_columns(model, data_collator: Callable, description: Optional[str] = None) -> Callable:"""Wrap the data collator in a callable removing unused columns."""@@ -174,7 +212,8 @@ def get_collator_with_removed_columns(model, data_collator: Callable, descriptio)return remove_columns_collator-def get_train_dataloader(train_dataset, model, data_collator, train_batch_size=1, num_workers=0) -> DataLoader:+def get_train_dataloader(train_dataset, model, data_collator=default_data_collator,+ train_batch_size=1, num_workers=0) -> DataLoader:"""Returns the training [`~torch.utils.data.DataLoader`].@@ -186,7 +225,8 @@ def get_train_dataloader(train_dataset, model, data_collator, train_batch_size=1if train_dataset is None:raise ValueError("Trainer: training requires a train_dataset.")- data_collator = get_collator_with_removed_columns(model, data_collator, description="training")+ if data_collator != default_data_collator:+ data_collator = get_collator_with_removed_columns(model, data_collator, description="training")dataloader_params = {"batch_size": train_batch_size,@@ -240,7 +280,7 @@ if __name__ == '__main__':parser.add_argument("--seqlen", default=512, type=int,help="sequence length")- parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")+ parser.add_argument("--gradient_accumulate_steps", default=8, type=int, help="gradient accumulate steps")parser.add_argument("--nblocks", default=1, type=int, help="num of blocks to tune together")@@ -331,10 +371,8 @@ if __name__ == '__main__':torch.manual_seed(1234)model_name = args.model_name- questions = json.load(open(args.question_file, "r"))tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code,padding_side="right", use_fast=False)- tokenizer.pad_token_id = tokenizer.eod_idseqlen = args.seqlenif hasattr(tokenizer, "model_max_length"):if tokenizer.model_max_length < seqlen:@@ -343,28 +381,40 @@ if __name__ == '__main__':seqlen = min(seqlen, tokenizer.model_max_length)args.seqlen = seqlen- config = transformers.AutoConfig.from_pretrained(- model_name,- trust_remote_code=True,- )- config.use_cache = False+ torch_dtype = "auto"if args.model_dtype != None:if args.model_dtype == "float16" or args.model_dtype == "fp16":torch_dtype = torch.float16if args.model_dtype == "bfloat16" or args.model_dtype == "bf16":torch_dtype = torch.bfloat16+dtype_str = convert_dtype_torch2str(torch_dtype)- if dtype_str == "bf16":- model = AutoModelForCausalLM.from_pretrained(args.model_name, config=config, trust_remote_code=not args.disable_trust_remote_code, bf16=True).eval()- elif dtype_str == "fp16":- model = AutoModelForCausalLM.from_pretrained(args.model_name, config=config, trust_remote_code=not args.disable_trust_remote_code, fp16=True).eval()- else:- model = AutoModelForCausalLM.from_pretrained(args.model_name, config=config, trust_remote_code=not args.disable_trust_remote_code).eval()- raw_data = DataFormating(questions, args.image_folder)- # dataset = SupervisedDataset(raw_data, tokenizer, max_len=tokenizer.model_max_length)- dataset = LazySupervisedDataset(raw_data, tokenizer, max_len=min(args.seqlen, tokenizer.model_max_length))- default_collator = default_data_collator if tokenizer is None else DataCollatorWithPadding(tokenizer)- dataloader = get_train_dataloader(dataset, model, default_collator, train_batch_size=args.train_bs)+ questions = json.load(open(args.question_file, "r"))+ config = transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)+ model_type = config.model_type+ if 'qwen2' not in model_type: # for Qwen-VL/Qwen-VL-Chat+ tokenizer.pad_token_id = tokenizer.eod_id+ config.use_cache = False+ if dtype_str == "bf16":+ model = AutoModelForCausalLM.from_pretrained(args.model_name, trust_remote_code=not args.disable_trust_remote_code, bf16=True).eval()+ elif dtype_str == "fp16":+ model = AutoModelForCausalLM.from_pretrained(args.model_name, trust_remote_code=not args.disable_trust_remote_code, fp16=True).eval()+ else:+ model = AutoModelForCausalLM.from_pretrained(args.model_name, trust_remote_code=not args.disable_trust_remote_code).eval()+ raw_data = DataFormating(questions, args.image_folder)+ default_collator = default_data_collator if tokenizer is None else DataCollatorWithPadding(tokenizer)+ else: # for Qwen2-VL-instruct+ from transformers import Qwen2VLForConditionalGeneration, AutoProcessor+ from qwen_vl_utils import process_vision_info, fetch_image+ model = Qwen2VLForConditionalGeneration.from_pretrained(args.model_name, torch_dtype=torch_dtype)+ processor = AutoProcessor.from_pretrained(args.model_name)+ tokenizer.processor = processor+ default_collator = default_data_collator++ raw_data = DataFormating(questions, args.image_folder, model_type=model_type)+ dataset = LazySupervisedDataset(raw_data, tokenizer,+ max_len=min(args.seqlen, tokenizer.model_max_length), image_folder=args.image_folder)+ dataloader = get_train_dataloader(dataset, model, data_collator=default_collator, train_batch_size=args.train_bs)from auto_round import (AutoRound,AutoAdamRound)diff --git a/examples/multimodal-modeling/Qwen-VL/mm_evaluation/main.py b/examples/multimodal-modeling/Qwen-VL/mm_evaluation/main.pyindex 4e1446705..50b26e67f 100644--- a/examples/multimodal-modeling/Qwen-VL/mm_evaluation/main.py+++ b/examples/multimodal-modeling/Qwen-VL/mm_evaluation/main.py@@ -45,10 +45,11 @@ if __name__ == "__main__":if hasattr(config, "quantization_config"):quantization_config = config.quantization_config- if "quant_method" in quantization_config and "auto-round" in quantization_config["quant_method"]:- from auto_round.auto_quantizer import AutoHfQuantizer- elif "quant_method" in quantization_config and quantization_config["quant_method"] == "gptq":- if args.device == "hpu":+ if "quant_method" in quantization_config and ("auto-round" in quantization_config["quant_method"] or+ ("gptq" in quantization_config["quant_method"] and args.device == "hpu")):+ try:+ from auto_round import AutoRoundConfig+ except:from auto_round.auto_quantizer import AutoHfQuantizermodel_name = args.model_nametorch_dtype = torch.float@@ -99,3 +100,4 @@ if __name__ == "__main__":print("cost time: ", time.time() - s)+