Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

enable_qwen2-vl_quantization (#248)

WeiweiZhang13275df9309ed
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 6f84275c7..043893ce8 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -28,7 +28,10 @@ from tqdm import tqdm
from .calib_dataset import get_dataloader
from .quantizer import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layer, \
WrapperTransformerConv1d
-from .special_model_handler import check_hidden_state_dim, check_share_attention_mask, check_not_share_position_ids
+from .special_model_handler import (check_hidden_state_dim,
+ check_share_attention_mask,
+ check_not_share_position_ids,
+ check_not_share_rotary_pos_emb)
from .utils import (
CpuInfo,
block_forward,
@@ -452,12 +455,16 @@ class AutoRound(object):
self.seqlen,
self.share_attention_mask_flag,
self.not_share_position_ids_flag,
+ self.not_share_rotary_pos_emb_flag,
self.input_dim
)
tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(
cache_device
)
- output.extend(list(torch.split(tmp_output, 1, dim=self.input_dim)))
+ if self.train_bs == 1 and self.not_share_rotary_pos_emb_flag:
+ output.append(tmp_output)
+ else:
+ output.extend(list(torch.split(tmp_output, 1, dim=self.input_dim)))
torch.cuda.empty_cache()
return output
@@ -657,11 +664,19 @@ class AutoRound(object):
self.input_dim = check_hidden_state_dim(self.model, positional_args)
self.share_attention_mask_flag = check_share_attention_mask(self.model, hidden_states, **kwargs)
self.not_share_position_ids_flag = check_not_share_position_ids(self.model, **kwargs)
+ self.not_share_rotary_pos_emb_flag = check_not_share_rotary_pos_emb(self.model, **kwargs)
if name in self.inputs:
- self.inputs[name]["input_ids"].extend(list(torch.split(hidden_states.to("cpu"), 1, dim=self.input_dim)))
+ if self.train_bs == 1 and self.not_share_rotary_pos_emb_flag:
+ self.inputs[name]["input_ids"].append(hidden_states.to("cpu"))
+ else:
+ self.inputs[name]["input_ids"].extend(
+ list(torch.split(hidden_states.to("cpu"), 1, dim=self.input_dim)))
else:
self.inputs[name] = {}
- self.inputs[name]["input_ids"] = list(torch.split(hidden_states.to("cpu"), 1, dim=self.input_dim))
+ if self.train_bs == 1 and self.not_share_rotary_pos_emb_flag:
+ self.inputs[name]["input_ids"] = [hidden_states.to("cpu")]
+ else:
+ self.inputs[name]["input_ids"] = list(torch.split(hidden_states.to("cpu"), 1, dim=self.input_dim))
if "positional_inputs" not in self.inputs[name]:
self.inputs[name]["positional_inputs"] = []
@@ -690,13 +705,25 @@ class AutoRound(object):
self.inputs[name][key].extend(list(torch.split(alibi.to("cpu"), 1, dim=0)))
else:
self.inputs[name][key] = list(torch.split(alibi.to("cpu"), 1, dim=0))
- elif "position_ids" in key:
- if key not in self.inputs[name].keys():
- self.inputs[name][key] = list(torch.split(kwargs[key].to("cpu"), 1, dim=0)) \
- if self.not_share_position_ids_flag \
- else to_device(kwargs[key], device=torch.device("cpu"))
+ elif "position_ids" in key or 'cache_position' in key:
+ if self.train_bs == 1 and self.not_share_rotary_pos_emb_flag:
+ if key not in self.inputs[name].keys():
+ self.inputs[name][key] = [to_device(kwargs[key], device=torch.device("cpu"))]
+ else:
+ self.inputs[name][key].append(to_device(kwargs[key], device=torch.device("cpu")))
+ elif key not in self.inputs[name].keys():
+ self.inputs[name][key] = list(torch.split(kwargs[key].to("cpu"), 1, dim=0)) \
+ if self.not_share_position_ids_flag \
+ else to_device(kwargs[key], device=torch.device("cpu"))
elif kwargs[key] is not None and self.not_share_position_ids_flag:
self.inputs[name][key].extend(list(torch.split(kwargs[key].to("cpu"), 1, dim=0)))
+ elif 'rotary_pos_emb' in key or 'cu_seqlens' in key:
+ if key not in self.inputs[name].keys():
+ self.inputs[name][key] = [to_device(kwargs[key], device=torch.device("cpu"))] \
+ if self.not_share_rotary_pos_emb_flag \
+ else to_device(kwargs[key], device=torch.device("cpu"))
+ elif kwargs[key] is not None and self.not_share_rotary_pos_emb_flag:
+ self.inputs[name][key].append(to_device(kwargs[key], device=torch.device("cpu")))
elif key not in self.inputs[name].keys():
self.inputs[name][key] = to_device(kwargs[key], device=torch.device("cpu"))
if name == self.last_cache_name:
@@ -939,10 +966,11 @@ class AutoRound(object):
seqlen=self.seqlen,
share_attention_mask_flag=self.share_attention_mask_flag,
not_share_position_ids_flag=self.not_share_position_ids_flag,
+ not_share_rotary_pos_emb_flag=self.not_share_rotary_pos_emb_flag,
input_dim=self.input_dim,
)
- current_output = [output[i] for i in indices]
+ current_output = [output[x] for x in indices]
current_output = torch.cat(current_output, dim=self.input_dim)
current_output = to_device(current_output, device)
@@ -1560,3 +1588,4 @@ class AutoAdamRound(AutoOPTRound):
optimizer,
**kwargs,
)
+
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index b8ca27d26..e46b4800e 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -232,6 +232,9 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
if output_dir is None:
return model
tokenizer = kwargs["tokenizer"]
+ if output_dir is None:
+ model.tokenizer = tokenizer
+ return model
if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
modules_to_not_convert = []
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index ce79cfb2a..722d51a9e 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -16,8 +16,8 @@ import torch
share_attention_mask_tuple = ("baichuan",)
special_states_dim_tuple = ("chatglm",)
-not_share_position_ids_tuple = ("llava", "phi3_v",)
-
+not_share_position_ids_tuple = ("llava", "phi3_v", "qwen2_vl",)
+not_share_rotary_pos_emb_tuple = ("qwen2_vl",)
def check_share_attention_mask(model, hidden_states, attention_mask=None, **kwargs):
"""Checks if the attention mask states of the hidden states are shared in the model.
@@ -65,3 +65,11 @@ def check_not_share_position_ids(model, **kwargs):
return bool(is_special)
+def check_not_share_rotary_pos_emb(model, **kwargs):
+ is_special = False
+ for key in not_share_rotary_pos_emb_tuple:
+ if hasattr(model, "config") and key in model.config.model_type:
+ is_special = True
+ break
+ return bool(is_special)
+
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 60ca6d99c..5fb597dd7 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -346,7 +346,9 @@ def collect_best_params(block):
@torch.no_grad()
def sampling_inputs(input_ids, input_others, indices, seqlen,
- share_attention_mask_flag=False, not_share_position_ids_flag=False, input_dim=0):
+ share_attention_mask_flag=False,
+ not_share_position_ids_flag=False,
+ not_share_rotary_pos_emb_flag=False, input_dim=0):
"""Samples inputs based on the given indices and sequence length.
Args:
@@ -361,15 +363,16 @@ def sampling_inputs(input_ids, input_others, indices, seqlen,
"""
current_input_ids = [input_ids[i] for i in indices]
current_input_ids = torch.cat(current_input_ids, dim=input_dim)
-
current_input_others = {"positional_inputs": input_others["positional_inputs"]}
for key in input_others.keys():
if not share_attention_mask_flag and ("attention_mask" in key or "alibi" in key) \
- or (not_share_position_ids_flag and "position_ids" in key):
+ or (not_share_position_ids_flag and ("position_ids" in key or "cache_position" in key)) \
+ or (not_share_rotary_pos_emb_flag and ("rotary_pos_emb" in key or 'cu_seqlens' in key)):
current_input_others[key] = None
if input_others[key] is not None:
current_input_others[key] = [input_others[key][i] for i in indices]
- current_input_others[key] = torch.cat(current_input_others[key], dim=0)
+ if not isinstance(current_input_others[key], torch.Tensor):
+ current_input_others[key] = torch.cat(current_input_others[key], dim=0)
else:
current_input_others[key] = input_others[key]
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 48ed2770f..3d514e75e 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -94,10 +94,10 @@ if __name__ == '__main__':
parser.add_argument("--enable_minmax_tuning", action='store_true',
help="enable_minmax_tuning is deprecated")
- parser.add_argument("--deployment_device", default=None, type=str,
- help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'gpu' and 'xpu'."
- "default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")
-
+ parser.add_argument("--deployment_device", default='auto_round', type=str,
+ help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'xpu', 'gpu(auto_gptq)' and 'auto_round'."
+ "default to 'auto_round', 'fake' indicating that it only performs fake quantization and won't be exported to any device.")
+
parser.add_argument("--format", default=None, type=str,
help="The format in which to save the model. "
"The options are 'auto_round', 'auto_gptq', 'auto_awq', 'itrex', 'itrex_xpu' and 'fake'."
@@ -387,6 +387,7 @@ if __name__ == '__main__':
tokenizer.save_pretrained(output_dir)
if eval_folder is None:
eval_folder = output_dir
+
if (not ('gpu' in deployment_device or len(gpu_formats) > 0)) and 'fake' not in deployment_device:
print('does not support cpu, xpu model evaluation.')
exit() ## does not support cpu,xpu model eval
@@ -454,3 +455,4 @@ if __name__ == '__main__':
from lm_eval.utils import make_table
print(make_table(res))
+
diff --git a/examples/multimodal-modeling/Phi-3-vision/README.md b/examples/multimodal-modeling/Phi-3-vision/README.md
index b0557bae2..1ca9e3340 100644
--- a/examples/multimodal-modeling/Phi-3-vision/README.md
+++ b/examples/multimodal-modeling/Phi-3-vision/README.md
@@ -61,7 +61,7 @@ import requests
import io
from transformers import AutoModelForCausalLM
from transformers import AutoProcessor
-from auto_round.auto_quantizer import AutoHfQuantizer
+from auto_round import AutoRoundConfig
quantized_model_path = "./tmp_autoround"
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True, torch_dtype="auto", _attn_implementation='flash_attention_2') # use _attn_implementation='eager' to disable flash attention
@@ -137,3 +137,4 @@ If you find SignRound useful for your research, please cite our paper:
+
diff --git a/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py b/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py
index dd440f106..5aa57ae7a 100644
--- a/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py
+++ b/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py
@@ -597,11 +597,14 @@ if __name__ == "__main__":
model_args = f"pretrained={args.model_name}"
if hasattr(config, "quantization_config"):
quantization_config = config.quantization_config
- if "quant_method" in quantization_config and "auto-round" in quantization_config["quant_method"]:
- from auto_round.auto_quantizer import AutoHfQuantizer
- else:
- from phi3_v import Phi3VGPTQForCausalLM
- model_args += f",autogptq=True,gptq_use_triton=True"
+ if "quant_method" in quantization_config and ("auto-round" in quantization_config["quant_method"] or
+ ("gptq" in quantization_config["quant_method"] and args.device == "hpu")):
+ try:
+ from auto_round import AutoRoundConfig
+ except:
+ from auto_round.auto_quantizer import AutoHfQuantizer
+
+ model_args += f",autogptq=True,gptq_use_triton=True"
if args.trust_remote_code:
model_args += f",trust_remote_code=True"
model_args += ",dtype=bfloat16"
@@ -621,3 +624,4 @@ if __name__ == "__main__":
print("cost time: ", time.time() - s)
+
diff --git a/examples/multimodal-modeling/Qwen-VL/README.md b/examples/multimodal-modeling/Qwen-VL/README.md
index 4980eb28a..9882f4fb1 100644
--- a/examples/multimodal-modeling/Qwen-VL/README.md
+++ b/examples/multimodal-modeling/Qwen-VL/README.md
@@ -4,7 +4,7 @@ Step-by-Step
This document presents step-by-step instructions for auto-round.
# Run Quantization on Qwen-VL Models
-In this example, we introduce an straight-forward way to execute quantization on some popular multimodal models such as Qwen-VL.
+In this example, we introduce an straight-forward way to execute quantization on some popular multimodal models such as Qwen-VL and Qwen2-VL-Instruct.
## Download the calibration data
@@ -75,6 +75,7 @@ Enter into the examples folder and install requirements
```bash
pip install -r requirements.txt
```
+For Qwen2-VL quantization/Inference, please follow [Qwen2 official requirements](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct)
- **Default Settings:**
```bash
@@ -105,13 +106,15 @@ bash run_autoround.sh
## 3. run inference
+- Qwen-VL inference
+
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
from transformers.generation import GenerationConfig
import torch
from transformers import set_seed
set_seed(1234)
- from auto_round.auto_quantizer import AutoHfQuantizer
+ from auto_round import AutoRoundConfig
quantized_model_path = "./tmp_autoround"
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, trust_remote_code=True)
# use bf16
@@ -141,6 +144,62 @@ bash run_autoround.sh
```
+- Qwen2-VL-7B-Instruct inference
+
+```python
+from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
+from qwen_vl_utils import process_vision_info
+from auto_round import AutoRoundConfig
+quantized_model_path="./tmp_autoround"
+model = Qwen2VLForConditionalGeneration.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)
+processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct")
+messages = [{
+ "role": "user",
+ "content": [
+ {
+ "type": "image",
+ "image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg",
+ },
+ {"type": "text", "text": "Describe this image."},]
+}]
+# Preparation for inference
+text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+image_inputs, video_inputs = process_vision_info(messages)
+inputs = processor(
+ text=[text],
+ images=image_inputs,
+ videos=video_inputs,
+ padding=True,
+ return_tensors="pt",
+)
+inputs = inputs.to("cuda")
+
+# Inference: Generation of the output
+generated_ids = model.generate(**inputs, max_new_tokens=50)
+generated_ids_trimmed = [
+ out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+]
+output_text = processor.batch_decode(
+ generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+)
+print(output_text)
+# The image depicts a serene beach scene at sunset. A woman is sitting on the sand, facing a large dog that appears to be a Labrador Retriever. The dog is wearing a harness and is extending its paw towards the woman's hand, possibly
+
+# messages = [{
+# "role": "user",
+# "content": [
+# {
+# "type": "image",
+# "image": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg",
+# },
+# {"type": "text", "text": "What does the label 15 represent? (1) lava (2) core (3) tunnel (4) ash cloud"},]
+# }]
+
+# The label 15 represents an ash cloud. In the context of a volcano, an ash cloud is formed when volcanic ash is ejected into the atmosphere during an eruption. Therefore, the correct answer is:\n\n(4) ash cloud
+
+```
+
+
## 4. Results
Using [COCO 2017](https://cocodataset.org/) and [LLaVA-Instruct-150K](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K) datasets for quantization calibration, and TextVQA dataset for evaluation. please follow the [recipe](./run_autoround.sh) and [evaluate script](./run_eval.sh). The results for Qwen-VL are as follows:
| Metric | bf16 | INT4 |
@@ -163,7 +222,6 @@ Using [COCO 2017](https://cocodataset.org/) and [LLaVA-Instruct-150K](https://hu
| scienceVQA | 0.6748 | 0.6574 |
-
## 5. Environment
PyTorch 1.8 or higher version is needed
@@ -188,3 +246,4 @@ If you find SignRound useful for your research, please cite our paper:
+
diff --git a/examples/multimodal-modeling/Qwen-VL/main.py b/examples/multimodal-modeling/Qwen-VL/main.py
index 0c5f35bf4..fd7d0a19e 100644
--- a/examples/multimodal-modeling/Qwen-VL/main.py
+++ b/examples/multimodal-modeling/Qwen-VL/main.py
@@ -22,8 +22,9 @@ DEFAULT_IM_END_TOKEN = '</img>'
from transformers.trainer_pt_utils import LabelSmoother
IGNORE_TOKEN_ID = LabelSmoother.ignore_index
import inspect
+from PIL import Image
-def DataFormating(raw_data, image_folder):
+def DataFormating(raw_data, image_folder=None, model_type='qwen'):
for source in raw_data:
source_inputs = source['conversations']
for sentence in source_inputs:
@@ -31,23 +32,67 @@ def DataFormating(raw_data, image_folder):
sentence['from'] = sentence['from'].replace('gpt', 'assistant')
if OLD_IMAGE_TOKEN in sentence['value']:
sentence['value'] = sentence['value'].replace(OLD_IMAGE_TOKEN, '').strip()
- sentence['value'] = OLD_IMAGE_TOKEN + '\n' + sentence['value']
+ sentence['value'] = OLD_IMAGE_TOKEN + sentence['value']
sentence['value'] = sentence['value'].strip()
- replace_img = os.path.join(image_folder, os.path.basename(source["image"]))
- replace_token = DEFAULT_IM_START_TOKEN + replace_img + DEFAULT_IM_END_TOKEN
+ if 'qwen2' in model_type: # for Qwen2-vl
+ replace_token = '<|vision_start|><|image_pad|><|vision_end|>'
+ else:
+ replace_img = os.path.join(image_folder, os.path.basename(source["image"]))
+ replace_token = DEFAULT_IM_START_TOKEN + replace_img + DEFAULT_IM_END_TOKEN + '\n'
sentence["value"] = sentence["value"].replace(OLD_IMAGE_TOKEN, replace_token)
return raw_data
-def preprocess(
+
+def qwen2_preprocess(
sources,
tokenizer: transformers.PreTrainedTokenizer,
max_len: int,
system_message: str = "You are a helpful assistant."
) -> Dict:
roles = {"user": "<|im_start|>user", "assistant": "<|im_start|>assistant"}
+ im_start = "<|im_start|>"
+ im_end = "<|im_end|>"
+ nl_tokens = '\n'
+ _system = 'system' + nl_tokens
+
+ # Apply prompt templates
+ inputs, targets = [], []
+ for i, source in enumerate(sources):
+ if roles[source[0]["from"]] != roles["user"]:
+ source = source[1:]
+
+ text, target = "", None
+ system = im_start + _system + system_message + im_end + nl_tokens
+ text += system
+ for j, sentence in enumerate(source):
+ role = roles[sentence["from"]]
+ _text = role + nl_tokens + \
+ sentence["value"] + im_end + nl_tokens
+ text += _text
+ token_length = len(tokenizer(text).input_ids)
+ if token_length < max_len:
+ text += tokenizer.pad_token * (max_len - token_length)
+ else:
+ text = tokenizer.decode(tokenizer(text).input_ids[:max_len])
+ pass
+ inputs.append(text)
+
+ return inputs
- im_start = tokenizer.im_start_id
- im_end = tokenizer.im_end_id
+
+def preprocess(
+ sources,
+ tokenizer: transformers.PreTrainedTokenizer,
+ max_len: int,
+ system_message: str = "You are a helpful assistant."
+) -> Dict:
+ roles = {"user": "<|im_start|>user", "assistant": "<|im_start|>assistant"}
+ if 'qwen2' not in model_type:
+ im_start = tokenizer.im_start_id
+ im_end = tokenizer.im_end_id
+ else:
+ im_start = tokenizer('<|im_start|>')
+ im_end = tokenizer('<|im_end|>')
nl_tokens = tokenizer('\n').input_ids
_system = tokenizer('system').input_ids + nl_tokens
_user = tokenizer('user').input_ids + nl_tokens
@@ -82,51 +127,25 @@ def preprocess(
target += [IGNORE_TOKEN_ID] * (max_len - len(target))
input_ids.append(input_id[:max_len])
targets.append(target[:max_len])
- if i >= 512:
- break
input_ids = torch.tensor(input_ids, dtype=torch.int)
targets = torch.tensor(targets, dtype=torch.int)
-
+
return dict(
input_ids=input_ids,
labels=targets,
attention_mask=input_ids.ne(tokenizer.pad_token_id),
)
-
-class SupervisedDataset(Dataset):
- """Dataset for supervised fine-tuning."""
-
- def __init__(self, raw_data, tokenizer: transformers.PreTrainedTokenizer, max_len: int):
- super(SupervisedDataset, self).__init__()
-
- print("Formatting inputs...")
- sources = [example["conversations"] for example in raw_data]
- data_dict = preprocess(sources, tokenizer, max_len)
-
- self.input_ids = data_dict["input_ids"]
- self.labels = data_dict["labels"]
- self.attention_mask = data_dict["attention_mask"]
-
- def __len__(self):
- return len(self.input_ids)
- def __getitem__(self, i) -> Dict[str, torch.Tensor]:
- return dict(
- input_ids=self.input_ids[i],
- labels=self.labels[i],
- attention_mask=self.attention_mask[i],
- )
-
class LazySupervisedDataset(Dataset):
"""Dataset for supervised fine-tuning."""
- def __init__(self, raw_data, tokenizer: transformers.PreTrainedTokenizer, max_len: int):
+ def __init__(self, raw_data, tokenizer: transformers.PreTrainedTokenizer,
+ max_len: int, image_folder=None, model_type='qwen_vl'):
super(LazySupervisedDataset, self).__init__()
self.tokenizer = tokenizer
self.max_len = max_len
-
+ self.image_folder = image_folder
print("Formatting inputs...Skip in lazy mode")
- self.tokenizer = tokenizer
self.raw_data = raw_data
self.cached_data_dict = {}
@@ -137,12 +156,31 @@ class LazySupervisedDataset(Dataset):
if i in self.cached_data_dict:
return self.cached_data_dict[i]
- ret = preprocess([self.raw_data[i]["conversations"]], self.tokenizer, self.max_len)
- ret = dict(
- input_ids=ret["input_ids"][0],
- labels=ret["labels"][0],
- attention_mask=ret["attention_mask"][0],
- )
+ if 'qwen2' not in model_type:
+ ret = preprocess([self.raw_data[i]["conversations"]], self.tokenizer, self.max_len)
+ ret = dict(
+ input_ids=ret["input_ids"][0],
+ labels=ret["labels"][0],
+ attention_mask=ret["attention_mask"][0],
+ )
+ else:
+ texts = qwen2_preprocess([self.raw_data[i]["conversations"]], self.tokenizer, self.max_len)
+ image_path = os.path.join(f"file://{self.image_folder}", os.path.basename(self.raw_data[i]["image"]))
+ image_inputs = fetch_image({'image':image_path})
+ ret = self.tokenizer.processor(
+ text=texts,
+ images=image_inputs,
+ videos=None,
+ padding=True,
+ return_tensors="pt",
+ )
+ ret = dict(
+ input_ids=ret["input_ids"][0],
+ # labels=ret["labels"][0],
+ attention_mask=ret["attention_mask"][0],
+ image_grid_thw=ret["image_grid_thw"][0],
+ pixel_values=ret["pixel_values"],
+ )
self.cached_data_dict[i] = ret
return ret
@@ -160,7 +198,7 @@ def set_signature_columns_if_needed(model):
# Labels may be named label or label_ids, the default data collator handles that.
signature_columns += list(set(["label", "label_ids", 'labels']))
return signature_columns
-
+
def get_collator_with_removed_columns(model, data_collator: Callable, description: Optional[str] = None
) -> Callable:
"""Wrap the data collator in a callable removing unused columns."""
@@ -174,7 +212,8 @@ def get_collator_with_removed_columns(model, data_collator: Callable, descriptio
)
return remove_columns_collator
-def get_train_dataloader(train_dataset, model, data_collator, train_batch_size=1, num_workers=0) -> DataLoader:
+def get_train_dataloader(train_dataset, model, data_collator=default_data_collator,
+ train_batch_size=1, num_workers=0) -> DataLoader:
"""
Returns the training [`~torch.utils.data.DataLoader`].
@@ -186,7 +225,8 @@ def get_train_dataloader(train_dataset, model, data_collator, train_batch_size=1
if train_dataset is None:
raise ValueError("Trainer: training requires a train_dataset.")
- data_collator = get_collator_with_removed_columns(model, data_collator, description="training")
+ if data_collator != default_data_collator:
+ data_collator = get_collator_with_removed_columns(model, data_collator, description="training")
dataloader_params = {
"batch_size": train_batch_size,
@@ -240,7 +280,7 @@ if __name__ == '__main__':
parser.add_argument("--seqlen", default=512, type=int,
help="sequence length")
- parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")
+ parser.add_argument("--gradient_accumulate_steps", default=8, type=int, help="gradient accumulate steps")
parser.add_argument("--nblocks", default=1, type=int, help="num of blocks to tune together")
@@ -331,10 +371,8 @@ if __name__ == '__main__':
torch.manual_seed(1234)
model_name = args.model_name
- questions = json.load(open(args.question_file, "r"))
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code,
padding_side="right", use_fast=False)
- tokenizer.pad_token_id = tokenizer.eod_id
seqlen = args.seqlen
if hasattr(tokenizer, "model_max_length"):
if tokenizer.model_max_length < seqlen:
@@ -343,28 +381,40 @@ if __name__ == '__main__':
seqlen = min(seqlen, tokenizer.model_max_length)
args.seqlen = seqlen
- config = transformers.AutoConfig.from_pretrained(
- model_name,
- trust_remote_code=True,
- )
- config.use_cache = False
+ torch_dtype = "auto"
if args.model_dtype != None:
if args.model_dtype == "float16" or args.model_dtype == "fp16":
torch_dtype = torch.float16
if args.model_dtype == "bfloat16" or args.model_dtype == "bf16":
torch_dtype = torch.bfloat16
+
dtype_str = convert_dtype_torch2str(torch_dtype)
- if dtype_str == "bf16":
- model = AutoModelForCausalLM.from_pretrained(args.model_name, config=config, trust_remote_code=not args.disable_trust_remote_code, bf16=True).eval()
- elif dtype_str == "fp16":
- model = AutoModelForCausalLM.from_pretrained(args.model_name, config=config, trust_remote_code=not args.disable_trust_remote_code, fp16=True).eval()
- else:
- model = AutoModelForCausalLM.from_pretrained(args.model_name, config=config, trust_remote_code=not args.disable_trust_remote_code).eval()
- raw_data = DataFormating(questions, args.image_folder)
- # dataset = SupervisedDataset(raw_data, tokenizer, max_len=tokenizer.model_max_length)
- dataset = LazySupervisedDataset(raw_data, tokenizer, max_len=min(args.seqlen, tokenizer.model_max_length))
- default_collator = default_data_collator if tokenizer is None else DataCollatorWithPadding(tokenizer)
- dataloader = get_train_dataloader(dataset, model, default_collator, train_batch_size=args.train_bs)
+ questions = json.load(open(args.question_file, "r"))
+ config = transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
+ model_type = config.model_type
+ if 'qwen2' not in model_type: # for Qwen-VL/Qwen-VL-Chat
+ tokenizer.pad_token_id = tokenizer.eod_id
+ config.use_cache = False
+ if dtype_str == "bf16":
+ model = AutoModelForCausalLM.from_pretrained(args.model_name, trust_remote_code=not args.disable_trust_remote_code, bf16=True).eval()
+ elif dtype_str == "fp16":
+ model = AutoModelForCausalLM.from_pretrained(args.model_name, trust_remote_code=not args.disable_trust_remote_code, fp16=True).eval()
+ else:
+ model = AutoModelForCausalLM.from_pretrained(args.model_name, trust_remote_code=not args.disable_trust_remote_code).eval()
+ raw_data = DataFormating(questions, args.image_folder)
+ default_collator = default_data_collator if tokenizer is None else DataCollatorWithPadding(tokenizer)
+ else: # for Qwen2-VL-instruct
+ from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
+ from qwen_vl_utils import process_vision_info, fetch_image
+ model = Qwen2VLForConditionalGeneration.from_pretrained(args.model_name, torch_dtype=torch_dtype)
+ processor = AutoProcessor.from_pretrained(args.model_name)
+ tokenizer.processor = processor
+ default_collator = default_data_collator
+
+ raw_data = DataFormating(questions, args.image_folder, model_type=model_type)
+ dataset = LazySupervisedDataset(raw_data, tokenizer,
+ max_len=min(args.seqlen, tokenizer.model_max_length), image_folder=args.image_folder)
+ dataloader = get_train_dataloader(dataset, model, data_collator=default_collator, train_batch_size=args.train_bs)
from auto_round import (AutoRound,
AutoAdamRound)
diff --git a/examples/multimodal-modeling/Qwen-VL/mm_evaluation/main.py b/examples/multimodal-modeling/Qwen-VL/mm_evaluation/main.py
index 4e1446705..50b26e67f 100644
--- a/examples/multimodal-modeling/Qwen-VL/mm_evaluation/main.py
+++ b/examples/multimodal-modeling/Qwen-VL/mm_evaluation/main.py
@@ -45,10 +45,11 @@ if __name__ == "__main__":
if hasattr(config, "quantization_config"):
quantization_config = config.quantization_config
- if "quant_method" in quantization_config and "auto-round" in quantization_config["quant_method"]:
- from auto_round.auto_quantizer import AutoHfQuantizer
- elif "quant_method" in quantization_config and quantization_config["quant_method"] == "gptq":
- if args.device == "hpu":
+ if "quant_method" in quantization_config and ("auto-round" in quantization_config["quant_method"] or
+ ("gptq" in quantization_config["quant_method"] and args.device == "hpu")):
+ try:
+ from auto_round import AutoRoundConfig
+ except:
from auto_round.auto_quantizer import AutoHfQuantizer
model_name = args.model_name
torch_dtype = torch.float
@@ -99,3 +100,4 @@ if __name__ == "__main__":
print("cost time: ", time.time() - s)
+