Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

enable llama3.2-vision model quantization (#269)

WeiweiZhang16b99d10a3c0e
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index ecd026435..44cbf92e6 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -704,6 +704,7 @@ class AutoRound(object):
for key in kwargs.keys():
if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) \
+ or isinstance(kwargs[key], tuple) \
or (key == "alibi") or (key == "attention_mask"):
if "attention_mask" in key:
if key not in self.inputs[name].keys():
@@ -732,8 +733,8 @@ class AutoRound(object):
self.inputs[name][key].append(to_device(kwargs[key], device=torch.device("cpu")))
elif key not in self.inputs[name].keys():
self.inputs[name][key] = list(torch.split(kwargs[key].to("cpu"), 1, dim=0)) \
- if self.not_share_position_ids_flag \
- else to_device(kwargs[key], device=torch.device("cpu"))
+ if self.not_share_position_ids_flag \
+ else to_device(kwargs[key], device=torch.device("cpu"))
elif kwargs[key] is not None and self.not_share_position_ids_flag:
self.inputs[name][key].extend(list(torch.split(kwargs[key].to("cpu"), 1, dim=0)))
elif 'rotary_pos_emb' in key or 'cu_seqlens' in key:
@@ -743,8 +744,14 @@ class AutoRound(object):
else to_device(kwargs[key], device=torch.device("cpu"))
elif kwargs[key] is not None and self.not_share_rotary_pos_emb_flag:
self.inputs[name][key].append(to_device(kwargs[key], device=torch.device("cpu")))
+ elif "cross_attention_states" in key:
+ if key not in self.inputs[name].keys():
+ self.inputs[name][key] = [to_device(kwargs[key], device=torch.device("cpu"))]
+ else:
+ self.inputs[name][key].extend(list(torch.split(kwargs[key].to("cpu"), 1, dim=0)))
elif key not in self.inputs[name].keys():
self.inputs[name][key] = to_device(kwargs[key], device=torch.device("cpu"))
+
if name == self.last_cache_name:
raise NotImplementedError
else:
@@ -1617,3 +1624,4 @@ class AutoAdamRound(AutoOPTRound):
optimizer,
**kwargs,
)
+
diff --git a/auto_round/utils.py b/auto_round/utils.py
index c0354318f..064bed6eb 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -367,7 +367,8 @@ def sampling_inputs(input_ids, input_others, indices, seqlen,
for key in input_others.keys():
if not share_attention_mask_flag and ("attention_mask" in key or "alibi" in key) \
or (not_share_position_ids_flag and ("position_ids" in key or "cache_position" in key)) \
- or (not_share_rotary_pos_emb_flag and ("rotary_pos_emb" in key or 'cu_seqlens' in key)):
+ or (not_share_rotary_pos_emb_flag and ("rotary_pos_emb" in key or 'cu_seqlens' in key)) \
+ or "cross_attention_states" in key:
current_input_others[key] = None
if input_others[key] is not None:
current_input_others[key] = [input_others[key][i] for i in indices]
@@ -970,3 +971,4 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):
use_marlin=not disable_marlin,
)
return QuantLinear
+
diff --git a/examples/multimodal-modeling/Qwen-VL/README.md b/examples/multimodal-modeling/Qwen-VL/README.md
index 9882f4fb1..d49b2c2ba 100644
--- a/examples/multimodal-modeling/Qwen-VL/README.md
+++ b/examples/multimodal-modeling/Qwen-VL/README.md
@@ -1,8 +1,8 @@
Step-by-Step
============
-
This document presents step-by-step instructions for auto-round.
-# Run Quantization on Qwen-VL Models
+
+# Run Quantization on Qwen-VL/Qwen2-VL-Instruct Models
In this example, we introduce an straight-forward way to execute quantization on some popular multimodal models such as Qwen-VL and Qwen2-VL-Instruct.
@@ -17,8 +17,9 @@ COCO: [train2017](http://images.cocodataset.org/zips/train2017.zip), and unzip t
You can also refer to the official Qwen-VL finetuning requirements to create a [custom dataset](https://github.com/QwenLM/Qwen-VL/blob/master/README.md#data-preparation)
## Download the evaluation data
-
Please refer to [Qwen-VL evaluation](https://github.com/cognitedata/Qwen-VL-finetune/blob/master/eval_mm/EVALUATION.md)
+
+Note: The quantized Qwen2-VL model evaluation is currently not supported.
<details>
<summary>TextVQA Data Preparation</summary>
@@ -71,15 +72,18 @@ cd ../..
<br />
## 2. Run Examples
+
Enter into the examples folder and install requirements
+
```bash
pip install -r requirements.txt
```
-For Qwen2-VL quantization/Inference, please follow [Qwen2 official requirements](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct)
+
+For Qwen2-VL quantization/Inference, please follow [Qwen2 official requirements](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct) and install the latest version of transformers by **building from source**.
- **Default Settings:**
```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Qwen/Qwen-VL --bits 4 --group_size 128
+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Qwen/Qwen2-VL-7B-Instruct --bits 4 --group_size 128
```
- **Speedup the tuning:**
@@ -88,17 +92,6 @@ reduce the seqlen to 512(potential large accuracy drop)
or combine them
-- **Enable quantized lm-head:**
-
-Currently only support in Intel xpu and AutoRound format,however, we found the fake tuning could improve the accuracy is some scenarios. low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing nsamples,e.g. 128, to alleviate the issue.
-```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Qwen/Qwen-VL --bits 4 --group_size 128 --quant_lm_head
-```
-
-- **Utilizing the AdamW Optimizer:**
-
-Include the flag `--adam`. Note that AdamW is less effective than sign gradient descent in many scenarios we tested.
-
- **Running on Intel Gaudi2**
```bash
bash run_autoround.sh
@@ -247,3 +240,4 @@ If you find SignRound useful for your research, please cite our paper:
+
diff --git a/examples/multimodal-modeling/Qwen-VL/main.py b/examples/multimodal-modeling/Qwen-VL/main.py
index fd7d0a19e..b6d0f7ac3 100644
--- a/examples/multimodal-modeling/Qwen-VL/main.py
+++ b/examples/multimodal-modeling/Qwen-VL/main.py
@@ -1,10 +1,9 @@
import argparse
import sys
-
sys.path.insert(0, '../../..')
parser = argparse.ArgumentParser()
-import torch
import os
+import torch
import transformers
os.environ["TOKENIZERS_PARALLELISM"] = "false"
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
@@ -12,8 +11,7 @@ torch.use_deterministic_algorithms(True, warn_only=True)
from transformers import set_seed
import json
from torch.utils.data import Dataset, DataLoader
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import torch
+from transformers import AutoTokenizer, AutoProcessor, AutoModelForCausalLM
from auto_round.utils import convert_dtype_torch2str
from typing import Dict, Optional, List
OLD_IMAGE_TOKEN = '<image>'
@@ -36,6 +34,8 @@ def DataFormating(raw_data, image_folder=None, model_type='qwen'):
sentence['value'] = sentence['value'].strip()
if 'qwen2' in model_type: # for Qwen2-vl
replace_token = '<|vision_start|><|image_pad|><|vision_end|>'
+ if 'mllama' in model_type:
+ replace_token = '<|image|>'
else:
replace_img = os.path.join(image_folder, os.path.basename(source["image"]))
replace_token = DEFAULT_IM_START_TOKEN + replace_img + DEFAULT_IM_END_TOKEN + '\n'
@@ -43,17 +43,25 @@ def DataFormating(raw_data, image_folder=None, model_type='qwen'):
return raw_data
-def qwen2_preprocess(
+def common_preprocess(
sources,
tokenizer: transformers.PreTrainedTokenizer,
max_len: int,
- system_message: str = "You are a helpful assistant."
+ system_message: str = "You are a helpful assistant.",
+ model_type='qwen2'
) -> Dict:
- roles = {"user": "<|im_start|>user", "assistant": "<|im_start|>assistant"}
- im_start = "<|im_start|>"
- im_end = "<|im_end|>"
+ if 'mllama' in model_type:
+ roles = {"user": "<|start_header_id|>user<|end_header_id|>\n", "assistant": "<|start_header_id|>assistant<|end_header_id|>\n"}
+ im_start = "<|start_header_id|>"
+ im_end = "<|end_header_id|>\n"
+ im_dot = '<|eot_id|>'
+ text_start = '<|begin_of_text|>'
+ else :
+ roles = {"user": "<|im_start|>user", "assistant": "<|im_start|>assistant"}
+ im_start = "<|im_start|>"
+ im_end = "<|im_end|>"
nl_tokens = '\n'
- _system = 'system' + nl_tokens
+ _system = 'system'
# Apply prompt templates
inputs, targets = [], []
@@ -62,12 +70,19 @@ def qwen2_preprocess(
source = source[1:]
text, target = "", None
- system = im_start + _system + system_message + im_end + nl_tokens
+ if 'mllama' in model_type:
+ system = text_start + im_start + _system + im_end + nl_tokens + system_message + im_dot
+ else:
+ system = im_start + _system + nl_tokens + system_message + im_end + nl_tokens
text += system
for j, sentence in enumerate(source):
role = roles[sentence["from"]]
- _text = role + nl_tokens + \
- sentence["value"] + im_end + nl_tokens
+ if 'mllama' in model_type:
+ _text = role + nl_tokens + \
+ sentence["value"] + im_dot
+ else:
+ _text = role + nl_tokens + \
+ sentence["value"] + im_end + nl_tokens
text += _text
token_length = len(tokenizer(text).input_ids)
if token_length < max_len:
@@ -156,33 +171,46 @@ class LazySupervisedDataset(Dataset):
if i in self.cached_data_dict:
return self.cached_data_dict[i]
- if 'qwen2' not in model_type:
+ if 'qwen' == model_type: # for Qwen-VL
ret = preprocess([self.raw_data[i]["conversations"]], self.tokenizer, self.max_len)
ret = dict(
input_ids=ret["input_ids"][0],
labels=ret["labels"][0],
attention_mask=ret["attention_mask"][0],
)
- else:
- texts = qwen2_preprocess([self.raw_data[i]["conversations"]], self.tokenizer, self.max_len)
- image_path = os.path.join(f"file://{self.image_folder}", os.path.basename(self.raw_data[i]["image"]))
- image_inputs = fetch_image({'image':image_path})
+ else: # Qwen2-VL and Llama-3.2
+ texts = common_preprocess([self.raw_data[i]["conversations"]], self.tokenizer, self.max_len, model_type=model_type)
+ if 'qwen2' in model_type:
+ image_path = os.path.join(f"file://{self.image_folder}", os.path.basename(self.raw_data[i]["image"]))
+ image = fetch_image({'image':image_path})
+ else:
+ image = Image.open(os.path.join(self.image_folder, os.path.basename(self.raw_data[i]["image"]))) #.convert('RGB')
ret = self.tokenizer.processor(
text=texts,
- images=image_inputs,
- videos=None,
+ images=image,
padding=True,
+ truncation=True,
return_tensors="pt",
+ # videos=None,
)
- ret = dict(
- input_ids=ret["input_ids"][0],
- # labels=ret["labels"][0],
- attention_mask=ret["attention_mask"][0],
- image_grid_thw=ret["image_grid_thw"][0],
- pixel_values=ret["pixel_values"],
- )
+ if 'qwen2' in model_type:
+ ret = dict(
+ input_ids=ret["input_ids"][0],
+ # labels=ret["labels"][0],
+ attention_mask=ret["attention_mask"][0],
+ image_grid_thw=ret["image_grid_thw"][0],
+ pixel_values=ret["pixel_values"],
+ )
+ else:
+ ret = dict(
+ input_ids=ret["input_ids"][0],
+ attention_mask=ret["attention_mask"][0],
+ aspect_ratio_ids=ret["aspect_ratio_ids"][0],
+ aspect_ratio_mask=ret["aspect_ratio_mask"][0],
+ cross_attention_mask=ret["cross_attention_mask"][0],
+ pixel_values=ret["pixel_values"][0],
+ )
self.cached_data_dict[i] = ret
-
return ret
@@ -340,7 +368,7 @@ if __name__ == '__main__':
# ================= Evaluation Related =====================
# parser.add_argument("--eval-path", type=str, default=None)
- parser.add_argument("--eval-dataset", type=str, default="textvqa_val,scienceqa_test_img")
+ parser.add_argument("--eval_dataset", type=str, default="textvqa_val,scienceqa_test_img")
args = parser.parse_args()
@@ -392,7 +420,14 @@ if __name__ == '__main__':
questions = json.load(open(args.question_file, "r"))
config = transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
model_type = config.model_type
- if 'qwen2' not in model_type: # for Qwen-VL/Qwen-VL-Chat
+ if "mllama" in model_type:
+ from transformers import MllamaForConditionalGeneration
+ model = MllamaForConditionalGeneration.from_pretrained(args.model_name,
+ trust_remote_code=not args.disable_trust_remote_code) # torch_dtype=torch.bfloat16
+ processor = AutoProcessor.from_pretrained(args.model_name)
+ tokenizer.processor = processor
+ default_collator = default_data_collator
+ elif 'qwen2' not in model_type: # for Qwen-VL/Qwen-VL-Chat
tokenizer.pad_token_id = tokenizer.eod_id
config.use_cache = False
if dtype_str == "bf16":
@@ -401,7 +436,6 @@ if __name__ == '__main__':
model = AutoModelForCausalLM.from_pretrained(args.model_name, trust_remote_code=not args.disable_trust_remote_code, fp16=True).eval()
else:
model = AutoModelForCausalLM.from_pretrained(args.model_name, trust_remote_code=not args.disable_trust_remote_code).eval()
- raw_data = DataFormating(questions, args.image_folder)
default_collator = default_data_collator if tokenizer is None else DataCollatorWithPadding(tokenizer)
else: # for Qwen2-VL-instruct
from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
@@ -410,7 +444,7 @@ if __name__ == '__main__':
processor = AutoProcessor.from_pretrained(args.model_name)
tokenizer.processor = processor
default_collator = default_data_collator
-
+
raw_data = DataFormating(questions, args.image_folder, model_type=model_type)
dataset = LazySupervisedDataset(raw_data, tokenizer,
max_len=min(args.seqlen, tokenizer.model_max_length), image_folder=args.image_folder)
@@ -518,7 +552,7 @@ if __name__ == '__main__':
if eval_folder is None:
eval_folder = output_dir
- if not args.disable_eval and "fake" in deployment_device: ## TODO
+ if not args.disable_eval and "fake" in deployment_device and model_type == "qwen": ## for Qwen-VL
model = model.half()
model = model.to(torch_device)
datasets=args.eval_dataset.split(',')