Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix merge error (#316)

Heng Guo606a5c35b46f
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index ffa68d39c..a615a99bb 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -117,6 +117,7 @@ class AutoRoundMLLM(AutoRound):
act_dynamic: bool = True,
quant_block_list: list = None,
enable_norm_bias_tuning: bool = False,
+ truncation: bool = False,
enable_torch_compile: bool = None,
**kwargs,
):
@@ -128,6 +129,7 @@ class AutoRoundMLLM(AutoRound):
self.template = template if template is not None else model.config.model_type
self.template = get_template(
self.template, model=model, tokenizer=tokenizer, image_processor=image_processor)
+ self.truncation = truncation
assert dataset is not None, "dataset should not be None"
batch_size, gradient_accumulate_steps = check_mllm_model_batch(model, batch_size, gradient_accumulate_steps)
@@ -193,7 +195,8 @@ class AutoRoundMLLM(AutoRound):
extra_data_dir=self.extra_data_dir,
seqlen=self.seqlen,
bs=bs,
- seed=self.seed
+ seed=self.seed,
+ truncation=self.truncation,
)
else:
self.dataloader = self.dataset
diff --git a/auto_round/mllm/mllm_dataset.py b/auto_round/mllm/mllm_dataset.py
index cf319fbf0..dae71fcad 100644
--- a/auto_round/mllm/mllm_dataset.py
+++ b/auto_round/mllm/mllm_dataset.py
@@ -57,6 +57,7 @@ class LlavaDataset(Dataset):
"llava_instruct_150k": BASE_LLAVA_URL + "llava_instruct_150k.json?download=true",
}
_COCO_DATA_URL = "http://images.cocodataset.org/train2017/"
+ IMAGE_TOKEN = "<image>"
def __init__(
self,
@@ -101,16 +102,18 @@ class LlavaDataset(Dataset):
self.image_fold = image_fold
- @staticmethod
- def check(questions, seqlen):
+ def check(self, questions, seqlen):
new_questions = []
for source in questions:
text_lenght = 0
for text in source['conversations']:
+ if self.IMAGE_TOKEN in text['value']:
+ text['value'] = self.IMAGE_TOKEN + text['value'].replace(self.IMAGE_TOKEN, '')
text_lenght += len(text['value'].split(' '))
if text_lenght >= seqlen:
new_questions.append(source)
- assert len(new_questions) > 0, f"no data with length greater than {seqlen}, please check"
+ assert len(new_questions) > 0, \
+ f"no data with length greater than {seqlen}, please reduce the seqlen or change another dataset."
return new_questions
@@ -175,6 +178,7 @@ def get_mllm_dataloader(
bs=1,
split=None,
apply_template=None,
+ truncation=False,
seed=42,
):
"""Generate a DataLoader for calibration using specified parameters.
@@ -202,11 +206,11 @@ def get_mllm_dataloader(
if os.path.isfile(dataset):
dataset = MLLM_DATASET['liuhaotian/llava'](
template, model, tokenizer, dataset, extra_data_dir,
- seqlen=min(seqlen, tokenizer.model_max_length))
+ seqlen=min(seqlen, tokenizer.model_max_length), truncation=truncation)
elif "liuhaotian/llava" in dataset:
dataset = MLLM_DATASET["liuhaotian/llava"](
template, model, tokenizer, dataset, extra_data_dir,
- seqlen=min(seqlen, tokenizer.model_max_length))
+ seqlen=min(seqlen, tokenizer.model_max_length), truncation=truncation)
else:
from datasets import load_dataset
from ..calib_dataset import get_tokenizer_function
diff --git a/auto_round/mllm/processor.py b/auto_round/mllm/processor.py
index 3b285e015..7de74d7f3 100644
--- a/auto_round/mllm/processor.py
+++ b/auto_round/mllm/processor.py
@@ -45,6 +45,7 @@ class BasicProcessor:
return_tensors="pt",
squeeze=True,
max_length=None,
+ truncation=False,
truncation_strategy="text",
**kwargs):
@@ -69,8 +70,8 @@ class BasicProcessor:
if images is not None:
images = self.image_processor(images)
- if truncation_strategy == "text" and max_length is not None:
- text = text[:max_length]
+ if truncation is True and truncation_strategy == "text":
+ text = self.tokenizer.decode(self.tokenizer(text).input_ids[:max_length])
ret = self.tokenizer.processor(
text=text,
@@ -78,7 +79,7 @@ class BasicProcessor:
return_tensors=return_tensors,
# videos = None
)
- if truncation_strategy == "token" and max_length:
+ if truncation is True and truncation_strategy == "token":
seqlen = ret['input_ids'].shape[-1]
for key in ret:
shape_ = ret[key].shape
@@ -118,7 +119,7 @@ class Qwen2VLProcessor(BasicProcessor):
@regist_processor("cogvlm2")
class CogVLM2Processor(BasicProcessor):
def get_input(
- self, text, images,
+ self, text, images, truncation=False,
squeeze=True, **kwargs):
if images is not None:
@@ -127,7 +128,6 @@ class CogVLM2Processor(BasicProcessor):
padding_len = 2303
max_length = 0
max_length += padding_len
- truncation = True
padding = False
input_data = self.model.build_conversation_input_ids(
self.tokenizer,
@@ -207,7 +207,7 @@ class LlavaProcessor(BasicProcessor):
def get_input(
self, text, images,max_length=None,
- squeeze=True, truncation_strategy="text", **kwargs):
+ squeeze=True, truncation=False, truncation_strategy="text", **kwargs):
if images is not None:
images = fetch_image(images).convert('RGB')
@@ -217,13 +217,13 @@ class LlavaProcessor(BasicProcessor):
is_multimodal = True
mm_use_im_start_end = False
- if truncation_strategy == "text" and max_length is not None:
- text = text[:max_length]
+ if truncation is True and truncation_strategy == "text":
+ text = self.tokenizer.decode(self.tokenizer(text).input_ids[:max_length])
input_data = llava_train.preprocess_multimodal([text], DataArgs())
ret = llava_train.preprocess(input_data, self.tokenizer, has_image=(images is not None))
- if truncation_strategy == "token" and max_length:
+ if truncation is True and truncation_strategy == "token":
seqlen = ret['input_ids'].shape[-1]
for key in ret:
if ret[key].shape[-1] == seqlen:
diff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.py
index 6d477f04f..9154187d3 100644
--- a/auto_round/mllm/template.py
+++ b/auto_round/mllm/template.py
@@ -48,12 +48,12 @@ class Template:
format_separator: str
default_system: str
replace_tokens: List[tuple]
- add_special_token: bool
+ extra_encode: bool
processor: "BasicProcessor"
def _encode(self, sources):
"""Encodes formatted inputs to pairs of token ids."""
- if self.add_special_token:
+ if self.extra_encode:
element = ""
for i, source in enumerate(sources):
if i == 0:
@@ -84,7 +84,7 @@ def _register_template(
format_separator: Optional[str] = None,
default_system: str = "",
replace_tokens: List[tuple] = None,
- add_special_token: Optional[bool] = True,
+ extra_encode: Optional[bool] = True,
processor: "BasicProcessor" = PROCESSORS["basic"],
):
"""Registers a chat template."""
@@ -105,7 +105,7 @@ def _register_template(
format_separator = format_separator or default_format_separator,
default_system = default_system,
replace_tokens = replace_tokens,
- add_special_token = add_special_token,
+ extra_encode = extra_encode,
processor = processor()
)
return TEMPLATES[model_type]
diff --git a/auto_round/mllm/templates/default.json b/auto_round/mllm/templates/default.json
index 51ec37f34..2a14283d6 100644
--- a/auto_round/mllm/templates/default.json
+++ b/auto_round/mllm/templates/default.json
@@ -9,5 +9,5 @@
"default_system": "You are a helpful assistant.",
"replace_tokens": null,
"processor": "basic",
- "add_special_token" : false
+ "extra_encode" : false
}
\ No newline at end of file
diff --git a/auto_round/mllm/templates/llava.json b/auto_round/mllm/templates/llava.json
index 13d5a6b49..87b4be027 100644
--- a/auto_round/mllm/templates/llava.json
+++ b/auto_round/mllm/templates/llava.json
@@ -2,5 +2,5 @@
"model_type": "llava",
"replace_tokens": null,
"processor": "llava",
- "add_special_token" : false
+ "extra_encode" : false
}
\ No newline at end of file
diff --git a/auto_round/mllm/templates/phi3_v.json b/auto_round/mllm/templates/phi3_v.json
index 09949ff47..955c15f7a 100644
--- a/auto_round/mllm/templates/phi3_v.json
+++ b/auto_round/mllm/templates/phi3_v.json
@@ -2,5 +2,5 @@
"model_type": "phi3_v",
"replace_tokens": ["<image>", "<|image_1|>"],
"processor": "basic",
- "add_special_token" : false
+ "extra_encode" : false
}
\ No newline at end of file
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 54a25f3a2..ca5d03510 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -147,6 +147,9 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--template", default=None, type=str,
help="the template for building training dataset. It can be a custom one.")
+
+ self.add_argument("--truncation", action="store_true",
+ help="whether to truncate sequences at the maximum length.")
## ======================= VLM eval=======================
self.add_argument("--tasks", type=str,
@@ -331,7 +334,7 @@ def tune(args):
extra_data_dir=args.extra_data_dir, bits=args.bits, group_size=args.group_size,
sym=not args.asym, batch_size=args.batch_size, seqlen=seqlen, nblocks=args.nblocks,
iters=args.iters, lr=args.lr, minmax_lr=args.minmax_lr, amp=not args.disable_amp,
- enable_quanted_input=not args.disable_quanted_input,
+ enable_quanted_input=not args.disable_quanted_input, truncation=args.truncation,
nsamples=args.nsamples, low_gpu_mem_usage=args.low_gpu_mem_usage,
device=device_str, seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
scale_dtype=args.scale_dtype, layer_config=layer_config, template=args.template,