Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

set default mllm dataset (#327)

Heng Guoab0a477ec6c1
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index dad317d93..2ce7abdff 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -579,7 +579,7 @@ class AutoRound(object):
elif total_cnt < nsamples:
logger.warning(
f"Insufficient number of samples collected may affect the quantification. "
- f"Valid samples size:{total_cnt}, Target sample size:{nsamples}"
+ f"target samples count is {nsamples}, while valid samples count is {total_cnt}"
)
# clean embed weight to save memory
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index a3c1f8adf..3d95f26b5 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -28,6 +28,7 @@ from ..autoround import AutoRound
from .template import get_template, Template
from .mllm_dataset import get_mllm_dataloader
from ..low_cpu_mem.utils import get_layers_before_block
+from ..calib_dataset import CALIB_DATASETS
class AutoRoundMLLM(AutoRound):
@@ -130,8 +131,21 @@ class AutoRoundMLLM(AutoRound):
self.template = template if template is not None else model.config.model_type
self.template = get_template(
self.template, model=model, tokenizer=tokenizer, image_processor=image_processor)
+
+ dataset = self.template.default_dataset if dataset is None else dataset
+ if truncation is None:
+ truncation = True if dataset in CALIB_DATASETS.keys() else False
self.truncation = truncation
- assert dataset is not None, "dataset should not be None"
+
+ if dataset in CALIB_DATASETS.keys() and not self._only_text_test():
+ logger.warning(f"{model.config.model_type} not support for {dataset},"
+ " will use liuhaotian/llava_conv_58k with default config as an alternative.")
+ dataset = "liuhaotian/llava_conv_58k"
+ self.truncation = False
+ batch_size = 1
+ gradient_accumulate_steps = 4
+ seqlen = 512
+
super(AutoRoundMLLM, self).__init__(
model=model,
@@ -172,6 +186,18 @@ class AutoRoundMLLM(AutoRound):
**kwargs,
)
+
+ def _only_text_test(self):
+ """Test if the model whether can use text-only datasets."""
+ try:
+ text = ["only text", "test"]
+ inputs = self.tokenizer(text, return_tensors="pt").to(self.model.device)
+ self.model(**inputs)
+ return True
+ except:
+ return False
+
+
def calib(self, nsamples, bs):
"""Perform calibration for quantization.
@@ -210,7 +236,8 @@ class AutoRoundMLLM(AutoRound):
for n, m in embed_layers:
m = m.to(self.device)
- with tqdm(total=nsamples-1, desc="calib") as pbar:
+ total = nsamples if not hasattr(self.dataloader, "len") else min(nsamples, len(self.dataloader))
+ with tqdm(range(1, total + 1), desc="calib") as pbar:
for data in self.dataloader:
if data is None:
continue
@@ -285,7 +312,7 @@ class AutoRoundMLLM(AutoRound):
elif total_cnt < nsamples:
logger.warning(
f"Insufficient number of samples collected may affect the quantification. "
- f"Valid samples size:{total_cnt}, Target sample size:{nsamples}"
+ f"target samples count is {nsamples}, while valid samples count is {total_cnt}"
)
# clean embed weight to save memory
diff --git a/auto_round/mllm/mllm_dataset.py b/auto_round/mllm/mllm_dataset.py
index 804beb57d..4d5e7f30b 100644
--- a/auto_round/mllm/mllm_dataset.py
+++ b/auto_round/mllm/mllm_dataset.py
@@ -29,7 +29,7 @@ from ..special_model_handler import check_mllm_model_batch
MLLM_DATASET : Dict[str, Dataset] = {}
-def register_dataset(name):
+def register_dataset(name_list):
"""Class decorator to register a DATASET subclass to the registry.
Decorator function used before a Pattern subclass.
@@ -42,13 +42,14 @@ def register_dataset(name):
"""
def register(dataset):
- MLLM_DATASET[name] = dataset
- return dataset
+ for name in name_list.replace(' ', '').split(','):
+ MLLM_DATASET[name] = dataset
return register
-@register_dataset("liuhaotian/llava")
+@register_dataset(
+ "liuhaotian/llava,liuhaotian/llava_conv_58k,liuhaotian/llava_instruct_80k,liuhaotian/llava_instruct_150k")
class LlavaDataset(Dataset):
"""Dataset for supervised fine-tuning."""
BASE_LLAVA_URL = "https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/resolve/main/"
@@ -59,6 +60,7 @@ class LlavaDataset(Dataset):
}
_COCO_DATA_URL = "http://images.cocodataset.org/train2017/"
IMAGE_TOKEN = "<image>"
+ MAX_SEQLEN = 512
def __init__(
self,
@@ -67,15 +69,20 @@ class LlavaDataset(Dataset):
tokenzier,
dataset_path,
extra_data_dir=None,
- seqlen=None,
+ seqlen=512,
padding=True,
truncation=True,
+ nsamples=512
) -> None:
super().__init__()
self.model = model
self.model_type = template.model_type
self.template = template
self.tokenizer = tokenzier
+ if dataset_path == "liuhaotian/llava":
+ dataset_path = "llava_conv_58k"
+ else:
+ dataset_path = dataset_path.split("/")[-1]
if os.path.exists(dataset_path):
logger.info(f'use dataset {dataset_path}, loading from disk...')
self.questions = json.load(open(dataset_path, "r"))
@@ -87,8 +94,8 @@ class LlavaDataset(Dataset):
self.questions = requests.get(self.LLAVA_DATASET[dataset_name], stream=True).json()
else:
raise KeyError(f"{dataset_path} is not support, we support {self.LLAVA_DATASET.keys()}.")
- self.seqlen = seqlen
- self.questions = self.check(self.questions, seqlen)
+ self.seqlen = min(seqlen, self.MAX_SEQLEN)
+ self.questions = self.check(self.questions, seqlen, nsamples)
self.padding = padding
self.truncation = truncation
self.extra_data_dir = extra_data_dir
@@ -103,19 +110,33 @@ class LlavaDataset(Dataset):
self.image_fold = image_fold
- def check(self, questions, seqlen):
- new_questions = []
- for source in questions:
- text_lenght = 0
- for text in source['conversations']:
- if self.IMAGE_TOKEN in text['value']:
- text['value'] = self.IMAGE_TOKEN + text['value'].replace(self.IMAGE_TOKEN, '')
- text_lenght += len(text['value'].split(' '))
- if text_lenght >= seqlen:
- new_questions.append(source)
- assert len(new_questions) > 0, \
- f"no data with length greater than {seqlen}, please reduce the seqlen or change another dataset."
- return new_questions
+ def check(self, questions, seqlen, nsamples):
+ def _check(questions, min_seqlen, max_seqlen, nsamples):
+ new_questions = []
+ max_len = 0
+ for source in questions:
+ str_len = 0
+ for text in source['conversations']:
+ if self.IMAGE_TOKEN in text['value']:
+ text['value'] = self.IMAGE_TOKEN + text['value'].replace(self.IMAGE_TOKEN, '')
+ str_len += len(text['value'].split(' '))
+ if str_len > max_len:
+ max_len = str_len
+ if min_seqlen <= str_len < max_seqlen:
+ new_questions.append(source)
+ if len(new_questions) >= nsamples:
+ return new_questions
+ else:
+ if seqlen > max_len:
+ logger.warning(f"seqlen={seqlen} is greater than the max length of dataset {max_len},"
+ f" will change seqlen to {max_len - 128}")
+ new_min_seqlen = max_len - 128
+ else:
+ logger.warning(f"no enough sample for seqlen greater than {min_seqlen},"
+ f" will decrease to {min_seqlen-128}")
+ new_min_seqlen = min_seqlen - 128
+ return new_questions + _check(questions, new_min_seqlen, min_seqlen, nsamples-len(new_questions))
+ return _check(questions, seqlen, float("inf"), nsamples)
def __len__(self):
@@ -205,40 +226,35 @@ def get_mllm_dataloader(
if isinstance(template, str):
from .template import get_template
template = get_template(template, model=model, tokenizer=tokenizer, image_processor=image_processor)
+
- if os.path.isfile(dataset):
+ if os.path.isfile(dataset) or dataset in MLLM_DATASET.keys():
dataset = MLLM_DATASET['liuhaotian/llava'](
template, model, tokenizer, dataset, extra_data_dir,
- seqlen=min(seqlen, tokenizer.model_max_length), truncation=truncation)
- elif "liuhaotian/llava" in dataset:
- dataset = MLLM_DATASET["liuhaotian/llava"](
- template, model, tokenizer, dataset, extra_data_dir,
- seqlen=min(seqlen, tokenizer.model_max_length), truncation=truncation)
+ seqlen=seqlen, truncation=truncation, nsamples=nsamples)
+
+ bs, gradient_accumulate_steps = check_mllm_model_batch(
+ model, batch_size=bs, gradient_accumulate_steps=gradient_accumulate_steps)
+
+ set_seed(seed)
+ dataloader_params = {
+ "batch_size": bs,
+ "shuffle": True,
+ "collate_fn": dataset.template.processor.data_collator
+ }
+
+ return DataLoader(dataset, **dataloader_params), bs, gradient_accumulate_steps
else:
# try to load text calibration dataset
- from ..calib_dataset import get_dataloader
- dataloader = get_dataloader(
- tokenizer, seqlen, dataset, seed, bs, nsamples)
- if quant_nontext_module:
- logger.error(
- f"Quantitative nontext module is not supported for plain text datasets," \
- " please disable arg '--quant_nontext_module'")
- exit(-1)
- if "mllama" in model.config.model_type:
- logger.error(
- f"The llama3.2-vision model does not support the quantization of text-only calibration datasets.")
- exit(-1)
- return dataloader, bs, gradient_accumulate_steps
+ from ..calib_dataset import get_dataloader
+ dataloader = get_dataloader(
+ tokenizer, seqlen, dataset, seed, bs, nsamples)
+ if quant_nontext_module:
+ logger.error(
+ f"Quantitative nontext module is not supported for plain text datasets," \
+ " please disable arg '--quant_nontext_module'")
+ exit(-1)
+ return dataloader, bs, gradient_accumulate_steps
- bs, gradient_accumulate_steps = check_mllm_model_batch(
- model, batch_size=bs, gradient_accumulate_steps=gradient_accumulate_steps)
- set_seed(seed)
- dataloader_params = {
- "batch_size": bs,
- "shuffle": True,
- "collate_fn": dataset.template.processor.data_collator
- }
-
- return DataLoader(dataset, **dataloader_params), bs, gradient_accumulate_steps
diff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.py
index 9154187d3..16bb4e21f 100644
--- a/auto_round/mllm/template.py
+++ b/auto_round/mllm/template.py
@@ -49,6 +49,7 @@ class Template:
default_system: str
replace_tokens: List[tuple]
extra_encode: bool
+ default_dataset: str
processor: "BasicProcessor"
def _encode(self, sources):
@@ -84,7 +85,8 @@ def _register_template(
format_separator: Optional[str] = None,
default_system: str = "",
replace_tokens: List[tuple] = None,
- extra_encode: Optional[bool] = True,
+ extra_encode: Optional[bool] = False,
+ default_dataset: Optional[bool] = "NeelNanda/pile-10k",
processor: "BasicProcessor" = PROCESSORS["basic"],
):
"""Registers a chat template."""
@@ -106,6 +108,7 @@ def _register_template(
default_system = default_system,
replace_tokens = replace_tokens,
extra_encode = extra_encode,
+ default_dataset = default_dataset,
processor = processor()
)
return TEMPLATES[model_type]
diff --git a/auto_round/mllm/templates/cogvlm2.json b/auto_round/mllm/templates/cogvlm2.json
index 8aa8cfa3e..c3edf7f21 100644
--- a/auto_round/mllm/templates/cogvlm2.json
+++ b/auto_round/mllm/templates/cogvlm2.json
@@ -3,5 +3,7 @@
"format_user": "Question: {{content}} ",
"format_assistant": "Answer: {{content}}\n",
"replace_tokens": ["<image>\n", ""],
- "processor": "cogvlm2"
+ "processor": "cogvlm2",
+ "extra_encode" : true,
+ "default_dataset": "NeelNanda/pile-10k"
}
\ No newline at end of file
diff --git a/auto_round/mllm/templates/default.json b/auto_round/mllm/templates/default.json
index 2a14283d6..321a0c485 100644
--- a/auto_round/mllm/templates/default.json
+++ b/auto_round/mllm/templates/default.json
@@ -8,6 +8,7 @@
"format_separator": "\n",
"default_system": "You are a helpful assistant.",
"replace_tokens": null,
- "processor": "basic",
- "extra_encode" : false
+ "extra_encode" : false,
+ "default_dataset": "NeelNanda/pile-10k",
+ "processor": "basic"
}
\ No newline at end of file
diff --git a/auto_round/mllm/templates/llava.json b/auto_round/mllm/templates/llava.json
index 87b4be027..85a243348 100644
--- a/auto_round/mllm/templates/llava.json
+++ b/auto_round/mllm/templates/llava.json
@@ -2,5 +2,6 @@
"model_type": "llava",
"replace_tokens": null,
"processor": "llava",
- "extra_encode" : false
+ "extra_encode" : false,
+ "default_dataset": "NeelNanda/pile-10k"
}
\ No newline at end of file
diff --git a/auto_round/mllm/templates/mllama.json b/auto_round/mllm/templates/mllama.json
index 0a31392a2..2434dc4ba 100644
--- a/auto_round/mllm/templates/mllama.json
+++ b/auto_round/mllm/templates/mllama.json
@@ -4,5 +4,7 @@
"format_assistant": "<|start_header_id|>assistant<|end_header_id|>\n{{content}}<|eot_id|>",
"format_system": "<|begin_of_text|><|start_header_id|>system|end_header_id|>\n{{content}}<<eot_id>",
"default_system": "You are a helpful assistant.",
- "replace_tokens": ["<image>", "<|image|>"]
+ "replace_tokens": ["<image>", "<|image|>"],
+ "extra_encode" : true,
+ "default_dataset": "liuhaotian/llava_conv_58k"
}
\ No newline at end of file
diff --git a/auto_round/mllm/templates/phi3_v.json b/auto_round/mllm/templates/phi3_v.json
index 955c15f7a..65e7445dc 100644
--- a/auto_round/mllm/templates/phi3_v.json
+++ b/auto_round/mllm/templates/phi3_v.json
@@ -2,5 +2,6 @@
"model_type": "phi3_v",
"replace_tokens": ["<image>", "<|image_1|>"],
"processor": "basic",
- "extra_encode" : false
+ "extra_encode" : false,
+ "default_dataset": "NeelNanda/pile-10k"
}
\ No newline at end of file
diff --git a/auto_round/mllm/templates/qwen2_vl.json b/auto_round/mllm/templates/qwen2_vl.json
index 9435d4236..6b72a0637 100644
--- a/auto_round/mllm/templates/qwen2_vl.json
+++ b/auto_round/mllm/templates/qwen2_vl.json
@@ -7,5 +7,7 @@
"format_separator": "\n",
"default_system": "You are a helpful assistant.",
"replace_tokens": ["<image>", "<|vision_start|><|image_pad|><|vision_end|>"],
- "processor": "qwen2_vl"
+ "processor": "qwen2_vl",
+ "extra_encode" : true,
+ "default_dataset": "NeelNanda/pile-10k"
}
\ No newline at end of file
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index bc6233e5e..0067161ee 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -13,6 +13,7 @@
# limitations under the License.
import os
+import sys
import argparse
import json
@@ -51,10 +52,10 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--asym", action='store_true',
help="whether to use asym quantization")
- self.add_argument("--dataset", type=str, default="liuhaotian/llava_conv_58k",
+ self.add_argument("--dataset", type=str, default=None,
help="the dataset for quantization training."
- " current support llava_conv_58k,llava_instruct_80k "
- "It can be a custom one.")
+ " current support NeelNanda/pile-10k,llava_conv_58k,llava_instruct_80k "
+ "It can be a custom one. Default is NeelNanda/pile-10k")
self.add_argument("--lr", default=None, type=float,
help="learning rate, if None, it will be set to 1.0/iters automatically")
@@ -68,7 +69,7 @@ class BasicArgumentParser(argparse.ArgumentParser):
self.add_argument("--adam", action='store_true',
help="whether to use adam optimizer instead of SignSGD")
- self.add_argument("--gradient_accumulate_steps", default=4, type=int,
+ self.add_argument("--gradient_accumulate_steps", default=1, type=int,
help="gradient accumulate steps")
self.add_argument("--nblocks", default=1, type=int,
@@ -153,7 +154,8 @@ class BasicArgumentParser(argparse.ArgumentParser):
help="the template for building training dataset. It can be a custom one.")
self.add_argument("--truncation", action="store_true",
- help="whether to truncate sequences at the maximum length.")
+ help="whether to truncate sequences at the maximum length."
+ " Default True for pile and False for llava dataset.")
self.add_argument("--to_quant_block_names", default=None, type=str,
help="Names of quantitative blocks, please use commas to separate them.")
@@ -197,13 +199,13 @@ def setup_parser():
parser.add_argument("--group_size", default=128, type=int,
help="group size")
- parser.add_argument("--batch_size", "--train_bs", default=1, type=int,
+ parser.add_argument("--batch_size", "--train_bs", default=8, type=int,
help="train batch size")
parser.add_argument("--iters", "--iter", default=200, type=int,
help=" iters")
- parser.add_argument("--seqlen", "--seq_len", default=256, type=int,
+ parser.add_argument("--seqlen", "--seq_len", default=2048, type=int,
help="sequence length")
parser.add_argument("--nsamples", default=128, type=int,
@@ -230,8 +232,6 @@ def tune(args):
model_name = model_name[:-1]
logger.info(f"start to quantize {model_name}")
- assert args.dataset is not None, "dataset should not be None."
-
devices = args.device.replace(" ", "").split(',')
use_auto_mapping = False
@@ -335,6 +335,9 @@ def tune(args):
print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "
f"gpu")
+ if "--truncation" not in sys.argv:
+ args.truncation = None
+
autoround = round(model, tokenizer, image_processor=image_processor, dataset=args.dataset,
extra_data_dir=args.extra_data_dir, bits=args.bits, group_size=args.group_size,
sym=not args.asym, batch_size=args.batch_size, seqlen=seqlen, nblocks=args.nblocks,
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 3ae74f6a4..11d25c7a2 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -36,14 +36,33 @@ from .special_model_handler import shareable_keywords
def warning_once(self, msg: str):
self.warning(msg)
+class AutoRoundFormatter(logging.Formatter):
+ grey = "\x1b[38;20m"
+ yellow = "\x1b[33;1m"
+ red = "\x1b[31;20m"
+ bold_red = "\x1b[31;1m"
+ reset = "\x1b[0m"
+ _format = "%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s"
+
+ FORMATS = {
+ logging.DEBUG: grey + _format + reset,
+ logging.INFO: grey + _format + reset,
+ logging.WARNING: yellow + _format + reset,
+ logging.ERROR: bold_red + _format + reset,
+ logging.CRITICAL: bold_red + _format + reset
+ }
+
+ def format(self, record):
+ log_fmt = self.FORMATS.get(record.levelno)
+ formatter = logging.Formatter(log_fmt, "%Y-%m-%d %H:%M:%S")
+ return formatter.format(record)
logging.Logger.warning_once = warning_once
logger = logging.getLogger("autoround")
logger.setLevel(logging.INFO)
logger.propagate = False
fh = logging.StreamHandler()
-fh_formatter = logging.Formatter("%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s", "%Y-%m-%d %H:%M:%S")
-fh.setFormatter(fh_formatter)
+fh.setFormatter(AutoRoundFormatter())
logger.addHandler(fh)
import importlib