PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
set default mllm dataset (#327)
Heng Guoab0a477ec6c1
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex dad317d93..2ce7abdff 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -579,7 +579,7 @@ class AutoRound(object):elif total_cnt < nsamples:logger.warning(f"Insufficient number of samples collected may affect the quantification. "- f"Valid samples size:{total_cnt}, Target sample size:{nsamples}"+ f"target samples count is {nsamples}, while valid samples count is {total_cnt}")# clean embed weight to save memorydiff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex a3c1f8adf..3d95f26b5 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -28,6 +28,7 @@ from ..autoround import AutoRoundfrom .template import get_template, Templatefrom .mllm_dataset import get_mllm_dataloaderfrom ..low_cpu_mem.utils import get_layers_before_block+from ..calib_dataset import CALIB_DATASETSclass AutoRoundMLLM(AutoRound):@@ -130,8 +131,21 @@ class AutoRoundMLLM(AutoRound):self.template = template if template is not None else model.config.model_typeself.template = get_template(self.template, model=model, tokenizer=tokenizer, image_processor=image_processor)++ dataset = self.template.default_dataset if dataset is None else dataset+ if truncation is None:+ truncation = True if dataset in CALIB_DATASETS.keys() else Falseself.truncation = truncation- assert dataset is not None, "dataset should not be None"++ if dataset in CALIB_DATASETS.keys() and not self._only_text_test():+ logger.warning(f"{model.config.model_type} not support for {dataset},"+ " will use liuhaotian/llava_conv_58k with default config as an alternative.")+ dataset = "liuhaotian/llava_conv_58k"+ self.truncation = False+ batch_size = 1+ gradient_accumulate_steps = 4+ seqlen = 512+super(AutoRoundMLLM, self).__init__(model=model,@@ -172,6 +186,18 @@ class AutoRoundMLLM(AutoRound):**kwargs,)++ def _only_text_test(self):+ """Test if the model whether can use text-only datasets."""+ try:+ text = ["only text", "test"]+ inputs = self.tokenizer(text, return_tensors="pt").to(self.model.device)+ self.model(**inputs)+ return True+ except:+ return False++def calib(self, nsamples, bs):"""Perform calibration for quantization.@@ -210,7 +236,8 @@ class AutoRoundMLLM(AutoRound):for n, m in embed_layers:m = m.to(self.device)- with tqdm(total=nsamples-1, desc="calib") as pbar:+ total = nsamples if not hasattr(self.dataloader, "len") else min(nsamples, len(self.dataloader))+ with tqdm(range(1, total + 1), desc="calib") as pbar:for data in self.dataloader:if data is None:continue@@ -285,7 +312,7 @@ class AutoRoundMLLM(AutoRound):elif total_cnt < nsamples:logger.warning(f"Insufficient number of samples collected may affect the quantification. "- f"Valid samples size:{total_cnt}, Target sample size:{nsamples}"+ f"target samples count is {nsamples}, while valid samples count is {total_cnt}")# clean embed weight to save memorydiff --git a/auto_round/mllm/mllm_dataset.py b/auto_round/mllm/mllm_dataset.pyindex 804beb57d..4d5e7f30b 100644--- a/auto_round/mllm/mllm_dataset.py+++ b/auto_round/mllm/mllm_dataset.py@@ -29,7 +29,7 @@ from ..special_model_handler import check_mllm_model_batchMLLM_DATASET : Dict[str, Dataset] = {}-def register_dataset(name):+def register_dataset(name_list):"""Class decorator to register a DATASET subclass to the registry.Decorator function used before a Pattern subclass.@@ -42,13 +42,14 @@ def register_dataset(name):"""def register(dataset):- MLLM_DATASET[name] = dataset- return dataset+ for name in name_list.replace(' ', '').split(','):+ MLLM_DATASET[name] = datasetreturn register-@register_dataset("liuhaotian/llava")+@register_dataset(+ "liuhaotian/llava,liuhaotian/llava_conv_58k,liuhaotian/llava_instruct_80k,liuhaotian/llava_instruct_150k")class LlavaDataset(Dataset):"""Dataset for supervised fine-tuning."""BASE_LLAVA_URL = "https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/resolve/main/"@@ -59,6 +60,7 @@ class LlavaDataset(Dataset):}_COCO_DATA_URL = "http://images.cocodataset.org/train2017/"IMAGE_TOKEN = "<image>"+ MAX_SEQLEN = 512def __init__(self,@@ -67,15 +69,20 @@ class LlavaDataset(Dataset):tokenzier,dataset_path,extra_data_dir=None,- seqlen=None,+ seqlen=512,padding=True,truncation=True,+ nsamples=512) -> None:super().__init__()self.model = modelself.model_type = template.model_typeself.template = templateself.tokenizer = tokenzier+ if dataset_path == "liuhaotian/llava":+ dataset_path = "llava_conv_58k"+ else:+ dataset_path = dataset_path.split("/")[-1]if os.path.exists(dataset_path):logger.info(f'use dataset {dataset_path}, loading from disk...')self.questions = json.load(open(dataset_path, "r"))@@ -87,8 +94,8 @@ class LlavaDataset(Dataset):self.questions = requests.get(self.LLAVA_DATASET[dataset_name], stream=True).json()else:raise KeyError(f"{dataset_path} is not support, we support {self.LLAVA_DATASET.keys()}.")- self.seqlen = seqlen- self.questions = self.check(self.questions, seqlen)+ self.seqlen = min(seqlen, self.MAX_SEQLEN)+ self.questions = self.check(self.questions, seqlen, nsamples)self.padding = paddingself.truncation = truncationself.extra_data_dir = extra_data_dir@@ -103,19 +110,33 @@ class LlavaDataset(Dataset):self.image_fold = image_fold- def check(self, questions, seqlen):- new_questions = []- for source in questions:- text_lenght = 0- for text in source['conversations']:- if self.IMAGE_TOKEN in text['value']:- text['value'] = self.IMAGE_TOKEN + text['value'].replace(self.IMAGE_TOKEN, '')- text_lenght += len(text['value'].split(' '))- if text_lenght >= seqlen:- new_questions.append(source)- assert len(new_questions) > 0, \- f"no data with length greater than {seqlen}, please reduce the seqlen or change another dataset."- return new_questions+ def check(self, questions, seqlen, nsamples):+ def _check(questions, min_seqlen, max_seqlen, nsamples):+ new_questions = []+ max_len = 0+ for source in questions:+ str_len = 0+ for text in source['conversations']:+ if self.IMAGE_TOKEN in text['value']:+ text['value'] = self.IMAGE_TOKEN + text['value'].replace(self.IMAGE_TOKEN, '')+ str_len += len(text['value'].split(' '))+ if str_len > max_len:+ max_len = str_len+ if min_seqlen <= str_len < max_seqlen:+ new_questions.append(source)+ if len(new_questions) >= nsamples:+ return new_questions+ else:+ if seqlen > max_len:+ logger.warning(f"seqlen={seqlen} is greater than the max length of dataset {max_len},"+ f" will change seqlen to {max_len - 128}")+ new_min_seqlen = max_len - 128+ else:+ logger.warning(f"no enough sample for seqlen greater than {min_seqlen},"+ f" will decrease to {min_seqlen-128}")+ new_min_seqlen = min_seqlen - 128+ return new_questions + _check(questions, new_min_seqlen, min_seqlen, nsamples-len(new_questions))+ return _check(questions, seqlen, float("inf"), nsamples)def __len__(self):@@ -205,40 +226,35 @@ def get_mllm_dataloader(if isinstance(template, str):from .template import get_templatetemplate = get_template(template, model=model, tokenizer=tokenizer, image_processor=image_processor)+- if os.path.isfile(dataset):+ if os.path.isfile(dataset) or dataset in MLLM_DATASET.keys():dataset = MLLM_DATASET['liuhaotian/llava'](template, model, tokenizer, dataset, extra_data_dir,- seqlen=min(seqlen, tokenizer.model_max_length), truncation=truncation)- elif "liuhaotian/llava" in dataset:- dataset = MLLM_DATASET["liuhaotian/llava"](- template, model, tokenizer, dataset, extra_data_dir,- seqlen=min(seqlen, tokenizer.model_max_length), truncation=truncation)+ seqlen=seqlen, truncation=truncation, nsamples=nsamples)++ bs, gradient_accumulate_steps = check_mllm_model_batch(+ model, batch_size=bs, gradient_accumulate_steps=gradient_accumulate_steps)++ set_seed(seed)+ dataloader_params = {+ "batch_size": bs,+ "shuffle": True,+ "collate_fn": dataset.template.processor.data_collator+ }++ return DataLoader(dataset, **dataloader_params), bs, gradient_accumulate_stepselse:# try to load text calibration dataset- from ..calib_dataset import get_dataloader- dataloader = get_dataloader(- tokenizer, seqlen, dataset, seed, bs, nsamples)- if quant_nontext_module:- logger.error(- f"Quantitative nontext module is not supported for plain text datasets," \- " please disable arg '--quant_nontext_module'")- exit(-1)- if "mllama" in model.config.model_type:- logger.error(- f"The llama3.2-vision model does not support the quantization of text-only calibration datasets.")- exit(-1)- return dataloader, bs, gradient_accumulate_steps+ from ..calib_dataset import get_dataloader+ dataloader = get_dataloader(+ tokenizer, seqlen, dataset, seed, bs, nsamples)+ if quant_nontext_module:+ logger.error(+ f"Quantitative nontext module is not supported for plain text datasets," \+ " please disable arg '--quant_nontext_module'")+ exit(-1)+ return dataloader, bs, gradient_accumulate_steps- bs, gradient_accumulate_steps = check_mllm_model_batch(- model, batch_size=bs, gradient_accumulate_steps=gradient_accumulate_steps)- set_seed(seed)- dataloader_params = {- "batch_size": bs,- "shuffle": True,- "collate_fn": dataset.template.processor.data_collator- }-- return DataLoader(dataset, **dataloader_params), bs, gradient_accumulate_stepsdiff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.pyindex 9154187d3..16bb4e21f 100644--- a/auto_round/mllm/template.py+++ b/auto_round/mllm/template.py@@ -49,6 +49,7 @@ class Template:default_system: strreplace_tokens: List[tuple]extra_encode: bool+ default_dataset: strprocessor: "BasicProcessor"def _encode(self, sources):@@ -84,7 +85,8 @@ def _register_template(format_separator: Optional[str] = None,default_system: str = "",replace_tokens: List[tuple] = None,- extra_encode: Optional[bool] = True,+ extra_encode: Optional[bool] = False,+ default_dataset: Optional[bool] = "NeelNanda/pile-10k",processor: "BasicProcessor" = PROCESSORS["basic"],):"""Registers a chat template."""@@ -106,6 +108,7 @@ def _register_template(default_system = default_system,replace_tokens = replace_tokens,extra_encode = extra_encode,+ default_dataset = default_dataset,processor = processor())return TEMPLATES[model_type]diff --git a/auto_round/mllm/templates/cogvlm2.json b/auto_round/mllm/templates/cogvlm2.jsonindex 8aa8cfa3e..c3edf7f21 100644--- a/auto_round/mllm/templates/cogvlm2.json+++ b/auto_round/mllm/templates/cogvlm2.json@@ -3,5 +3,7 @@"format_user": "Question: {{content}} ","format_assistant": "Answer: {{content}}\n","replace_tokens": ["<image>\n", ""],- "processor": "cogvlm2"+ "processor": "cogvlm2",+ "extra_encode" : true,+ "default_dataset": "NeelNanda/pile-10k"}\ No newline at end of filediff --git a/auto_round/mllm/templates/default.json b/auto_round/mllm/templates/default.jsonindex 2a14283d6..321a0c485 100644--- a/auto_round/mllm/templates/default.json+++ b/auto_round/mllm/templates/default.json@@ -8,6 +8,7 @@"format_separator": "\n","default_system": "You are a helpful assistant.","replace_tokens": null,- "processor": "basic",- "extra_encode" : false+ "extra_encode" : false,+ "default_dataset": "NeelNanda/pile-10k",+ "processor": "basic"}\ No newline at end of filediff --git a/auto_round/mllm/templates/llava.json b/auto_round/mllm/templates/llava.jsonindex 87b4be027..85a243348 100644--- a/auto_round/mllm/templates/llava.json+++ b/auto_round/mllm/templates/llava.json@@ -2,5 +2,6 @@"model_type": "llava","replace_tokens": null,"processor": "llava",- "extra_encode" : false+ "extra_encode" : false,+ "default_dataset": "NeelNanda/pile-10k"}\ No newline at end of filediff --git a/auto_round/mllm/templates/mllama.json b/auto_round/mllm/templates/mllama.jsonindex 0a31392a2..2434dc4ba 100644--- a/auto_round/mllm/templates/mllama.json+++ b/auto_round/mllm/templates/mllama.json@@ -4,5 +4,7 @@"format_assistant": "<|start_header_id|>assistant<|end_header_id|>\n{{content}}<|eot_id|>","format_system": "<|begin_of_text|><|start_header_id|>system|end_header_id|>\n{{content}}<<eot_id>","default_system": "You are a helpful assistant.",- "replace_tokens": ["<image>", "<|image|>"]+ "replace_tokens": ["<image>", "<|image|>"],+ "extra_encode" : true,+ "default_dataset": "liuhaotian/llava_conv_58k"}\ No newline at end of filediff --git a/auto_round/mllm/templates/phi3_v.json b/auto_round/mllm/templates/phi3_v.jsonindex 955c15f7a..65e7445dc 100644--- a/auto_round/mllm/templates/phi3_v.json+++ b/auto_round/mllm/templates/phi3_v.json@@ -2,5 +2,6 @@"model_type": "phi3_v","replace_tokens": ["<image>", "<|image_1|>"],"processor": "basic",- "extra_encode" : false+ "extra_encode" : false,+ "default_dataset": "NeelNanda/pile-10k"}\ No newline at end of filediff --git a/auto_round/mllm/templates/qwen2_vl.json b/auto_round/mllm/templates/qwen2_vl.jsonindex 9435d4236..6b72a0637 100644--- a/auto_round/mllm/templates/qwen2_vl.json+++ b/auto_round/mllm/templates/qwen2_vl.json@@ -7,5 +7,7 @@"format_separator": "\n","default_system": "You are a helpful assistant.","replace_tokens": ["<image>", "<|vision_start|><|image_pad|><|vision_end|>"],- "processor": "qwen2_vl"+ "processor": "qwen2_vl",+ "extra_encode" : true,+ "default_dataset": "NeelNanda/pile-10k"}\ No newline at end of filediff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex bc6233e5e..0067161ee 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -13,6 +13,7 @@# limitations under the License.import os+import sysimport argparseimport json@@ -51,10 +52,10 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--asym", action='store_true',help="whether to use asym quantization")- self.add_argument("--dataset", type=str, default="liuhaotian/llava_conv_58k",+ self.add_argument("--dataset", type=str, default=None,help="the dataset for quantization training."- " current support llava_conv_58k,llava_instruct_80k "- "It can be a custom one.")+ " current support NeelNanda/pile-10k,llava_conv_58k,llava_instruct_80k "+ "It can be a custom one. Default is NeelNanda/pile-10k")self.add_argument("--lr", default=None, type=float,help="learning rate, if None, it will be set to 1.0/iters automatically")@@ -68,7 +69,7 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--adam", action='store_true',help="whether to use adam optimizer instead of SignSGD")- self.add_argument("--gradient_accumulate_steps", default=4, type=int,+ self.add_argument("--gradient_accumulate_steps", default=1, type=int,help="gradient accumulate steps")self.add_argument("--nblocks", default=1, type=int,@@ -153,7 +154,8 @@ class BasicArgumentParser(argparse.ArgumentParser):help="the template for building training dataset. It can be a custom one.")self.add_argument("--truncation", action="store_true",- help="whether to truncate sequences at the maximum length.")+ help="whether to truncate sequences at the maximum length."+ " Default True for pile and False for llava dataset.")self.add_argument("--to_quant_block_names", default=None, type=str,help="Names of quantitative blocks, please use commas to separate them.")@@ -197,13 +199,13 @@ def setup_parser():parser.add_argument("--group_size", default=128, type=int,help="group size")- parser.add_argument("--batch_size", "--train_bs", default=1, type=int,+ parser.add_argument("--batch_size", "--train_bs", default=8, type=int,help="train batch size")parser.add_argument("--iters", "--iter", default=200, type=int,help=" iters")- parser.add_argument("--seqlen", "--seq_len", default=256, type=int,+ parser.add_argument("--seqlen", "--seq_len", default=2048, type=int,help="sequence length")parser.add_argument("--nsamples", default=128, type=int,@@ -230,8 +232,6 @@ def tune(args):model_name = model_name[:-1]logger.info(f"start to quantize {model_name}")- assert args.dataset is not None, "dataset should not be None."-devices = args.device.replace(" ", "").split(',')use_auto_mapping = False@@ -335,6 +335,9 @@ def tune(args):print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "f"gpu")+ if "--truncation" not in sys.argv:+ args.truncation = None+autoround = round(model, tokenizer, image_processor=image_processor, dataset=args.dataset,extra_data_dir=args.extra_data_dir, bits=args.bits, group_size=args.group_size,sym=not args.asym, batch_size=args.batch_size, seqlen=seqlen, nblocks=args.nblocks,diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 3ae74f6a4..11d25c7a2 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -36,14 +36,33 @@ from .special_model_handler import shareable_keywordsdef warning_once(self, msg: str):self.warning(msg)+class AutoRoundFormatter(logging.Formatter):+ grey = "\x1b[38;20m"+ yellow = "\x1b[33;1m"+ red = "\x1b[31;20m"+ bold_red = "\x1b[31;1m"+ reset = "\x1b[0m"+ _format = "%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s"++ FORMATS = {+ logging.DEBUG: grey + _format + reset,+ logging.INFO: grey + _format + reset,+ logging.WARNING: yellow + _format + reset,+ logging.ERROR: bold_red + _format + reset,+ logging.CRITICAL: bold_red + _format + reset+ }++ def format(self, record):+ log_fmt = self.FORMATS.get(record.levelno)+ formatter = logging.Formatter(log_fmt, "%Y-%m-%d %H:%M:%S")+ return formatter.format(record)logging.Logger.warning_once = warning_oncelogger = logging.getLogger("autoround")logger.setLevel(logging.INFO)logger.propagate = Falsefh = logging.StreamHandler()-fh_formatter = logging.Formatter("%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s", "%Y-%m-%d %H:%M:%S")-fh.setFormatter(fh_formatter)+fh.setFormatter(AutoRoundFormatter())logger.addHandler(fh)import importlib