PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix tensor shape mismatch error for API usage (#582)
Weiwei9f4d31a3d48e
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 31361eb9e..e2ba0331c 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -52,7 +52,8 @@ from .utils import (llm_load_model,reset_params,init_cache, check_skippable_keywords, get_shared_keys, SUPPORTED_DTYPES, infer_bits_by_data_type,- _gguf_args_check+ _gguf_args_check,+ check_seqlen_compatible)from .low_cpu_mem.utils import get_layers_before_block@@ -204,8 +205,8 @@ class AutoRound(object):self.minmax_lr = minmax_lr or self.lrself.data_type = data_type- tmp_bits = infer_bits_by_data_type(self.data_type)- if tmp_bits<16 and tmp_bits!=bits:+ tmp_bits = infer_bits_by_data_type(self.data_type)+ if tmp_bits < 16 and tmp_bits != bits:logger.warning(f"bits set in 'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")self.bits = tmp_bits@@ -439,6 +440,24 @@ class AutoRound(object):# ##check lm_head, mixed_bits, bits, each layer supporting, etc# pass+ def _check_compatibility(self):+ # Check the legitimacy of seqlen+ if self.seqlen is not None and hasattr(self.model, "config") and \+ hasattr(self.model.config, "max_position_embeddings"):+ if self.model.config.max_position_embeddings < self.seqlen:+ logger.warning(+ f"change sequence length to {self.model.config.max_position_embeddings} " \+ "due to the limitation of max_position_embeddings")+ self.seqlen = min(self.seqlen, self.model.config.max_position_embeddings)++ if self.seqlen is not None and hasattr(self.tokenizer, "model_max_length"):+ if self.tokenizer.model_max_length < self.seqlen:+ logger.warning(+ f"change sequence length to {self.tokenizer.model_max_length} " \+ "due to the limitation of model_max_length. " \+ "You can also try to increase the model_max_length to avoid this issue.")+ self.seqlen = min(self.seqlen, self.tokenizer.model_max_length)+def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace=True, **kwargs):"""Quantizes the model and saves it in the specified format(s).@@ -504,7 +523,7 @@ class AutoRound(object):# Adjust format settings based on compatibilityfor index in range(len(formats)):format = formats[index]- if format=="auto_round":+ if format == "auto_round":if self.sym or self.bits == 3:format = format.replace('auto_round', 'auto_round:auto_gptq')formats[index] = format@@ -516,7 +535,6 @@ class AutoRound(object):if enable_awq:formats[index] = format.replace("auto_round", "auto_round:auto_awq")-# Remove duplicates from formats listdef remove_duplicates(lst):seen = set()@@ -590,6 +608,9 @@ class AutoRound(object):Returns:The quantized model and layer configurations."""+ ## TODO add common check++ self._check_compatibility()if self.iters == 0:return self.quantize_rtn()@@ -926,10 +947,12 @@ class AutoRound(object):except NotImplementedError:passexcept RuntimeError as error:- logger.warning("When quantization encounters tensor" \- " shape mismatch error, you can try to avoid it with batch_size=1")- logger.error(error)- pass+ error_msg = str(error)+ if "The expanded size of the tensor" in str(error_msg) and "must match the existing size" in error_msg:+ check_seqlen_compatible(self.seqlen, self.tokenizer, self.model)+ logger.warning("When quantization encounters tensor shape mismatch error, " \+ "you can try to avoid it with batch_size=1")+ raise errorexcept Exception as error:raise errortotal_cnt += input_ids.shape[0] if len(input_ids.shape) > 1 else 1@@ -1712,7 +1735,7 @@ class AutoRound(object):"The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"" particularly for 2-bit quantization and smaller models."" We recommend exporting to either the AutoAWQ format ( only 4 bits) or "- "the AutoRound format(2/4/8 bits)."+ "the AutoRound format(2/3/4/8 bits).")if "awq" in format and not self.bits == 4:raise ValueError("The AWQ format only supports W4 quantization ")@@ -2204,4 +2227,3 @@ class AutoRoundAdam(AutoRoundOPT):super_group_size=super_group_size,**kwargs,)-diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 804d428be..190bcffa0 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -392,15 +392,6 @@ def tune(args):from auto_round import AutoRound, AutoRoundAdam- seqlen = args.seqlen-- if hasattr(tokenizer, "model_max_length"):- if tokenizer.model_max_length < seqlen:- logger.info(- f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length")- seqlen = min(seqlen, tokenizer.model_max_length)- args.seqlen = seqlen-if "bloom" in model_name:args.low_gpu_mem_usage = False@@ -467,7 +458,7 @@ def tune(args):sym=not args.asym,batch_size=args.batch_size,dataset=args.dataset,- seqlen=seqlen,+ seqlen=args.seqlen,nblocks=args.nblocks,iters=args.iters,lr=args.lr,@@ -769,3 +760,4 @@ def eval_task_by_task(print("total eval time:", time.time() - st)+diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 7f76d3204..768f7c394 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -328,6 +328,17 @@ def tune(args):from auto_round import AutoRoundMLLM+ seqlen = args.seqlen+ if seqlen is not None and hasattr(model, "config") and hasattr(model.config, "max_position_embeddings"):+ seqlen = min(seqlen, model.config.max_position_embeddings)++ if seqlen is not None and hasattr(tokenizer, "model_max_length"):+ if tokenizer.model_max_length < seqlen:+ logger.info(+ f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length")+ seqlen = min(seqlen, tokenizer.model_max_length)+ args.seqlen = seqlen+model = model.eval()round = AutoRoundMLLM@@ -420,7 +431,7 @@ def tune(args):group_size=args.group_size,sym=not args.asym,batch_size=args.batch_size,- seqlen=args.seqlen,+ seqlen=seqlen,nblocks=args.nblocks,iters=args.iters,lr=args.lr,@@ -564,3 +575,4 @@ def lmms_eval(args):return results+diff --git a/auto_round/utils.py b/auto_round/utils.pyindex ce46b5811..e9ae56cac 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1531,3 +1531,26 @@ def check_start_with_block_name(name: str, block_name_to_quantize: list):return Truereturn False++def check_seqlen_compatible(input_seqlen, tokenizer=None, model=None):+ """+ Check whether the input sequence length is within the limits defined+ by the tokenizer and the model configuration.++ Args:+ input_seqlen (int): The length of the input sequence.+ tokenizer: Optional, a HuggingFace tokenizer object.+ model: Optional, a HuggingFace model object.++ Returns:+ ValueError: if the input length is not valid, riase Error.+ """+ if model is not None and hasattr(model, 'config'):+ model_config = model.config+ if hasattr(model_config, 'max_position_embeddings') and input_seqlen > model_config.max_position_embeddings:+ raise ValueError(f"seqlen({input_seqlen}) exceeds model.config.max_position_embeddings(" \+ f"{model_config.max_position_embeddings}). Please lowering '--seqlen'")+ if tokenizer is not None and hasattr(tokenizer, 'model_max_length') and input_seqlen > tokenizer.model_max_length:+ raise ValueError(f"seqlen({input_seqlen}) exceeds tokenizer.model_max_length({tokenizer.model_max_length}). " \+ "Please oncider Consider lowering the '--seqlen' or increasing tokenizer.model_max_length.")+