PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix glm4-9b batch dim issue (#304)
wenhuach213376edd365ee
diff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex c8de3ea93..f031c758f 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -13,5 +13,7 @@# limitations under the License.from .autoround import AutoRound, AutoRoundAdam, AutoRoundOPTfrom .mllm import AutoRoundMLLM+from auto_round.utils import LazyImport+from .auto_quantizer import AutoHfQuantizer,AutoRoundConfig-from .version import __version__\ No newline at end of file+from .version import __version__diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 579248876..564ba6fa3 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -25,12 +25,12 @@ import acceleratefrom packaging import versionfrom .quantizer import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layer, \WrapperTransformerConv1d-from .special_model_handler import (check_hidden_state_dim,- shareable_keywords,- special_model_init,- reset_params,- check_skippable_keywords- )+from .special_model_handler import (+ shareable_keywords,+ init_cache_for_special_model,+ reset_params,+ check_skippable_keywords+)from .utils import (CpuInfo,block_forward,@@ -206,7 +206,7 @@ class AutoRound(object):self.dynamic_max_gap = dynamic_max_gapself.lr_scheduler = lr_schedulerself.optimizer = self.get_optimizer(None)- self.input_dim = None+ self.batch_dim = Noneself.infer_bs_coeff = 1self.set_layerwise_config(self.layer_config) ##better place in the end@@ -250,6 +250,12 @@ class AutoRound(object):if "mx_fp" in self.data_type and self.group_size != 32:logger.warning("mx_fp should only support group_size of 32 in real deployment")+ if self.batch_size > self.nsamples:+ logger.warning(+ f"reset batch_size to nsamples({self.nsamples}) "+ f"as batch_size({self.batch_size}) must be smaller than nsamples")+ self.batch_size = self.nsamples+def quantize(self):"""Quantize the model and return the quantized model along with layer configurations.the entry of AutoRound.@@ -283,8 +289,8 @@ class AutoRound(object):keys = inputs.keys()input_id_str = [key for key in keys if key.startswith('hidden_state')]if len(input_id_str) != 1:- raise RuntimeError("hidden_states arg mismatch error," \- " please check the input kwargs of block forward for more details.")+ raise RuntimeError(f"hidden_states arg mismatch error,"+ "please raise an issue in https://github.com/intel/auto-round/issues")inputs["input_ids"] = inputs.pop(input_id_str[0], None)clear_memory(self.inputs)@@ -462,12 +468,12 @@ class AutoRound(object):input_others,indices,self.seqlen,- self.input_dim+ self.batch_dim)tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(cache_device)- output.extend(list(torch.split(tmp_output, 1, dim=self.input_dim)))+ output.extend(list(torch.split(tmp_output, 1, dim=self.batch_dim)))if self.low_gpu_mem_usage:clear_memory()@@ -547,7 +553,7 @@ class AutoRound(object):passexcept RuntimeError as error:logger.warning("When quantization encounters tensor" \- " shape mismatch error, you can try to avoid it with batch_size=1")+ " shape mismatch error, you can try to avoid it with batch_size=1")logger.error(error)passexcept Exception as error:@@ -672,7 +678,7 @@ class AutoRound(object):Returns:function: The forward function."""-+def post_process_cache_data(batch_size, data, data_name):"""Processes store data for batch handling, reshaping if necessary.@@ -710,18 +716,28 @@ class AutoRound(object):"""if name not in self.inputs:self.inputs[name] = {}- special_model_init(self.model, positional_inputs, self.inputs[name])-- if self.input_dim is None:- self.input_dim = check_hidden_state_dim(self.model, positional_inputs)-+ init_cache_for_special_model(self.model, positional_inputs, self.inputs[name])++ if self.batch_dim is None :+ self.batch_dim = 0+ if hidden_states is not None and self.batch_size>1:+ if hidden_states.shape[0] > self.batch_size:+ self.batch_dim = 1+ if len(hidden_states.shape) > 1 and hidden_states.shape[1] > self.batch_size:+ logger.error(+ f"this model has not been supported, "+ f"please raise an issue in https://github.com/intel/auto-round/issues"+ f" or try to set the `batch_size` to 1 and "+ f"`gradient_accumulate_steps` to your current batch size.")+ exit(-1)+if hidden_states is not None:kwargs['hidden_states'] = hidden_statesfor key in kwargs.keys():if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) \or isinstance(kwargs[key], tuple):- if key not in self.inputs[name].keys(): # initialization+ if key not in self.inputs[name].keys(): # initializationdata = to_device(kwargs[key], device=torch.device("cpu"))if data is None or (self.batch_size > 1 and key in shareable_keywords):self.inputs[name][key] = data@@ -730,16 +746,16 @@ class AutoRound(object):self.inputs[name][key] = [data]else:data = post_process_cache_data(self.batch_size, data, key)- self.inputs[name][key] = list(torch.split(data, 1, dim=self.input_dim))- else: # append cache inputs+ self.inputs[name][key] = list(torch.split(data, 1, dim=self.batch_dim))+ else: # append cache inputsnew_data = post_process_cache_data(self.batch_size, kwargs[key], key)- if new_data is None: # shareable args or NoneType+ if new_data is None: # shareable args or NoneTypecontinuenew_data = to_device(new_data, device=torch.device("cpu"))if self.batch_size <= 1:self.inputs[name][key].append(new_data)else:- self.inputs[name][key].extend(list(torch.split(new_data, 1, dim=self.input_dim)))+ self.inputs[name][key].extend(list(torch.split(new_data, 1, dim=self.batch_dim)))elif isinstance(kwargs[key], (str, bool, type(None))):if key not in self.inputs[name].keys():self.inputs[name][key] = kwargs[key]@@ -756,7 +772,7 @@ class AutoRound(object):kwargs.pop('hidden_states')return m.orig_forward(hidden_states, *positional_inputs, **kwargs)else:- #Currently only for Llama-3.2-Vision-Instruct Series+ # Currently only for Llama-3.2-Vision-Instruct Seriesreturn m.orig_forward(*positional_inputs, **kwargs)return forward@@ -996,11 +1012,11 @@ class AutoRound(object):input_others,indices,seqlen=self.seqlen,- input_dim=self.input_dim,+ batch_dim=self.batch_dim,)current_output = [output[x] for x in indices]- current_output = torch.cat(current_output, dim=self.input_dim)+ current_output = torch.cat(current_output, dim=self.batch_dim)current_output = to_device(current_output, device)@@ -1627,4 +1643,3 @@ class AutoRoundAdam(AutoRoundOPT):optimizer=optimizer,**kwargs,)-diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 871e0e410..181fd53ef 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -43,116 +43,122 @@ from auto_round.eval.evaluation import simple_evaluatefrom auto_round.utils import detect_device, get_library_version, detect_device_countfrom auto_round.utils import logger+class BasicArgumentParser(argparse.ArgumentParser):def __init__(self, *args, **kwargs):- super().__init__( *args, **kwargs)- self.add_argument("--model", default="facebook/opt-125m", help="model name or path")+ super().__init__(*args, **kwargs)+ self.add_argument("--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m",+ help="model name or path")self.add_argument('--eval', action='store_true',- help="whether to use eval only mode.")+ help="whether to use eval only mode")self.add_argument("--bits", default=4, type=int,- help="number of bits")+ help="number of weight bits")self.add_argument("--eval_bs", default=None, type=int,- help="eval batch size")+ help="batch size in evaluation")self.add_argument("--device", default="auto", type=str,- help="The device to be used for tuning. The default is set to auto/None,"- "allowing for automatic detection."- " Currently, device settings support CPU, GPU, and HPU.")-+ help="the device to be used for tuning. The default is set to auto,"+ "allowing for automatic detection."+ "Currently, device settings support CPU, GPU, and HPU.")+self.add_argument("--asym", action='store_true',- help=" asym quantization")+ help="whether to use asym quantization")self.add_argument("--dataset", default="NeelNanda/pile-10k", type=str,- help="The dataset for quantization training. It can be a custom one.")+ help="the dataset for quantization training")self.add_argument("--lr", default=None, type=float,- help="learning rate, if None, it will be set to 1.0/iters automatically")+ help="learning rate, if None, it will be set to 1.0/iters automatically")self.add_argument("--minmax_lr", default=None, type=float,- help="minmax learning rate, if None,it will beset to be the same with lr")+ help="minmax learning rate, if None, it will beset to be the same with lr")self.add_argument("--seed", default=42, type=int,- help="seed")+ help="random seed")self.add_argument("--adam", action='store_true',- help="adam")-- self.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")+ help="whether to use adam optimizer instead of SignSGD")++ self.add_argument("--gradient_accumulate_steps", default=1, type=int,+ help="gradient accumulate steps")- self.add_argument("--nblocks", default=1, type=int, help="num of blocks to tune together")+ self.add_argument("--nblocks", default=1, type=int,+ help="how many blocks to tune together")self.add_argument("--low_gpu_mem_usage", action='store_true',- help="lower gpu memory usage but 50-100% slower")-+ help="offload intermediate features to cpu")+self.add_argument("--format", default=None, type=str,- help="The format in which to save the model. "- "The options are 'auto_round', 'auto_round:gptq','auto_round:awq',"- " 'auto_gptq', 'auto_awq', 'itrex', 'itrex_xpu' and 'fake'."- "default to 'auto_round."- )+ choices=["auto_round", "auto_gptq", "auto_awq", "auto_round:gptq", "auto_round:auto_gptq",+ "auto_round:auto_gptq:marlin", "auto_round:gptq:marlin", "auto_round:auto_awq",+ "auto_round:awq", "auto_awq", "itrex", "iterx_xpu", "fake"],+ help="the format to save the model"+ )- self.add_argument("--data_type", default='int',- help="data type for tuning, 'int', 'mx_fp' and etc.")+ self.add_argument("--data_type", "--dtype", default='int',+ help="data type for tuning, 'int', 'mx_fp' and etc")- self.add_argument("--scale_dtype", default='fp16',- help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")+ self.add_argument("--scale_dtype", default='fp16', choices=["fp16", "float16",+ "bf16", "bfloat16", "fp32", "float32"],+ help="scale data type to use for quantization")self.add_argument("--tasks",- default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1," \- "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge",- help="lm-eval tasks for lm_eval version 0.4")+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1," \+ "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge",+ help="lm-eval tasks")self.add_argument("--output_dir", default="./tmp_autoround", type=str,- help="Where to store the final model.")+ help="the directory to save quantized model")self.add_argument("--disable_eval", action='store_true',- help="Whether to do lm-eval evaluation after tuning.")+ help="whether to do lm-eval evaluation after tuning")self.add_argument("--disable_amp", action='store_true',- help="disable amp")+ help="disable amp")self.add_argument("--disable_minmax_tuning", action='store_true',- help="whether disable enable weight minmax tuning")+ help="whether to disable enable weight minmax tuning")self.add_argument("--enable_norm_bias_tuning", action='store_true',- help="whether enable norm bias tuning")+ help="whether to enable norm bias tuning")self.add_argument("--disable_trust_remote_code", action='store_true',- help="Whether to disable trust_remote_code")+ help="whether to disable trust_remote_code")self.add_argument("--disable_quanted_input", action='store_true',- help="whether to disuse the output of quantized block to tune the next block")+ help="whether to disuse the output of quantized block to tune the next block")self.add_argument("--quant_lm_head", action='store_true',- help="quant_lm_head")+ help="whether to quant lm_head")- self.add_argument("--low_cpu_mem_mode", default=0, type=int,- help="Choose which low cpu memory mode to use. "- "Can significantly reduce cpu memory footprint but cost more time."- "1 means choose block-wise mode, load the weights of each block"- " from disk when tuning and release the memory of the block after tuning."- "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"- " minimum memory consumption and also slowest running speed."- "others means not use low cpu memory. Default to 0, not use low cpu memory.")+ self.add_argument("--low_cpu_mem_mode", default=0, type=int, choices=[0, 1, 2],+ help="choose which low cpu memory mode to use. "+ "Can significantly reduce cpu memory footprint but cost more time."+ "1 means choose block-wise mode, load the weights of each block"+ " from disk when tuning and release the memory of the block after tuning."+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"+ " minimum memory consumption and also slowest running speed."+ "others means not use low cpu memory. Default to 0, not use low cpu memory.")self.add_argument("--low_cpu_mem_tmp_dir", default=None, type=str,- help="temp work space to store the temporary files "- "when using low cpu memory mode. Will remove after tuning.")+ help="temporary work space to store the temporary files "+ "when using low cpu memory mode. Will remove after tuning.")- self.add_argument("--model_dtype", default=None, type=str,- help="force to convert the dtype, some backends supports fp16 dtype better")+ self.add_argument("--model_dtype", default=None, type=str, choices=["fp16", "float16",+ "bf16", "bfloat16", "fp32", "float32"],+ help="force to convert the dtype, some backends supports fp16 dtype better")self.add_argument("--act_bits", default=16, type=int,- help="activation bits")+ help="activation bits")++ self.add_argument("--fp_layers", default="", type=str,+ help="list of Layer names to maintain original data type")- self.add_argument("--fp_layers_list", default="", type=str,- help="List of Layers to maintain original data type")-self.add_argument("--not_use_best_mse", action='store_true',- help="To determine whether the quantization should handle vision component.")+ help="whether to use the iter of best mes loss in the tuning phase")def setup_parser():@@ -161,14 +167,14 @@ def setup_parser():parser.add_argument("--group_size", default=128, type=int,help="group size")- parser.add_argument("--batch_size", default=8, type=int,+ parser.add_argument("--batch_size", "--train_bs", default=8, type=int,help="train batch size")- parser.add_argument("--iters", default=200, type=int,- help=" iters")+ parser.add_argument("--iters", "--iter", default=200, type=int,+ help="iteration to tune each block")- parser.add_argument("--seqlen", default=2048, type=int,- help="sequence length")+ parser.add_argument("--seqlen", "--seq_len", default=2048, type=int,+ help="sequence length of the calibration samples")parser.add_argument("--nsamples", default=128, type=int,help="number of samples")@@ -176,20 +182,21 @@ def setup_parser():args = parser.parse_args()return args+def setup_best_parser():parser = BasicArgumentParser()parser.add_argument("--group_size", default=128, type=int,help="group size")- parser.add_argument("--batch_size", default=8, type=int,+ parser.add_argument("--batch_size", "--train_bs", default=8, type=int,help="train batch size")- parser.add_argument("--iters", default=1000, type=int,- help=" iters")+ parser.add_argument("--iters","--iter", default=1000, type=int,+ help="iterations to tune each block")- parser.add_argument("--seqlen", default=2048, type=int,- help="sequence length")+ parser.add_argument("--seqlen", "--seq_len", default=2048, type=int,+ help="sequence length of the calibration samples")parser.add_argument("--nsamples", default=512, type=int,help="number of samples")@@ -199,20 +206,21 @@ def setup_best_parser():return args+def setup_fast_parser():parser = BasicArgumentParser()parser.add_argument("--group_size", default=128, type=int,help="group size")- parser.add_argument("--batch_size", default=4, type=int,+ parser.add_argument("--batch_size", "--train_bs", default=4, type=int,help="train batch size")parser.add_argument("--iters", default=200, type=int,- help=" iters")+ help="iterations to tune each block")- parser.add_argument("--seqlen", default=512, type=int,- help="sequence length")+ parser.add_argument("--seqlen", "--seq_len", default=512, type=int,+ help="sequence length of the calibration samples")parser.add_argument("--nsamples", default=128, type=int,help="number of samples")@@ -221,6 +229,7 @@ def setup_fast_parser():return args+def tune(args):tasks = args.tasksif args.format is None:@@ -303,7 +312,6 @@ def tune(args):if args.model_dtype == "bfloat16" or args.model_dtype == "bfp16":model = model.to(torch.bfloat16)-if hasattr(tokenizer, "model_max_length"):if tokenizer.model_max_length < seqlen:logger.info(@@ -326,12 +334,12 @@ def tune(args):logger.info(f"{n} will not be quantized due to its shape not being divisible by 32,"" resulting in an exporting issue to autogptq")- fp_layers_list = args.fp_layers_list.split(",")- if bool(fp_layers_list):+ fp_layers = args.fp_layers.split(",")+ if bool(fp_layers):for n, m in model.named_modules():if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):name = n.split('.')[-1]- if n in fp_layers_list or name in fp_layers_list:+ if n in fp_layers or name in fp_layers:layer_config[n] = {"bits": 16}logger.info(f"{n} will not be quantized.")@@ -376,12 +384,11 @@ def tune(args):if "cpu" not in device_str:torch.cuda.empty_cache()- if model_name.split('/')[-1] == ".":- export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")+ if model_name.split('/')[-1].strip('.') == "":+ export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")else:export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")-format_list = args.format.replace(' ', '').split(',')inplace = False if len(format_list) > 1 else Truefor format_ in format_list:@@ -432,4 +439,3 @@ def eval(args):from lm_eval.utils import make_table # pylint: disable=E0401print(make_table(res))-diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 3ec7404db..aba755584 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -25,138 +25,146 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig, AutoPrfrom auto_round.utils import detect_devicefrom auto_round.utils import logger+class BasicArgumentParser(argparse.ArgumentParser):def __init__(self, *args, **kwargs):- super().__init__( *args, **kwargs)- self.add_argument("--model", default="facebook/opt-125m", help="model name or path")+ super().__init__(*args, **kwargs)+ self.add_argument("--model", "--model_name", "--model_name_or_path",+ default="facebook/opt-125m",+ help="model name or path")self.add_argument('--eval', action='store_true',- help="whether to use eval only mode.")+ help="whether to use eval only mode.")self.add_argument("--bits", default=4, type=int,- help="number of bits")+ help="weight bits")self.add_argument("--eval_bs", default=None, type=int,- help="eval batch size")+ help="batch size in evaluation")self.add_argument("--device", default="auto", type=str,- help="The device to be used for tuning. The default is set to auto/None,"- "allowing for automatic detection."- " Currently, device settings support CPU, GPU, and HPU.")-+ help="the device to be used for tuning. The default is set to auto,"+ "allowing for automatic detection."+ "Currently, device settings support CPU, GPU, and HPU.")+self.add_argument("--asym", action='store_true',- help=" asym quantization")+ help="whether to use asym quantization")self.add_argument("--dataset", type=str, default=None,- help="The dataset for quantization training. It can be a custom one.")+ help="the dataset for quantization training. It can be a custom one.")self.add_argument("--lr", default=None, type=float,- help="learning rate, if None, it will be set to 1.0/iters automatically")+ help="learning rate, if None, it will be set to 1.0/iters automatically")self.add_argument("--minmax_lr", default=None, type=float,- help="minmax learning rate, if None,it will beset to be the same with lr")+ help="minmax learning rate, if None,it will beset to be the same with lr")self.add_argument("--seed", default=42, type=int,- help="seed")+ help="random seed")self.add_argument("--adam", action='store_true',- help="adam")-- self.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")+ help="whether to use adam optimizer instead of SignSGD")- self.add_argument("--nblocks", default=1, type=int, help="num of blocks to tune together")+ self.add_argument("--gradient_accumulate_steps", default=1, type=int,+ help="gradient accumulate steps")++ self.add_argument("--nblocks", default=1, type=int,+ help="how many blocks to tune together")self.add_argument("--low_gpu_mem_usage", action='store_true',- help="lower gpu memory usage but 50-100% slower")-- self.add_argument("--format", default=None, type=str,- help="The format in which to save the model. "- "The options are 'auto_round', 'auto_round:gptq','auto_round:awq',"- " and 'fake'.default to 'auto_round."- )+ help="offload intermediate features to cpu")++ self.add_argument("--format", default="auto_round", type=str,+ choices=["auto_round", "auto_round:gptq", "auto_round:auto_gptq", "auto_round:awq",+ "auto_round:auto_awq", "fake"],+ help="the format to save the model"+ )- self.add_argument("--data_type", default='int',- help="data type for tuning, 'int', 'mx_fp' and etc.")+ self.add_argument("--data_type", "--dtype", default='int',+ help="data type for tuning, 'int', 'mx_fp' and etc")- self.add_argument("--scale_dtype", default='fp16',- help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")+ self.add_argument("--scale_dtype", default='fp16', choices=["fp16", "float16",+ "bf16", "bfloat16", "fp32", "float32"],+ help="scale data type to use for quantization")self.add_argument("--output_dir", default="./tmp_autoround", type=str,- help="Where to store the final model.")+ help="the directory to save quantized model")self.add_argument("--disable_amp", action='store_true',- help="disable amp")+ help="disable amp")self.add_argument("--disable_minmax_tuning", action='store_true',- help="whether disable enable weight minmax tuning")+ help="whether disable enable weight minmax tuning")self.add_argument("--enable_norm_bias_tuning", action='store_true',- help="whether enable norm bias tuning")+ help="whether enable norm bias tuning")self.add_argument("--disable_trust_remote_code", action='store_true',- help="Whether to disable trust_remote_code")+ help="whether to disable trust_remote_code")self.add_argument("--disable_quanted_input", action='store_true',- help="whether to disuse the output of quantized block to tune the next block")+ help="whether to disuse the output of quantized block to tune the next block")self.add_argument("--quant_lm_head", action='store_true',- help="quant_lm_head")+ help="whether to quant lm_head")- self.add_argument("--low_cpu_mem_mode", default=0, type=int,- help="Choose which low cpu memory mode to use. "- "Can significantly reduce cpu memory footprint but cost more time."- "1 means choose block-wise mode, load the weights of each block"- " from disk when tuning and release the memory of the block after tuning."- "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"- " minimum memory consumption and also slowest running speed."- "others means not use low cpu memory. Default to 0, not use low cpu memory.")+ self.add_argument("--low_cpu_mem_mode", default=0, type=int, choices=[0, 1, 2],+ help="choose which low cpu memory mode to use. "+ "Can significantly reduce cpu memory footprint but cost more time."+ "1 means choose block-wise mode, load the weights of each block"+ " from disk when tuning and release the memory of the block after tuning."+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning,"+ " minimum memory consumption and also slowest running speed."+ "others means not use low cpu memory. Default to 0, not use low cpu memory.")self.add_argument("--low_cpu_mem_tmp_dir", default=None, type=str,- help="temp work space to store the temporary files "- "when using low cpu memory mode. Will remove after tuning.")+ help="temporary work space to store the temporary files "+ "when using low cpu memory mode. Will remove after tuning.")- self.add_argument("--model_dtype", default=None, type=str,- help="force to convert the dtype, some backends supports fp16 dtype better")+ self.add_argument("--model_dtype", default=None, type=str, choices=["fp16", "float16",+ "bf16", "bfloat16", "fp32", "float32"],+ help="force to convert the dtype, some backends supports fp16 dtype better")self.add_argument("--act_bits", default=32, type=int,- help="activation bits")+ help="activation bits")++ self.add_argument("--fp_layers", default="", type=str,+ help="layers to maintain original data type")- self.add_argument("--fp_layers_list", default="", type=str,- help="List of Layers to maintain original data type")-self.add_argument("--not_use_best_mse", action='store_true',- help="To determine whether the quantization should handle vision component.")+ help="whether to use the iter of best mes loss in the tuning phase")+## ======================= VLM =======================self.add_argument("--quant_nontext_module", action='store_true',- help="To determine whether the quantization should handle vision component.")+ help="whether to quantize non-text module, e.g. vision component")self.add_argument("--extra_data_dir", default="", type=str,- help="Dataset dir for storing images/audio/videos. "- "Can be a dir path or multiple dir path with format as "- "'image=path_to_image,video=path_to_video,audio=path_to_audio'"- "By default, it will search in the relative path.")-+ help="dataset dir for storing images/audio/videos. "+ "Can be a dir path or multiple dir path with format as "+ "'image=path_to_image,video=path_to_video,audio=path_to_audio'"+ "By default, it will search in the relative path.")+self.add_argument("--template", default=None, type=str,- help="The template for building training dataset. It can be a custom one.")-+ help="the template for building training dataset. It can be a custom one.")+## ======================= VLM eval=======================self.add_argument("--tasks", type=str, default="COCO_VAL",help="eval tasks for VLMEvalKit.")# Args that only apply to Video Datasetself.add_argument("--nframe", type=int, default=8,- help="The number of frames to sample from a video,"- " only applicable to the evaluation of video benchmarks.")- self.add_argument("--pack", action='store_true',- help="A video may associate with multiple questions, if pack==True,"- " will ask all questions for a video in a single")+ help="the number of frames to sample from a video,"+ " only applicable to the evaluation of video benchmarks.")+ self.add_argument("--pack", action='store_true',+ help="a video may associate with multiple questions, if pack==True,"+ " will ask all questions for a video in a single")self.add_argument("--use-subtitle", action='store_true')self.add_argument("--fps", type=float, default=-1)# Work Dir# Infer + Eval or Infer Onlyself.add_argument("--mode", type=str, default='all', choices=['all', 'infer'],- help="When mode set to 'all', will perform both inference and evaluation;"- " when set to 'infer' will only perform the inference.")+ help="when mode set to 'all', will perform both inference and evaluation;"+ " when set to 'infer' will only perform the inference.")self.add_argument('--eval_data_dir', type=str, default=None,help='path for VLMEvalKit to store the eval data. Default will store in ~/LMUData')# API Kwargs, Apply to API VLMs and Judge API LLMs@@ -167,23 +175,24 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument('--verbose', action='store_true')# Configuration for Resume# Ignore: will not rerun failed VLM inference- self.add_argument('--ignore', action='store_true', help='Ignore failed indices. ')+ self.add_argument('--ignore', action='store_true', help='ignore failed indices. ')# Rerun: will remove all evaluation temp filesself.add_argument('--rerun', action='store_true')+def setup_parser():parser = BasicArgumentParser()parser.add_argument("--group_size", default=128, type=int,help="group size")- parser.add_argument("--batch_size", default=1, type=int,+ parser.add_argument("--batch_size", "--train_bs", default=1, type=int,help="train batch size")- parser.add_argument("--iters", default=200, type=int,+ parser.add_argument("--iters", "--iter", default=200, type=int,help=" iters")- parser.add_argument("--seqlen", default=2048, type=int,+ parser.add_argument("--seqlen", "--seq_len", default=2048, type=int,help="sequence length")parser.add_argument("--nsamples", default=128, type=int,@@ -192,10 +201,11 @@ def setup_parser():args = parser.parse_args()return args+def tune(args):if args.format is None:args.format = "auto_round"-+model_name = args.modelif model_name[-1] == "/":model_name = model_name[:-1]@@ -207,7 +217,7 @@ def tune(args):torch_dtype = "auto"if "hpu" in device_str:torch_dtype = torch.bfloat16-+# load_modelconfig = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)tokenizer = AutoTokenizer.from_pretrained(model_name)@@ -223,7 +233,7 @@ def tune(args):else:cls = AutoModelForCausalLMmodel = cls.from_pretrained(- model_name,trust_remote_code=not args.disable_trust_remote_code, torch_dtype=torch_dtype)+ model_name, trust_remote_code=not args.disable_trust_remote_code, torch_dtype=torch_dtype)if "cogvlm2" in model_name:model.config.model_type = "cogvlm2"@@ -240,7 +250,7 @@ def tune(args):if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:layer_config[n] = {"bits": 32}logger.info(- f"{n} will not be quantized due to its shape not being divisible by 32,"+ f"{n} will not be quantized due to its shape not being divisible by 32,"" resulting in an exporting issue to autogptq")lm_head_layer_name = "lm_head"for n, _ in model.named_modules():@@ -266,7 +276,7 @@ def tune(args):if args.quant_lm_head and args.low_gpu_mem_usage:print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "f"gpu")-+autoround = round(model, tokenizer, dataset=args.dataset, extra_data_dir=args.extra_data_dir,bits=args.bits, group_size=args.group_size, sym=not args.asym,batch_size=args.batch_size, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters,@@ -281,9 +291,9 @@ def tune(args):model.eval()if args.device != "cpu":torch.cuda.empty_cache()-- if model_name.split('/')[-1] == ".":- export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")++ if model_name.split('/')[-1].strip('.') == "":+ export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")else:export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")@@ -304,12 +314,12 @@ def eval(args):data_store_dir=args.eval_data_dir,dataset=args.tasks,pack=args.pack,- use_subtitle = args.use_subtitle,- fps = args.fps,- nframe = args.nframe,- rerun = args.rerun,- judge = args.judge,- verbose = args.verbose,- mode = args.mode,- ignore = args.ignore- )\ No newline at end of file+ use_subtitle=args.use_subtitle,+ fps=args.fps,+ nframe=args.nframe,+ rerun=args.rerun,+ judge=args.judge,+ verbose=args.verbose,+ mode=args.mode,+ ignore=args.ignore+ )diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex d0f07c3f4..c8d1fdcbd 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -14,7 +14,6 @@import torchfrom collections import UserDict-special_states_dim_tuple = ("chatglm",) # input_dim is not the default dimension 0shareable_keywords = ("position_ids", "cache_position", "position_embeddings")mllms_with_limited_bs = ("llava", "qwen2-vl", "phi3_v", "mllama") # Limitations on batch_sizeskippable_cache_keys = ("past_key_value",)@@ -50,24 +49,7 @@ def to_device(input, device=torch.device("cpu")):return input-def check_hidden_state_dim(model, positional_inputs):- """Check the concatenable dimension of hidden states.-- Args:- positional_inputs: The positional arguments.-- Returns:- int: 1 if the model type is 'chatglm' and positional arguments are not None, 0 otherwise.- """- is_special = False- for key in special_states_dim_tuple:- if hasattr(model, "config") and key in model.config.model_type:- is_special = True- break- return int(is_special and positional_inputs is not None)---def special_model_init(model, positional_inputs, inputs):+def init_cache_for_special_model(model, positional_inputs, inputs):"""Initializes special model inputs by adding positional inputs if missing.diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 9b21a15cf..d12d2164e 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -348,7 +348,7 @@ def collect_best_params(block):@torch.no_grad()def sampling_inputs(input_ids, input_others, indices, seqlen,- input_dim=0):+ batch_dim=0):"""Samples inputs based on the given indices and sequence length.Args:@@ -362,7 +362,7 @@ def sampling_inputs(input_ids, input_others, indices, seqlen,current_input_others: The sampled other input data."""current_input_ids = [input_ids[i] for i in indices]- current_input_ids = torch.cat(current_input_ids, dim=input_dim)+ current_input_ids = torch.cat(current_input_ids, dim=batch_dim)current_input_others = {"positional_inputs": input_others["positional_inputs"]}for key in input_others.keys():if "positional_inputs" in key:diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex eab5069a1..f55551d33 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -31,7 +31,7 @@ os.environ["TOKENIZERS_PARALLELISM"] = "false"if __name__ == '__main__':parser.add_argument(- "--model_name", default="facebook/opt-125m"+ "--model_name", "--model", "--model_name_or_path", default="facebook/opt-125m")parser.add_argument("--bits", default=4, type=int,@@ -40,7 +40,7 @@ if __name__ == '__main__':parser.add_argument("--group_size", default=128, type=int,help="group size")- parser.add_argument("--batch_size", default=8, type=int,+ parser.add_argument("--batch_size", "--train_bs", default=8, type=int,help="train batch size")parser.add_argument("--eval_bs", default=None, type=int,@@ -53,7 +53,7 @@ if __name__ == '__main__':parser.add_argument("--asym", action='store_true',help=" asym quantization")- parser.add_argument("--iters", default=200, type=int,+ parser.add_argument("--iters", "--iter", default=200, type=int,help=" iters")parser.add_argument("--dataset", default="NeelNanda/pile-10k", type=str,@@ -77,7 +77,7 @@ if __name__ == '__main__':parser.add_argument("--adam", action='store_true',help="adam")- parser.add_argument("--seqlen", default=2048, type=int,+ parser.add_argument("--seqlen", "--seq_len", default=2048, type=int,help="sequence length")parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")@@ -88,22 +88,20 @@ if __name__ == '__main__':help="number of samples")parser.add_argument("--low_gpu_mem_usage", action='store_true',- help="lower gpu memory but 50%-100% slower")+ help="offload intermediate features to cpu")parser.add_argument("--enable_minmax_tuning", action='store_true',help="enable_minmax_tuning is deprecated")- parser.add_argument("--deployment_device", default=None, type=str,- help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'xpu', 'gpu(auto_gptq)' and 'auto_round'."- "default to 'auto_round', 'fake' indicating that it only performs fake quantization and won't be exported to any device.")parser.add_argument("--format", default=None, type=str,+ choices=["auto_round", "auto_gptq", "auto_awq", "auto_round:gptq", "auto_round:auto_gptq",+ "auto_round:auto_gptq:marlin", "auto_round:gptq:marlin", "auto_round:auto_awq",+ "auto_round:awq", "auto_awq", "itrex", "iterx_xpu", "fake"],help="The format in which to save the model. "- "The options are 'auto_round', 'auto_gptq', 'auto_awq', 'itrex', 'itrex_xpu' and 'fake'."- "default to 'auto_round.")- parser.add_argument("--data_type", default='int',+ parser.add_argument("--data_type", "--dtype", default='int',help="data type for tuning, 'int', 'mx_fp' and etc.")parser.add_argument("--scale_dtype", default='fp16',@@ -115,10 +113,10 @@ if __name__ == '__main__':help="lm-eval tasks for lm_eval version 0.4")parser.add_argument("--output_dir", default="./tmp_autoround", type=str,- help="Where to store the final model.")+ help="the directory to save quantized model")parser.add_argument("--disable_eval", action='store_true',- help="Whether to do lmeval evaluation.")+ help="whether to do lmeval evaluation.")parser.add_argument("--disable_amp", action='store_true',help="disable amp")@@ -175,22 +173,6 @@ if __name__ == '__main__':tasks = args.tasksuse_eval_legacy = False- if args.format and args.deployment_device:- assert False, "please only specify one of format and deployment_device"-- if args.deployment_device is None and args.format is None:- args.format = "auto_round"-- if args.deployment_device:- warnings.warn("The deployment_device is deprecated and will be removed in future version."- "Please use format instead", DeprecationWarning)- if "gpu" in args.deployment_device and args.asym is True:- print(- "warning: The auto_gptq kernel has issues with asymmetric quantization. It is recommended to use sym quantization or --format='auto_round'")-- if "marlin" in args.deployment_device and args.asym is True:- assert False, "marlin backend only supports sym quantization, please remove --asym"-model_name = args.model_nameif model_name[-1] == "/":model_name = model_name[:-1]@@ -318,10 +300,6 @@ if __name__ == '__main__':breakif args.quant_lm_head:layer_config[lm_head_layer_name] = {"bits": args.bits}- transformers_version = [int(item) for item in transformers.__version__.split('.')[:2]]- if transformers_version[0] == 4 and transformers_version[1] < 38:- error_message = "Please upgrade transformers>=4.38.0 to support lm-head quantization."- raise EnvironmentError(error_message)autoround = round(model, tokenizer, args.bits, args.group_size, sym=not args.asym, batch_size=args.batch_size,dataset=args.dataset, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,@@ -344,56 +322,20 @@ if __name__ == '__main__':if "cpu" not in device_str:torch.cuda.empty_cache()- if model_name.split('/')[-1] == ".":+ if model_name.split('/')[-1].strip(".") == "":export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")else:export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")eval_folder = None- if args.format:- format_list = args.format.replace(' ', '').split(',')- inplace = False if len(format_list) > 1 else True- for format_ in format_list:- save_format_ = format_.replace(":", "-")- save_format_ = save_format_.replace("_", "-")- eval_folder = f'{export_dir}-{save_format_}'- autoround.save_quantized(eval_folder, format=format_, inplace=inplace)- else:- deployment_device = args.deployment_device.split(',')- gpu_formats = []- for item in deployment_device:- if item in ["gpu", "auto_gptq", "auto_round", "auto_awq"]:- if item == "gpu":- if lm_head_layer_name in layer_config.keys() and layer_config[lm_head_layer_name][- "data_type"] == "int":- gpu_formats.append("auto_round")- else:- gpu_formats.append("auto_gptq")- else:- gpu_formats.append(item)-- gpu_formats = list(set(gpu_formats))-- inplace = True if len(deployment_device) < 2 else False- for gpu_format in gpu_formats:- eval_folder = f'{export_dir}-{gpu_format}'- autoround.save_quantized(eval_folder, format=gpu_format, inplace=inplace)-- if 'xpu' in deployment_device:- autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace)- if "cpu" in deployment_device:- autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace)- if "fake" in deployment_device:- output_dir = f'{export_dir}-qdq'- model = model.to("cpu")- model.save_pretrained(output_dir)- tokenizer.save_pretrained(output_dir)- if eval_folder is None:- eval_folder = output_dir-- if (not ('gpu' in deployment_device or len(gpu_formats) > 0)) and 'fake' not in deployment_device:- print('does not support cpu, xpu model evaluation.')- exit() ## does not support cpu,xpu model eval++ format_list = args.format.replace(' ', '').split(',')+ inplace = False if len(format_list) > 1 else True+ for format_ in format_list:+ save_format_ = format_.replace(":", "-")+ save_format_ = save_format_.replace("_", "-")+ eval_folder = f'{export_dir}-{save_format_}'+ autoround.save_quantized(eval_folder, format=format_, inplace=inplace)if args.disable_eval:exit()@@ -437,8 +379,9 @@ if __name__ == '__main__':user_model = model.to(device_str)if args.eval_bs == "auto":args.eval_bs = 16- from auto_round.eval.evaluation import simple_evaluate_user_model- res = simple_evaluate_user_model(user_model, tokenizer,tasks=tasks,batch_size=args.eval_bs)+ from auto_round.eval.evaluation import simple_evaluate_user_model++ res = simple_evaluate_user_model(user_model, tokenizer, tasks=tasks, batch_size=args.eval_bs)else:res = simple_evaluate(model="hf", model_args=model_args,tasks=tasks,@@ -446,4 +389,3 @@ if __name__ == '__main__':from lm_eval.utils import make_tableprint(make_table(res))-diff --git a/requirements.txt b/requirements.txtindex d010ea536..698d8c34b 100644--- a/requirements.txt+++ b/requirements.txt@@ -3,7 +3,7 @@ datasetspy-cpuinfosentencepiecetorch-transformers+transformers>=4.38tritonnumpy < 2.0threadpoolctl