PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
better align gradient_accumulate_steps for varied length input (#309)
wenhuach21103e7e6c9ea3
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 839448adb..ec093ff2e 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -14,7 +14,7 @@import sysdef run():- from .script.llm import setup_parser, tune, eval+ from auto_round.script.llm import setup_parser, tune, evalargs = setup_parser()if args.eval:eval(args)@@ -22,18 +22,18 @@ def run():tune(args)def run_best():- from .script.llm import setup_best_parser, tune+ from auto_round.script.llm import setup_best_parser, tuneargs = setup_best_parser()tune(args)def run_fast():- from .script.llm import setup_fast_parser, tune+ from auto_round.script.llm import setup_fast_parser, tuneargs = setup_fast_parser()tune(args)def run_mllm():- from .script.mllm import setup_parser, tune, eval+ from auto_round.script.mllm import setup_parser, tune, evalargs = setup_parser()if args.eval:eval(args)diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 564ba6fa3..6d5e6f973 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -875,10 +875,22 @@ class AutoRound(object):if self.sampler != "rand":whole_indices = torch.randperm(nsamples)[:pick_samples]total_loss = 0+ num_elm = 1+ mse_reduction = "mean"+ if gradient_accumulate_steps != 1:+ mse_reduction = "sum"+ mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)+for i in range(self.iters):total_loss = 0if self.sampler == "rand":whole_indices = torch.randperm(nsamples)[:pick_samples]+ if gradient_accumulate_steps != 1:+ if q_inputs is not None:+ current_input = [q_inputs[i] for i in whole_indices]+ else:+ current_input = [inputs[i] for i in whole_indices]+ num_elm = sum(id.numel() for id in current_input)for tmp_step in range(gradient_accumulate_steps):indices = whole_indices[tmp_step * batch_size: (tmp_step + 1) * batch_size]if q_inputs is not None:@@ -902,7 +914,7 @@ class AutoRound(object):loss = mse_loss( # pylint: disable=not-callableoutput_q.to(torch.float32), current_output.to(torch.float32))- total_loss += loss.item() / gradient_accumulate_steps+ total_loss += loss.item() / num_elmself.scale_loss_and_backward(scaler, loss)if i == 0:@@ -996,15 +1008,24 @@ class AutoRound(object):whole_indices = torch.randperm(nsamples)[:pick_samples]last_best_iter = 0best_loss = torch.finfo(torch.float).max- mse_loss = torch.nn.MSELoss().to(device)+ num_elm = 1+ mse_reduction = "mean"+ if self.gradient_accumulate_steps != 1:+ mse_reduction = "sum"+ mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)scaler = self.get_scaler() # pylint: disable=assignment-from-noneinit_loss = Nonebest_params = {}total_loss = 0+for i in range(self.iters):total_loss = 0if self.sampler == "rand":whole_indices = torch.randperm(nsamples)[:pick_samples]+ ##we assume the block input and output shape is same+ if self.gradient_accumulate_steps!=1:+ current_input_ids = [input_ids[i] for i in whole_indices]+ num_elm = sum(id.numel() for id in current_input_ids)for tmp_step in range(self.gradient_accumulate_steps):indices = whole_indices[tmp_step * self.batch_size: (tmp_step + 1) * self.batch_size]current_input_ids, current_input_others = sampling_inputs(@@ -1031,7 +1052,7 @@ class AutoRound(object):output_q.to(torch.float32), current_output.to(torch.float32))- total_loss += loss.item() / self.gradient_accumulate_steps+ total_loss += loss.item() / num_elmself.scale_loss_and_backward(scaler, loss)if i == 0:init_loss = total_lossdiff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 181fd53ef..7bf5163a8 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -91,10 +91,7 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--low_gpu_mem_usage", action='store_true',help="offload intermediate features to cpu")- self.add_argument("--format", default=None, type=str,- choices=["auto_round", "auto_gptq", "auto_awq", "auto_round:gptq", "auto_round:auto_gptq",- "auto_round:auto_gptq:marlin", "auto_round:gptq:marlin", "auto_round:auto_awq",- "auto_round:awq", "auto_awq", "itrex", "iterx_xpu", "fake"],+ self.add_argument("--format", default="auto_round", type=str,help="the format to save the model")@@ -192,7 +189,7 @@ def setup_best_parser():parser.add_argument("--batch_size", "--train_bs", default=8, type=int,help="train batch size")- parser.add_argument("--iters","--iter", default=1000, type=int,+ parser.add_argument("--iters", "--iter", default=1000, type=int,help="iterations to tune each block")parser.add_argument("--seqlen", "--seq_len", default=2048, type=int,@@ -234,6 +231,14 @@ def tune(args):tasks = args.tasksif args.format is None:args.format = "auto_round"+ supported_formats = ["auto_round", "auto_gptq", "auto_awq", "auto_round:gptq", "auto_round:auto_gptq",+ "auto_round:auto_gptq:marlin", "auto_round:gptq:marlin", "auto_round:auto_awq",+ "auto_round:awq", "auto_awq", "itrex", "iterx_xpu", "fake"]+ formats = args.format.replace(' ', '').split(",")+ for format in formats:+ if format not in supported_formats:+ raise ValueError(f"{format} is not supported, we only support {supported_formats}")+if "auto_gptq" in args.format and args.asym is True:print("warning: The auto_gptq kernel has issues with asymmetric quantization. "@@ -359,10 +364,12 @@ def tune(args):breakif args.quant_lm_head:layer_config[lm_head_layer_name] = {"bits": args.bits}- transformers_version = [int(item) for item in transformers.__version__.split('.')[:2]]- if transformers_version[0] == 4 and transformers_version[1] < 38:- error_message = "Please upgrade transformers>=4.38.0 to support lm-head quantization."- raise EnvironmentError(error_message)+ for format in formats:+ if "auto_round" not in format:+ auto_round_formats = [s for s in supported_formats if s.startswith("auto_round")]+ raise ValueError(+ f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}")+autoround = round(model, tokenizer, args.bits, args.group_size, sym=not args.asym, batch_size=args.batch_size,diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex aba755584..e62d72e28 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -75,8 +75,6 @@ class BasicArgumentParser(argparse.ArgumentParser):help="offload intermediate features to cpu")self.add_argument("--format", default="auto_round", type=str,- choices=["auto_round", "auto_round:gptq", "auto_round:auto_gptq", "auto_round:awq",- "auto_round:auto_awq", "fake"],help="the format to save the model")@@ -134,7 +132,6 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--not_use_best_mse", action='store_true',help="whether to use the iter of best mes loss in the tuning phase")-## ======================= VLM =======================self.add_argument("--quant_nontext_module", action='store_true',help="whether to quantize non-text module, e.g. vision component")@@ -205,6 +202,13 @@ def setup_parser():def tune(args):if args.format is None:args.format = "auto_round"+ supported_formats = ["auto_round", "auto_round:gptq", "auto_round:auto_gptq",+ "auto_round:auto_gptq:marlin", "auto_round:gptq:marlin", "auto_round:auto_awq",+ "auto_round:awq"]+ formats = args.format.replace(' ', '').split(",")+ for format in formats:+ if format not in supported_formats:+ raise ValueError(f"{format} is not supported, we only support {supported_formats}")model_name = args.modelif model_name[-1] == "/":@@ -268,10 +272,12 @@ def tune(args):breakif args.quant_lm_head:layer_config[lm_head_layer_name] = {"bits": args.bits}- transformers_version = [int(item) for item in transformers.__version__.split('.')[:2]]- if transformers_version[0] == 4 and transformers_version[1] < 38:- error_message = "Please upgrade transformers>=4.38.0 to support lm-head quantization."- raise EnvironmentError(error_message)+ for format in formats:+ if "auto_round" not in format:+ auto_round_formats = [s for s in supported_formats if s.startswith("auto_round")]+ raise ValueError(+ f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}")+if args.quant_lm_head and args.low_gpu_mem_usage:print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "diff --git a/auto_round/utils.py b/auto_round/utils.pyindex d12d2164e..111493d44 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -362,7 +362,9 @@ def sampling_inputs(input_ids, input_others, indices, seqlen,current_input_others: The sampled other input data."""current_input_ids = [input_ids[i] for i in indices]+current_input_ids = torch.cat(current_input_ids, dim=batch_dim)+current_input_others = {"positional_inputs": input_others["positional_inputs"]}for key in input_others.keys():if "positional_inputs" in key:diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex f55551d33..25b50e101 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -88,17 +88,13 @@ if __name__ == '__main__':help="number of samples")parser.add_argument("--low_gpu_mem_usage", action='store_true',- help="offload intermediate features to cpu")+ help="offload intermediate features to cpu")parser.add_argument("--enable_minmax_tuning", action='store_true',help="enable_minmax_tuning is deprecated")-- parser.add_argument("--format", default=None, type=str,- choices=["auto_round", "auto_gptq", "auto_awq", "auto_round:gptq", "auto_round:auto_gptq",- "auto_round:auto_gptq:marlin", "auto_round:gptq:marlin", "auto_round:auto_awq",- "auto_round:awq", "auto_awq", "itrex", "iterx_xpu", "fake"],- help="The format in which to save the model. "+ parser.add_argument("--format", default="auto_round", type=str,+ help="the format to save the quantized model")parser.add_argument("--data_type", "--dtype", default='int',@@ -170,6 +166,17 @@ if __name__ == '__main__':print("Warning, activation quantization is an experiment feature")+ if args.format is None:+ args.format = "auto_round"+ supported_formats = ["auto_round", "auto_gptq", "auto_awq", "auto_round:gptq", "auto_round:auto_gptq",+ "auto_round:auto_gptq:marlin", "auto_round:gptq:marlin", "auto_round:auto_awq",+ "auto_round:awq", "auto_awq", "itrex", "iterx_xpu", "fake"]+ formats = args.format.replace(' ', '').split(",")+ for format in formats:+ if format not in supported_formats:+ raise ValueError(f"{format} is not supported, we only support {supported_formats}")++tasks = args.tasksuse_eval_legacy = False@@ -300,6 +307,11 @@ if __name__ == '__main__':breakif args.quant_lm_head:layer_config[lm_head_layer_name] = {"bits": args.bits}+ for format in formats:+ if "auto_round" not in format:+ auto_round_formats = [s for s in supported_formats if s.startswith("auto_round")]+ raise ValueError(+ f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}")autoround = round(model, tokenizer, args.bits, args.group_size, sym=not args.asym, batch_size=args.batch_size,dataset=args.dataset, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,@@ -323,7 +335,7 @@ if __name__ == '__main__':torch.cuda.empty_cache()if model_name.split('/')[-1].strip(".") == "":- export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")+ export_dir = os.path.join(args.output_dir, f"w{args.bits}g{args.group_size}")else:export_dir = os.path.join(args.output_dir, model_name.split('/')[-1] + f"-w{args.bits}g{args.group_size}")