PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Enable export without dir (#23)
WeiweiZhang1b65830f3f6cb
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 5c5f53cd3..cf044c520 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -863,11 +863,11 @@ class AutoRound(object):)if self.amp and not check_is_cpu(device):with autocast(device_type="cuda", dtype=self.amp_dtype):- # pylint: disable=not-callable- loss = mse_loss(output_q, current_output)+ loss = mse_loss(output_q, current_output) # pylint: disable=not-callableelse:- # pylint: disable=not-callable- loss = mse_loss(output_q.to(torch.float32), current_output.to(torch.float32))+ loss = mse_loss( # pylint: disable=not-callable+ output_q.to(torch.float32), current_output.to(torch.float32)+ )total_loss += loss.item() / self.gradient_accumulate_stepsself.scale_loss_and_backward(scaler, loss)@@ -956,15 +956,23 @@ class AutoRound(object):torch.cuda.empty_cache()- def save_quantized(self, output_dir, format="itrex", **kwargs):- compress_model = None+ def save_quantized(self, output_dir=None, format="itrex", **kwargs):+ compressed_model = Noneif format == "itrex":- compress_model = self.save_quantized_as_itrex(output_dir, **kwargs)+ from auto_round.export.export_to_itrex import compress_model++ inplace = kwargs["inplace"] if "inplace" in kwargs.keys() else True+ compressed_model = compress_model(self.model, self.weight_config, inplace=inplace)+ if "use_triton" in kwargs.keys():+ kwargs.pop("use_triton")+ if output_dir is not None:+ self.save_quantized_as_itrex(output_dir, compressed_model, **kwargs)+elif format == "auto_gptq":self.save_quantized_as_autogptq(output_dir, **kwargs)else:logger.error("export only supports itrex and auto_gptq now")- return compress_model+ return compressed_modeldef save_quantized_as_autogptq(self, output_dir, use_triton=False, inplace=True):"""Export the model to autogptq format to easily leverage cuda kernel."""@@ -1058,12 +1066,11 @@ class AutoRound(object):modules_in_block_to_quantize=modules_in_block_to_quantize,)- def save_quantized_as_itrex(self, output_dir, inplace=True):+ def save_quantized_as_itrex(self, output_dir, compressed_model, **kwargs):"""Save configure file and weights for CPU backend inference."""- from auto_round.export.export_to_itrex import QuantConfig, compress_model-- compressed_model = compress_model(self.model, self.weight_config, inplace=inplace)sym = self.scheme == "sym"+ from auto_round.export.export_to_itrex import QuantConfig+quantize_config = QuantConfig(bits=self.bits,group_size=self.group_size,@@ -1080,7 +1087,6 @@ class AutoRound(object):setattr(config, "quantization_config", quantize_config.to_dict())config.save_pretrained(output_dir)quantize_config.save_pretrained(output_dir)-try:compressed_model.save_pretrained(output_dir, safe_serialization=True)if self.tokenizer is not None:@@ -1088,7 +1094,6 @@ class AutoRound(object):logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))except IOError as e: # pragma: no coverlogger.error("Fail to save configure file and weights due to {}.".format(e))- return compressed_modeldef quantize(self):"""Quantize the model and return the quantized model along with weight configurations.diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 141fe9315..b4b224b12 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -41,7 +41,7 @@ if __name__ == '__main__':parser.add_argument("--device", default=0, type=str,help="device gpu int number, or 'cpu' ")-+parser.add_argument("--sym", action='store_true',help=" sym quantization")@@ -84,14 +84,14 @@ if __name__ == '__main__':parser.add_argument("--enable_minmax_tuning", action='store_true',help="whether enable weight minmax tuning")-+parser.add_argument("--deployment_device", default='fake', type=str,help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu'.""default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")-+parser.add_argument("--scale_dtype", default='fp32',help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")-+parser.add_argument("--tasks",default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',"mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte",@@ -107,6 +107,7 @@ if __name__ == '__main__':parser.add_argument("--disable_lmeval", action='store_true',help="Whether to do lmeval evaluation.")+args = parser.parse_args()set_seed(args.seed)