PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
bugfix (#160)
WeiweiZhang131c566ccbb27
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 878eda467..c5df1ad1a 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -891,7 +891,7 @@ class AutoRound(object):output_q = block_forward(block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device)- if self.amp and not check_is_cpu(device):+ if self.amp:with autocast(device_type=device.split(":")[0], dtype=self.amp_dtype):loss = mse_loss(output_q, current_output) # pylint: disable=not-callableelse:@@ -1416,3 +1416,4 @@ class AutoAdamRound(AutoOPTRound):optimizer,**kwargs,)+diff --git a/examples/language-modeling/eval_042/evaluation.py b/examples/language-modeling/eval_042/evaluation.pyindex ea19c8fae..b14a67c82 100644--- a/examples/language-modeling/eval_042/evaluation.py+++ b/examples/language-modeling/eval_042/evaluation.py@@ -573,6 +573,10 @@ if __name__ == "__main__":parser.add_argument("--eval_bs", default=1,)+ parser.add_argument(+ "--trust_remote_code", action='store_true',+ help="Whether to enable trust_remote_code"+ )parser.add_argument("--tasks",default="lambada_openai,hellaswag,winogrande,piqa,mmlu,truthfulqa_mc1," \"openbookqa,boolq,rte,arc_easy,arc_challenge",@@ -582,7 +586,7 @@ if __name__ == "__main__":s = time.time()from transformers import AutoConfig- config = AutoConfig.from_pretrained(args.model_name)+ config = AutoConfig.from_pretrained(args.model_name, trust_remote_code=args.trust_remote_code)if hasattr(config, "quantization_config"):quantization_config = config.quantization_config@@ -599,6 +603,8 @@ if __name__ == "__main__":if config.torch_dtype == torch.float32:model_args += ",dtype=float16"# model_args += ",dtype=float16"+ if args.trust_remote_code:+ model_args += f",trust_remote_code=True"result = simple_evaluate(model="hf",model_args=model_args,tasks=test_tasks,@@ -606,3 +612,4 @@ if __name__ == "__main__":print(make_table(result))print("cost time: ", time.time() - s)+diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex d5171fe55..3569ce671 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -231,16 +231,12 @@ if __name__ == '__main__':args.seqlen = seqlenexcel_name = f"{model_name}_{args.bits}_{args.group_size}"- pt_dtype = torch.float16if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):dtype = 'bfloat16'- pt_dtype = torch.bfloat16else:- if str(args.device) != "cpu":- pt_dtype = torch.float16+ if "cpu" not in device_str:dtype = 'float16'else:- pt_dtype = torch.float32dtype = 'float32'excel_name = f"{model_name}_{args.bits}_{args.group_size}"@@ -313,7 +309,7 @@ if __name__ == '__main__':model_name = args.model_name.rstrip("/")model.eval()- if args.device != "cpu":+ if "cpu" not in device_str:torch.cuda.empty_cache()export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"@@ -357,3 +353,4 @@ if __name__ == '__main__':from lm_eval.utils import make_tableprint(make_table(res))+