PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix low_gpu_mem_usage regression (#43)
wenhuach21101404878bd1
diff --git a/README.md b/README.mdindex 2e6c084ac..831ac7eaf 100644--- a/README.md+++ b/README.md@@ -38,19 +38,16 @@ pip install auto-round### Gaudi2/ CPU/ GPU```python-import torchfrom transformers import AutoModelForCausalLM, AutoTokenizer-device = "hpu" ## or "cpu", "cuda:0"-dtype = "auto" if device != "hpu" else torch.bfloat16model_name = "meta-llama/Llama-2-7b-hf"-model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=dtype, trust_remote_code=True)+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)from auto_round import AutoRoundbits, group_size, sym = 4, 128, False-autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, device=tuning_device)+autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym)autoround.quantize()output_dir = "./tmp_autoround"autoround.save_quantized(output_dir)diff --git a/examples/code-generation/main.py b/examples/code-generation/main.pyindex a50131fc6..46b7c9a6a 100644--- a/examples/code-generation/main.py+++ b/examples/code-generation/main.py@@ -169,7 +169,7 @@ if __name__ == '__main__':excel_name = f"{model_name}_{args.bits}_{args.group_size}"- if not args.low_gpu_mem_usage:+ if args.disable_low_gpu_mem_usage:model = model.to(torch_device)round = AutoRounddiff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 34d10306b..c6863a8e8 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -57,8 +57,6 @@ if __name__ == '__main__':parser.add_argument("--seed", default=42, type=int,help="seed")--parser.add_argument("--eval_fp16_baseline", action='store_true',help="whether to eval FP16 baseline")@@ -241,7 +239,7 @@ if __name__ == '__main__':device=torch_device, excel_file=excel_name)exit()- if not args.low_gpu_mem_usage:+ if args.disable_low_gpu_mem_usage:model = model.to(torch_device)round = AutoRound@@ -288,5 +286,5 @@ if __name__ == '__main__':output_dir += "/"print(excel_name, flush=True)eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,+ eval_bs=args.eval_bs, use_accelerate=not args.disable_low_gpu_mem_usage,device=torch_device, excel_file=excel_name)