Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix low_gpu_mem_usage regression (#43)

wenhuach21101404878bd1
diff --git a/README.md b/README.md
index 2e6c084ac..831ac7eaf 100644
--- a/README.md
+++ b/README.md
@@ -38,19 +38,16 @@ pip install auto-round
### Gaudi2/ CPU/ GPU
```python
-import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
-device = "hpu" ## or "cpu", "cuda:0"
-dtype = "auto" if device != "hpu" else torch.bfloat16
model_name = "meta-llama/Llama-2-7b-hf"
-model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=dtype, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
from auto_round import AutoRound
bits, group_size, sym = 4, 128, False
-autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym, device=tuning_device)
+autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym)
autoround.quantize()
output_dir = "./tmp_autoround"
autoround.save_quantized(output_dir)
diff --git a/examples/code-generation/main.py b/examples/code-generation/main.py
index a50131fc6..46b7c9a6a 100644
--- a/examples/code-generation/main.py
+++ b/examples/code-generation/main.py
@@ -169,7 +169,7 @@ if __name__ == '__main__':
excel_name = f"{model_name}_{args.bits}_{args.group_size}"
- if not args.low_gpu_mem_usage:
+ if args.disable_low_gpu_mem_usage:
model = model.to(torch_device)
round = AutoRound
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 34d10306b..c6863a8e8 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -57,8 +57,6 @@ if __name__ == '__main__':
parser.add_argument("--seed", default=42, type=int,
help="seed")
-
-
parser.add_argument("--eval_fp16_baseline", action='store_true',
help="whether to eval FP16 baseline")
@@ -241,7 +239,7 @@ if __name__ == '__main__':
device=torch_device, excel_file=excel_name)
exit()
- if not args.low_gpu_mem_usage:
+ if args.disable_low_gpu_mem_usage:
model = model.to(torch_device)
round = AutoRound
@@ -288,5 +286,5 @@ if __name__ == '__main__':
output_dir += "/"
print(excel_name, flush=True)
eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,
- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,
+ eval_bs=args.eval_bs, use_accelerate=not args.disable_low_gpu_mem_usage,
device=torch_device, excel_file=excel_name)