Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix cpu device issue (#8)

WeiweiZhang1d3bca7778441
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 3db7d6f55..6653889ff 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -574,6 +574,9 @@ class AutoRound(object):
if self.amp:
self.model = self.model.to(self.amp_dtype)
logger.info(f"using {self.amp_dtype}")
+ elif self.device == "cpu" and self.model.dtype == torch.float16:
+ self.model = self.model.to(torch.float32)
+ logger.info(f"using {torch.float32} for cpu quantization")
self.dataset_name = dataset_name
if dataloader is None:
@@ -1310,3 +1313,4 @@ class AutoAdamRound(AutoOPTRound):
optimizer,
**kwargs,
)
+
diff --git a/examples/eval/evaluation.py b/examples/eval/evaluation.py
index 19430def7..a43202120 100644
--- a/examples/eval/evaluation.py
+++ b/examples/eval/evaluation.py
@@ -264,7 +264,7 @@ def eval_model(output_dir=None, model=None, tokenizer=None,
tmp_eval_bs = eval_bs
tmp_results, lm = simple_evaluate(model=model_type, model_args=model_args, tasks=task_names,
num_fewshot=shot, limit=limit, batch_size=tmp_eval_bs,
- max_batch_size=tmp_eval_bs, lm=lm)
+ max_batch_size=tmp_eval_bs, lm=lm, device=str(device))
sub_name = f'{tmp_tasks} {shot}-shot'
print(f'{sub_name}: ')
pprint.pprint(tmp_results["results"])
diff --git a/examples/main.py b/examples/main.py
index dc3b5a065..3ef71b559 100644
--- a/examples/main.py
+++ b/examples/main.py
@@ -117,9 +117,13 @@ if __name__ == '__main__':
device_str = "cpu"
else:
device_str = f"cuda:{int(args.device)}"
- cuda_device = torch.device(device_str)
+ torch_device = torch.device(device_str)
is_glm = bool(re.search("chatglm", model_name.lower()))
- if is_glm:
+ is_llava = bool(re.search("llava", model_name.lower()))
+ if is_llava:
+ from transformers import LlavaForConditionalGeneration
+ model = LlavaForConditionalGeneration.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype="auto")
+ elif is_glm:
model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
else:
model = AutoModelForCausalLM.from_pretrained(
@@ -153,15 +157,15 @@ if __name__ == '__main__':
excel_name = f"{model_name}_{args.bits}_{args.group_size}"
if args.eval_fp16_baseline:
if not args.low_gpu_mem_usage:
- model = model.to(cuda_device)
+ model = model.to(torch_device)
excel_name += "_fp16.xlsx"
eval_model(output_dir=model_name, model=model, tokenizer=tokenizer, tasks=args.tasks, \
- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device,
+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=torch_device,
eval_orig_float=True, excel_file=excel_name)
exit()
if not args.low_gpu_mem_usage:
- model = model.to(cuda_device)
+ model = model.to(torch_device)
scheme = "asym"
if args.sym:
@@ -172,25 +176,26 @@ if __name__ == '__main__':
autoround = round(model, tokenizer, args.bits, args.group_size, scheme, bs=args.train_bs,
seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,
- minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input,
+ minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input, device=device_str,
amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,
seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps, scale_dtype=args.scale_dtype) ##TODO args pass
model, q_config = autoround.quantize()
-
- export_dir = args.output_dir + "/compressed_" + args.model_name.split('/')[-1] + "/"
+ model_name = args.model_name.rstrip("/")
+ export_dir = args.output_dir + "/compressed_" + model_name.split('/')[-1] + "/"
if args.deployment_device == 'cpu':
- autoround.export_to_speed(output_dir=export_dir)
+ autoround.export(output_dir=export_dir)
del q_config
elif args.deployment_device == 'gpu':
- autoround.export_to_autogptq(export_dir, use_triton=True)
-
- torch.cuda.empty_cache()
+ autoround.export(export_dir, target="auto_gptq", use_triton=True)
+ if args.device != "cpu":
+ torch.cuda.empty_cache()
model.eval()
- output_dir = args.output_dir + "_" + args.model_name.split('/')[-1] + f"_w{args.bits}_g{args.group_size}"
+ output_dir = args.output_dir + "_" + model_name.split('/')[-1] + f"_w{args.bits}_g{args.group_size}"
excel_name = f"{output_dir}_result.xlsx"
output_dir += "/"
print(excel_name, flush=True)
eval_model(output_dir=output_dir, model=model, tokenizer=tokenizer, tasks=args.tasks, \
- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device, excel_file=excel_name,
+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=torch_device, excel_file=excel_name,
limit=None)
+