PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix cpu device issue (#8)
WeiweiZhang1d3bca7778441
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 3db7d6f55..6653889ff 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -574,6 +574,9 @@ class AutoRound(object):if self.amp:self.model = self.model.to(self.amp_dtype)logger.info(f"using {self.amp_dtype}")+ elif self.device == "cpu" and self.model.dtype == torch.float16:+ self.model = self.model.to(torch.float32)+ logger.info(f"using {torch.float32} for cpu quantization")self.dataset_name = dataset_nameif dataloader is None:@@ -1310,3 +1313,4 @@ class AutoAdamRound(AutoOPTRound):optimizer,**kwargs,)+diff --git a/examples/eval/evaluation.py b/examples/eval/evaluation.pyindex 19430def7..a43202120 100644--- a/examples/eval/evaluation.py+++ b/examples/eval/evaluation.py@@ -264,7 +264,7 @@ def eval_model(output_dir=None, model=None, tokenizer=None,tmp_eval_bs = eval_bstmp_results, lm = simple_evaluate(model=model_type, model_args=model_args, tasks=task_names,num_fewshot=shot, limit=limit, batch_size=tmp_eval_bs,- max_batch_size=tmp_eval_bs, lm=lm)+ max_batch_size=tmp_eval_bs, lm=lm, device=str(device))sub_name = f'{tmp_tasks} {shot}-shot'print(f'{sub_name}: ')pprint.pprint(tmp_results["results"])diff --git a/examples/main.py b/examples/main.pyindex dc3b5a065..3ef71b559 100644--- a/examples/main.py+++ b/examples/main.py@@ -117,9 +117,13 @@ if __name__ == '__main__':device_str = "cpu"else:device_str = f"cuda:{int(args.device)}"- cuda_device = torch.device(device_str)+ torch_device = torch.device(device_str)is_glm = bool(re.search("chatglm", model_name.lower()))- if is_glm:+ is_llava = bool(re.search("llava", model_name.lower()))+ if is_llava:+ from transformers import LlavaForConditionalGeneration+ model = LlavaForConditionalGeneration.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype="auto")+ elif is_glm:model = AutoModel.from_pretrained(model_name, trust_remote_code=True)else:model = AutoModelForCausalLM.from_pretrained(@@ -153,15 +157,15 @@ if __name__ == '__main__':excel_name = f"{model_name}_{args.bits}_{args.group_size}"if args.eval_fp16_baseline:if not args.low_gpu_mem_usage:- model = model.to(cuda_device)+ model = model.to(torch_device)excel_name += "_fp16.xlsx"eval_model(output_dir=model_name, model=model, tokenizer=tokenizer, tasks=args.tasks, \- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device,+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=torch_device,eval_orig_float=True, excel_file=excel_name)exit()if not args.low_gpu_mem_usage:- model = model.to(cuda_device)+ model = model.to(torch_device)scheme = "asym"if args.sym:@@ -172,25 +176,26 @@ if __name__ == '__main__':autoround = round(model, tokenizer, args.bits, args.group_size, scheme, bs=args.train_bs,seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,- minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input,+ minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input, device=device_str,amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps, scale_dtype=args.scale_dtype) ##TODO args passmodel, q_config = autoround.quantize()-- export_dir = args.output_dir + "/compressed_" + args.model_name.split('/')[-1] + "/"+ model_name = args.model_name.rstrip("/")+ export_dir = args.output_dir + "/compressed_" + model_name.split('/')[-1] + "/"if args.deployment_device == 'cpu':- autoround.export_to_speed(output_dir=export_dir)+ autoround.export(output_dir=export_dir)del q_configelif args.deployment_device == 'gpu':- autoround.export_to_autogptq(export_dir, use_triton=True)-- torch.cuda.empty_cache()+ autoround.export(export_dir, target="auto_gptq", use_triton=True)+ if args.device != "cpu":+ torch.cuda.empty_cache()model.eval()- output_dir = args.output_dir + "_" + args.model_name.split('/')[-1] + f"_w{args.bits}_g{args.group_size}"+ output_dir = args.output_dir + "_" + model_name.split('/')[-1] + f"_w{args.bits}_g{args.group_size}"excel_name = f"{output_dir}_result.xlsx"output_dir += "/"print(excel_name, flush=True)eval_model(output_dir=output_dir, model=model, tokenizer=tokenizer, tasks=args.tasks, \- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device, excel_file=excel_name,+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=torch_device, excel_file=excel_name,limit=None)+