PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
disable quantizing lm-head with tied weights as a workaround (#101)
wenhuach21c839e825ddc4
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 3859c65da..8300fb958 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -94,7 +94,7 @@ if __name__ == '__main__':parser.add_argument("--tasks",default="lambada_openai,hellaswag,winogrande,piqa,mmlu,wikitext,truthfulqa_mc1," \- "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge,wikitext2,ptb-new,c4-new",+ "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge,wikitext2,ptb-new,c4-new",help="lm-eval tasks for lm_eval version 0.4")parser.add_argument("--output_dir", default="./tmp_autoround", type=str,@@ -114,7 +114,7 @@ if __name__ == '__main__':parser.add_argument("--disable_trust_remote_code", action='store_true',help="Whether to disable trust_remote_code")-+parser.add_argument("--disable_quanted_input", action='store_true',help="whether to disuse the output of quantized block to tune the next block")@@ -268,10 +268,23 @@ if __name__ == '__main__':print(f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")if args.quant_lm_head:+ from transformers import AutoConfig++ config = AutoConfig.from_pretrained(model_name)weight_config['lm_head'] = {"data_type": "int"}- if not args.disable_low_gpu_mem_usage:- print(f"warning, disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to "- f"gpu")+ if config.tie_word_embeddings:+ if hasattr(model, "_tied_weights_keys"):+ tied_keys = model._tied_weights_keys+ for item in tied_keys:+ if "lm_head" in item:##TODO extend to encoder-decoder layer, seq classification model+ args.quant_lm_head = False+ print(+ f"warning, disable quant_lm_head as quantizing lm_head with tied weights has not been "+ f"supported currently")+ break+ if args.quant_lm_head and not args.disable_low_gpu_mem_usage:+ print(f"warning, disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to "+ f"gpu")autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,dataset=args.dataset, seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,@@ -311,4 +324,3 @@ if __name__ == '__main__':eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,eval_bs=args.eval_bs, use_accelerate=not args.disable_low_gpu_mem_usage,device=torch_device, excel_file=excel_name)-