PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix export format issue (#120)
wenhuach21ed29cf503177
diff --git a/docs/Meta-Llama-3-8B-Instruct-acc.md b/docs/Meta-Llama-3-8B-Instruct-acc.mdindex 50322c70a..24db99924 100644--- a/docs/Meta-Llama-3-8B-Instruct-acc.md+++ b/docs/Meta-Llama-3-8B-Instruct-acc.md@@ -5,20 +5,24 @@ For evaluating w4g128 without quantized lm-head,lm_eval --model hf --model_args pretrained="./",autogptq=True,gptq_use_triton=True --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 16```-To assess the performance of w4g128 with a quantized lm-head, we chose to evaluate the qdq model instead. This decision was made due to the lack of support for lm-head quantization in AutoGPTQ, which results in random initialization of the lm-head.-+for evaluation with quantized lm-head+```bash+git clone https://github.com/intel/auto-round+cd auto-round/examples/language-modeling+python3 eval_042/evluation.py --model_name "./" --eval_bs 16+```-| Metric | **BF16** | w4g128 w/o lm-head | w4g128 with lm-head qdq |-| ---------------- | :------- |--------------------|---------------------------------|-| Avg. | 0.6352 | 0.6312 | 0.6303 |-| mmlu | 0.6386 | 0.6306 | 0.6318 |-| winogrande | 0.7143 | 0.7238 | 0.7269 |-| truthfulqa_mc1 | 0.3623 | 0.3537 | 0.3525 |-| rte | 0.6751 | 0.6859 | 0.6679 |-| piqa | 0.7867 | 0.7797 | 0.7802 |-| openbookqa | 0.3400 | 0.3300 | 0.3320 |-| lambada_openai | 0.7182 | 0.7200 | 0.7173 |-| hellaswag | 0.5769 | 0.5699 | 0.5701 |-| boolq | 0.8297 | 0.8309 | 0.8284 |-| arc_easy | 0.8152 | 0.8089 | 0.8106 |-| arc_challenge | 0.5299 | 0.5102 | 0.5154 |+| Metric | **BF16** | w4g128 w/o lm-head | w4g128 with lm-head |+| ---------------- | :------- |--------------------|-----------------------------|+| Avg. | 0.6352 | 0.6312 | 0.6303 |+| mmlu | 0.6386 | 0.6306 | 0.6243 |+| winogrande | 0.7143 | 0.7238 | 0.7261 |+| truthfulqa_mc1 | 0.3623 | 0.3537 | 0.3574 |+| rte | 0.6751 | 0.6859 | 0.6715 |+| piqa | 0.7867 | 0.7797 | 0.7775 |+| openbookqa | 0.3400 | 0.3300 | 0.3340 |+| lambada_openai | 0.7182 | 0.7200 | 0.7118 |+| hellaswag | 0.5769 | 0.5699 | 0.5686 |+| boolq | 0.8297 | 0.8309 | 0.8266 |+| arc_easy | 0.8152 | 0.8089 | 0.8123 |+| arc_challenge | 0.5299 | 0.5102 | 0.5111 |diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 546d61cfb..df856236c 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -286,6 +286,12 @@ if __name__ == '__main__':if args.quant_lm_head and not args.disable_low_gpu_mem_usage:print(f"warning, disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to "f"gpu")+ deployment_device = args.deployment_device.split(',')+ gpu_format = "auto_gptq"+ if 'gpu' in deployment_device:+ if lm_head_layer_name in weight_config.keys():+ gpu_format = "autoround"+autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,dataset=args.dataset, seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,@@ -303,13 +309,10 @@ if __name__ == '__main__':export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"- deployment_device = args.deployment_device.split(',')+inplace = True if len(deployment_device) < 2 else False- if 'gpu' in deployment_device :- if lm_head_layer_name in weight_config.keys():- autoround.save_quantized(f'{export_dir}-gpu', format="autoround", use_triton=True, inplace=inplace)- else:- autoround.save_quantized(f'{export_dir}-gpu', format="auto_gptq", use_triton=True, inplace=inplace)+ if 'gpu' in deployment_device:+ autoround.save_quantized(f'{export_dir}-gpu', format=gpu_format, use_triton=True, inplace=inplace)if 'xpu' in deployment_device:autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,compression_dtype=torch.int8, compression_dim=0, use_optimum_format=False,