Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix export format issue (#120)

wenhuach21ed29cf503177
diff --git a/docs/Meta-Llama-3-8B-Instruct-acc.md b/docs/Meta-Llama-3-8B-Instruct-acc.md
index 50322c70a..24db99924 100644
--- a/docs/Meta-Llama-3-8B-Instruct-acc.md
+++ b/docs/Meta-Llama-3-8B-Instruct-acc.md
@@ -5,20 +5,24 @@ For evaluating w4g128 without quantized lm-head,
lm_eval --model hf --model_args pretrained="./",autogptq=True,gptq_use_triton=True --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 16
```
-To assess the performance of w4g128 with a quantized lm-head, we chose to evaluate the qdq model instead. This decision was made due to the lack of support for lm-head quantization in AutoGPTQ, which results in random initialization of the lm-head.
-
+for evaluation with quantized lm-head
+```bash
+git clone https://github.com/intel/auto-round
+cd auto-round/examples/language-modeling
+python3 eval_042/evluation.py --model_name "./" --eval_bs 16
+```
-| Metric | **BF16** | w4g128 w/o lm-head | w4g128 with lm-head qdq |
-| ---------------- | :------- |--------------------|---------------------------------|
-| Avg. | 0.6352 | 0.6312 | 0.6303 |
-| mmlu | 0.6386 | 0.6306 | 0.6318 |
-| winogrande | 0.7143 | 0.7238 | 0.7269 |
-| truthfulqa_mc1 | 0.3623 | 0.3537 | 0.3525 |
-| rte | 0.6751 | 0.6859 | 0.6679 |
-| piqa | 0.7867 | 0.7797 | 0.7802 |
-| openbookqa | 0.3400 | 0.3300 | 0.3320 |
-| lambada_openai | 0.7182 | 0.7200 | 0.7173 |
-| hellaswag | 0.5769 | 0.5699 | 0.5701 |
-| boolq | 0.8297 | 0.8309 | 0.8284 |
-| arc_easy | 0.8152 | 0.8089 | 0.8106 |
-| arc_challenge | 0.5299 | 0.5102 | 0.5154 |
+| Metric | **BF16** | w4g128 w/o lm-head | w4g128 with lm-head |
+| ---------------- | :------- |--------------------|-----------------------------|
+| Avg. | 0.6352 | 0.6312 | 0.6303 |
+| mmlu | 0.6386 | 0.6306 | 0.6243 |
+| winogrande | 0.7143 | 0.7238 | 0.7261 |
+| truthfulqa_mc1 | 0.3623 | 0.3537 | 0.3574 |
+| rte | 0.6751 | 0.6859 | 0.6715 |
+| piqa | 0.7867 | 0.7797 | 0.7775 |
+| openbookqa | 0.3400 | 0.3300 | 0.3340 |
+| lambada_openai | 0.7182 | 0.7200 | 0.7118 |
+| hellaswag | 0.5769 | 0.5699 | 0.5686 |
+| boolq | 0.8297 | 0.8309 | 0.8266 |
+| arc_easy | 0.8152 | 0.8089 | 0.8123 |
+| arc_challenge | 0.5299 | 0.5102 | 0.5111 |
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 546d61cfb..df856236c 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -286,6 +286,12 @@ if __name__ == '__main__':
if args.quant_lm_head and not args.disable_low_gpu_mem_usage:
print(f"warning, disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to "
f"gpu")
+ deployment_device = args.deployment_device.split(',')
+ gpu_format = "auto_gptq"
+ if 'gpu' in deployment_device:
+ if lm_head_layer_name in weight_config.keys():
+ gpu_format = "autoround"
+
autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,
dataset=args.dataset, seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,
minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,
@@ -303,13 +309,10 @@ if __name__ == '__main__':
export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"
- deployment_device = args.deployment_device.split(',')
+
inplace = True if len(deployment_device) < 2 else False
- if 'gpu' in deployment_device :
- if lm_head_layer_name in weight_config.keys():
- autoround.save_quantized(f'{export_dir}-gpu', format="autoround", use_triton=True, inplace=inplace)
- else:
- autoround.save_quantized(f'{export_dir}-gpu', format="auto_gptq", use_triton=True, inplace=inplace)
+ if 'gpu' in deployment_device:
+ autoround.save_quantized(f'{export_dir}-gpu', format=gpu_format, use_triton=True, inplace=inplace)
if 'xpu' in deployment_device:
autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,
compression_dtype=torch.int8, compression_dim=0, use_optimum_format=False,