PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
update gemma recipe (#121)
wenhuach21ecc1dd65ec7a
diff --git a/README.md b/README.mdindex e2f693656..51c6fbe42 100644--- a/README.md+++ b/README.md@@ -51,7 +51,7 @@ pip install auto-round## Model quantization### Gaudi2/ CPU/ GPU-+We found a significant accuracy discrepancy with the qdq model using the AutoGPTQ GPU backend with asymmetric quantization in some scenarios. Please switch to symmetric quantization to alleviate this issue.```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizer@@ -142,7 +142,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizer-+##from auto_round.export import AutoHfQuantizer ## uncomment it for models with quantized lm-headquantized_model_path = "./tmp_autoround"model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, use_fast=True)@@ -163,7 +163,6 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))| google/gemma-2b | [HF-int4-model](https://huggingface.co/Intel/gemma-2b-int4-inc), [accuracy](./docs/gemma-2b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-2b.sh), [example](./examples/language-modeling/)| mistralai/Mistral-7B-Instruct-v0.2 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-Instruct-v0.2-int4-inc) (under review), [accuracy](./docs/Mistral-7B-Instruct-v0.2-acc.md), [recipe](./examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh), [example](./examples/language-modeling/) || google/gemma-7b | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-int4-inc) (under review), [accuracy](./docs/gemma-7b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b.sh), [example](./examples/language-modeling/) |-| google/gemma-7b-it | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-it-int4-inc) (under review), [accuracy](./docs/gemma-7b-it-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b-it.sh), [example](./examples/language-modeling/) || mistralai/Mixtral-8x7B-Instruct-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-Instruct-v0.1-int4-inc) (under review), [accuracy](./docs/Mixtral-8x7B-Instruct-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh), [example](./examples/language-modeling/) || mistralai/Mixtral-8x7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-v0.1-int4-inc) (under review), [accuracy](./docs/Mixtral-8x7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh), [example](./examples/language-modeling/) || meta-llama/Meta-Llama-3-8B-Instruct | [accuracy](./docs/Meta-Llama-3-8B-Instruct-acc.md), [recipe](./examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh), [example](./examples/language-modeling/) |diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 4da651074..52ce1a7ed 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -503,6 +503,7 @@ class AutoRound(object):"gradient_accumulate_steps","iters","amp",+ "n_samples","low_gpu_mem_usage",]if isinstance(dataset, str):diff --git a/auto_round/export/export_to_autoround/autoround_quantizer.py b/auto_round/export/export_to_autoround/autoround_quantizer.pyindex 9b75b9dd4..11f788450 100644--- a/auto_round/export/export_to_autoround/autoround_quantizer.py+++ b/auto_round/export/export_to_autoround/autoround_quantizer.py@@ -140,9 +140,9 @@ class AutoHfQuantizer:@classmethoddef merge_quantization_configs(- cls,- quantization_config: Union[dict, QuantizationConfigMixin],- quantization_config_from_args: Optional[QuantizationConfigMixin],+ cls,+ quantization_config: Union[dict, QuantizationConfigMixin],+ quantization_config_from_args: Optional[QuantizationConfigMixin],):"""Handles situations where both quantization_configfrom args and quantization_config from model config are present."""@@ -156,7 +156,10 @@ class AutoHfQuantizer:warning_msg = ""if isinstance(quantization_config, dict):- quantization_config = AutoRoundConfig.from_dict(quantization_config)+ if "auto-round" in quantization_config["quant_method"]:+ quantization_config = AutoRoundConfig.from_dict(quantization_config)+ else:+ quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101if isinstance(quantization_config, (GPTQConfig, AwqConfig)) and quantization_config_from_args is not None:# special case for GPTQ / AWQ config collision@@ -191,22 +194,22 @@ class AutoRoundConfig(QuantizationConfigMixin):"""def __init__(- self,- bits: int,- tokenizer: Any = None,- dataset: str = None,- group_size: int = 128,- sym: bool = False,- backend="gptq:exllamav2",- iters: int = 200,- weight_config: dict = None,- enable_quanted_input=True,- enable_minmax_tuning=True,- lr=None,- minmax_lr=None,- n_samples=512,- seqlen=2048,- **kwargs,+ self,+ bits: int,+ tokenizer: Any = None,+ dataset: str = None,+ group_size: int = 128,+ sym: bool = False,+ backend="gptq:exllamav2",+ iters: int = 200,+ weight_config: dict = None,+ enable_quanted_input=True,+ enable_minmax_tuning=True,+ lr=None,+ minmax_lr=None,+ n_samples=512,+ seqlen=2048,+ **kwargs,):self.bits = bitsself.tokenizer = tokenizer@@ -358,13 +361,13 @@ class AutoRoundQuantizer(HfQuantizer):in_features = layer.weight.shape[0]out_features = layer.weight.shape[1]bias = layer.bias is not None- new_layer = QuantLinear(+ new_layer = QuantLinear( # pylint: disable=E1123bits,group_size,in_features,out_features,bias,- weight_dtype=layer.weight.dtype, # pylint: disable=E1123+ weight_dtype=layer.weight.dtype,)new_layer.device = devicediff --git a/auto_round/export/export_to_autoround/export_to_autoround.py b/auto_round/export/export_to_autoround/export_to_autoround.pyindex fab55a26c..34d0ac132 100644--- a/auto_round/export/export_to_autoround/export_to_autoround.py+++ b/auto_round/export/export_to_autoround/export_to_autoround.py@@ -140,8 +140,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="gptq:exllamavout_features = layer.weight.shape[1]bias = layer.bias is not None and torch.any(layer.bias)- new_layer = QuantLinear(- bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype ##pylint: disable=E1123+ new_layer = QuantLinear( ##pylint: disable=E1123+ bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype)new_layer.device = devicediff --git a/docs/gemma-2b-acc.md b/docs/gemma-2b-acc.mdindex 033d37dca..7ba60de03 100644--- a/docs/gemma-2b-acc.md+++ b/docs/gemma-2b-acc.md@@ -1,27 +1,29 @@### Evaluate the model-Install [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness.git) from source, and the git id we used is 96d185fa6232a5ab685ba7c43e45d1dbb3bb906d+Install [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness.git) from source, lm-eval 0.4.2 is used++pip install auto-gptq-pip install auto-gptq[triton]-pip install triton==2.2.0Please note that there is a discrepancy between the baseline result and the official data, which is a known issue within the official model card community.+Given that the Gemma model family exhibits inconsistent results between FP16 and BF16 on lm-eval, we recommend converting to FP16 for both tuning and evaluation.```bash-lm_eval --model hf --model_args pretrained="Intel/gemma-2b-int4-inc",autogptq=True,gptq_use_triton=True --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 16+lm_eval --model hf --model_args pretrained="Intel/gemma-2b-int4-inc",autogptq=True,gptq_use_triton=True,dtype=float16 --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 16```-| Metric | BF16 | INT4 |-| -------------- |--------|--------|-| Avg. | 0.5383 | 0.5338 |-| mmlu | 0.3337 | 0.3276 |-| lambada_openai | 0.6398 | 0.6319 |-| hellaswag | 0.5271 | 0.5161 |-| winogrande | 0.6472 | 0.6472 |-| piqa | 0.7699 | 0.7622 |-| truthfulqa_mc1 | 0.2203 | 0.2191 |-| openbookqa | 0.3020 | 0.2980 |-| boolq | 0.6939 | 0.6939 |-| rte | 0.6426 | 0.6498 |-| arc_easy | 0.7424 | 0.7348 |-| arc_challenge | 0.4019 | 0.3908 |+++| Metric | BF16 | FP16 | AutoRound v0.1 | AutoRound v0.2 |+| -------------- | ---- | ------ |----------------|----------------|+| Avg.| 0.5263 | 0.5277 | 0.5235 | 0.5248 |+| mmlu | 0.3287 | 0.3287 | 0.3297 | 0.3309 |+| lambada_openai | 0.6344 | 0.6375 | 0.6307 | 0.6379 |+| hellaswag | 0.5273 | 0.5281 | 0.5159 | 0.5184 |+| winogrande | 0.6504 | 0.6488 | 0.6543 | 0.6575 |+| piqa | 0.7671 | 0.7720 | 0.7612 | 0.7606 |+| truthfulqa_mc1 | 0.2203 | 0.2203 | 0.2203 | 0.2191 |+| openbookqa | 0.2980 | 0.3020 | 0.3000 | 0.3060 |+| boolq | 0.6927 | 0.6936 | 0.6939 | 0.6966 |+| arc_easy | 0.7420 | 0.7403 | 0.7353 | 0.7357 |+| arc_challenge | 0.4019 | 0.4061 | 0.3933 | 0.3857 |diff --git a/docs/gemma-7b-acc.md b/docs/gemma-7b-acc.mdindex b371eadd9..aefcb8b9d 100644--- a/docs/gemma-7b-acc.md+++ b/docs/gemma-7b-acc.md@@ -1,22 +1,23 @@-Install [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness.git) from source, and the git id 96d185fa6232a5ab685ba7c43e45d1dbb3bb906d, Install the latest [AutoGPTQ](https://github.com/AutoGPTQ/AutoGPTQ) from source first+pip install lm-eval==0.4.2+pip install auto-gptqPlease note that there is a discrepancy between the baseline result and the official data, which is a known issue within the official model card community.+Given that the Gemma model family exhibits inconsistent results between FP16 and BF16 on lm-eval, we recommend converting to FP16 for both tuning and evaluation.```bash-lm_eval --model hf --model_args pretrained="Intel/gemma-7b-int4-inc",autogptq=True,gptq_use_triton=True --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 32+lm_eval --model hf --model_args pretrained="Intel/gemma-7b-int4-inc",autogptq=True,gptq_use_triton=True,dtype=float16 --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 32```--| Metric | BF16 | int4 |-| -------------- |--------| ------ |-| Avg. | 0.6239 | 0.6307 |-| mmlu | 0.6162 | 0.6147 |-| lambada_openai | 0.6751 | 0.7204 |-| hellaswag | 0.6047 | 0.5903 |-| winogrande | 0.7324 | 0.7514 |-| piqa | 0.7943 | 0.7949 |-| truthfulqa_mc1 | 0.3097 | 0.3011 |-| openbookqa | 0.3320 | 0.3400 |-| boolq | 0.8278 | 0.8269 |-| rte | 0.6534 | 0.7076 |-| arc_easy | 0.8178 | 0.7959 |-| arc_challenge | 0.4991 | 0.4940 |+| Metric | BF16 | FP16 | AutoRound v0.1 | AutoRound V0.2 |+| -------------- | ---- | ------ |----------------|----------------|+| Avg. | 0.6208 | 0.6302 | 0.6242 | 0.6254 |+| mmlu | 0.6126 | 0.6189 | 0.6085 | 0.6147 |+| lambada_openai | 0.6707 | 0.7308 | 0.7165 | 0.7270 |+| hellaswag | 0.6039 | 0.6063 | 0.6017 | 0.6017 |+| winogrande | 0.7356 | 0.7506 | 0.7482 | 0.7490 |+| piqa | 0.8014 | 0.8025 | 0.7976 | 0.7982 |+| truthfulqa_mc1 | 0.3121 | 0.3121 | 0.3060 | 0.2840 |+| openbookqa | 0.3300 | 0.3220 | 0.3340 | 0.3240 |+| boolq | 0.8254 | 0.8324 | 0.8300 | 0.8407 |+| rte | 0.6643 | 0.6859 | 0.6787 | 0.6968 |+| arc_easy | 0.8068 | 0.8262 | 0.8089 | 0.8194 |+| arc_challenge | 0.5043 | 0.5000 | 0.4915 | 0.4949 |diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex df856236c..078329244 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -121,6 +121,9 @@ if __name__ == '__main__':parser.add_argument("--quant_lm_head", action='store_true',help="quant_lm_head")+ parser.add_argument("--model_dtype", default=None, type=str,+ help="force to convert the dtype, some backends supports fp16 dtype better")+args = parser.parse_args()if args.low_gpu_mem_usage:print(@@ -212,6 +215,12 @@ if __name__ == '__main__':model.seqlen = seqlenseqlen = args.seqlen+ if args.model_dtype != None:+ if args.model_dtype == "float16" or args.model_dtype == "fp16":+ model = model.to(torch.float16)+ if args.model_dtype == "bfloat16" or args.model_dtype == "bfp16":+ model = model.to(torch.bfloat16)+# if "llama" in model_name:# from transformers import LlamaTokenizer#@@ -312,7 +321,7 @@ if __name__ == '__main__':inplace = True if len(deployment_device) < 2 else Falseif 'gpu' in deployment_device:- autoround.save_quantized(f'{export_dir}-gpu', format=gpu_format, use_triton=True, inplace=inplace)+ autoround.save_quantized(f'{export_dir}-gpu', format=gpu_format, use_triton=True, inplace=inplace)if 'xpu' in deployment_device:autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,compression_dtype=torch.int8, compression_dim=0, use_optimum_format=False,diff --git a/examples/language-modeling/scripts/gemma-2b.sh b/examples/language-modeling/scripts/gemma-2b.shindex e562d55c1..b7136bfb7 100644--- a/examples/language-modeling/scripts/gemma-2b.sh+++ b/examples/language-modeling/scripts/gemma-2b.sh@@ -7,5 +7,7 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 400 \+--model_dtype "float16" \+--disable_low_gpu_mem_usage \--deployment_device 'gpu' \--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/gemma-7b-it.sh b/examples/language-modeling/scripts/gemma-7b-it.shindex 4e19ef55d..5b13c99bd 100644--- a/examples/language-modeling/scripts/gemma-7b-it.sh+++ b/examples/language-modeling/scripts/gemma-7b-it.sh@@ -5,5 +5,6 @@ python3 main.py \--bits 4 \--iters 1000 \--minmax_lr 2e-3 \+--model_dtype "float16" \--deployment_device 'gpu' \--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/gemma-7b.sh b/examples/language-modeling/scripts/gemma-7b.shindex 0ad3a3f16..bf89b0704 100644--- a/examples/language-modeling/scripts/gemma-7b.sh+++ b/examples/language-modeling/scripts/gemma-7b.sh@@ -6,4 +6,6 @@ python3 main.py \--iters 1000 \--minmax_lr 2e-3 \--deployment_device 'gpu' \+--model_dtype "float16" \+--disable_low_gpu_mem_usage \--output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh b/examples/language-modeling/scripts/neural-chat-7b-v3-1.shindex 208694e0b..5bc14d124 100644--- a/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh+++ b/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh@@ -5,7 +5,7 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \---minmax_lr 0.0002 \+--minmax_lr 2e-3 \--deployment_device 'gpu' \--output_dir "./tmp_autoround" \--disable_quanted_input\ No newline at end of filediff --git a/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh b/examples/language-modeling/scripts/neural-chat-7b-v3-3.shindex c7db8b1f4..15a3d4b71 100644--- a/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh+++ b/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh@@ -4,7 +4,7 @@ python3 main.py \--group_size 128 \--bits 4 \--iters 1000 \---minmax_lr 0.0002 \+--minmax_lr 2e-3 \--deployment_device 'gpu' \--output_dir "./tmp_autoround" \--disable_quanted_input\ No newline at end of file