Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

update gemma recipe (#121)

wenhuach21ecc1dd65ec7a
diff --git a/README.md b/README.md
index e2f693656..51c6fbe42 100644
--- a/README.md
+++ b/README.md
@@ -51,7 +51,7 @@ pip install auto-round
## Model quantization
### Gaudi2/ CPU/ GPU
-
+We found a significant accuracy discrepancy with the qdq model using the AutoGPTQ GPU backend with asymmetric quantization in some scenarios. Please switch to symmetric quantization to alleviate this issue.
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -142,7 +142,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
-
+##from auto_round.export import AutoHfQuantizer ## uncomment it for models with quantized lm-head
quantized_model_path = "./tmp_autoround"
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, use_fast=True)
@@ -163,7 +163,6 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
| google/gemma-2b | [HF-int4-model](https://huggingface.co/Intel/gemma-2b-int4-inc), [accuracy](./docs/gemma-2b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-2b.sh), [example](./examples/language-modeling/)
| mistralai/Mistral-7B-Instruct-v0.2 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-Instruct-v0.2-int4-inc) (under review), [accuracy](./docs/Mistral-7B-Instruct-v0.2-acc.md), [recipe](./examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh), [example](./examples/language-modeling/) |
| google/gemma-7b | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-int4-inc) (under review), [accuracy](./docs/gemma-7b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b.sh), [example](./examples/language-modeling/) |
-| google/gemma-7b-it | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-it-int4-inc) (under review), [accuracy](./docs/gemma-7b-it-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b-it.sh), [example](./examples/language-modeling/) |
| mistralai/Mixtral-8x7B-Instruct-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-Instruct-v0.1-int4-inc) (under review), [accuracy](./docs/Mixtral-8x7B-Instruct-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh), [example](./examples/language-modeling/) |
| mistralai/Mixtral-8x7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-v0.1-int4-inc) (under review), [accuracy](./docs/Mixtral-8x7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh), [example](./examples/language-modeling/) |
| meta-llama/Meta-Llama-3-8B-Instruct | [accuracy](./docs/Meta-Llama-3-8B-Instruct-acc.md), [recipe](./examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh), [example](./examples/language-modeling/) |
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 4da651074..52ce1a7ed 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -503,6 +503,7 @@ class AutoRound(object):
"gradient_accumulate_steps",
"iters",
"amp",
+ "n_samples",
"low_gpu_mem_usage",
]
if isinstance(dataset, str):
diff --git a/auto_round/export/export_to_autoround/autoround_quantizer.py b/auto_round/export/export_to_autoround/autoround_quantizer.py
index 9b75b9dd4..11f788450 100644
--- a/auto_round/export/export_to_autoround/autoround_quantizer.py
+++ b/auto_round/export/export_to_autoround/autoround_quantizer.py
@@ -140,9 +140,9 @@ class AutoHfQuantizer:
@classmethod
def merge_quantization_configs(
- cls,
- quantization_config: Union[dict, QuantizationConfigMixin],
- quantization_config_from_args: Optional[QuantizationConfigMixin],
+ cls,
+ quantization_config: Union[dict, QuantizationConfigMixin],
+ quantization_config_from_args: Optional[QuantizationConfigMixin],
):
"""Handles situations where both quantization_config
from args and quantization_config from model config are present."""
@@ -156,7 +156,10 @@ class AutoHfQuantizer:
warning_msg = ""
if isinstance(quantization_config, dict):
- quantization_config = AutoRoundConfig.from_dict(quantization_config)
+ if "auto-round" in quantization_config["quant_method"]:
+ quantization_config = AutoRoundConfig.from_dict(quantization_config)
+ else:
+ quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101
if isinstance(quantization_config, (GPTQConfig, AwqConfig)) and quantization_config_from_args is not None:
# special case for GPTQ / AWQ config collision
@@ -191,22 +194,22 @@ class AutoRoundConfig(QuantizationConfigMixin):
"""
def __init__(
- self,
- bits: int,
- tokenizer: Any = None,
- dataset: str = None,
- group_size: int = 128,
- sym: bool = False,
- backend="gptq:exllamav2",
- iters: int = 200,
- weight_config: dict = None,
- enable_quanted_input=True,
- enable_minmax_tuning=True,
- lr=None,
- minmax_lr=None,
- n_samples=512,
- seqlen=2048,
- **kwargs,
+ self,
+ bits: int,
+ tokenizer: Any = None,
+ dataset: str = None,
+ group_size: int = 128,
+ sym: bool = False,
+ backend="gptq:exllamav2",
+ iters: int = 200,
+ weight_config: dict = None,
+ enable_quanted_input=True,
+ enable_minmax_tuning=True,
+ lr=None,
+ minmax_lr=None,
+ n_samples=512,
+ seqlen=2048,
+ **kwargs,
):
self.bits = bits
self.tokenizer = tokenizer
@@ -358,13 +361,13 @@ class AutoRoundQuantizer(HfQuantizer):
in_features = layer.weight.shape[0]
out_features = layer.weight.shape[1]
bias = layer.bias is not None
- new_layer = QuantLinear(
+ new_layer = QuantLinear( # pylint: disable=E1123
bits,
group_size,
in_features,
out_features,
bias,
- weight_dtype=layer.weight.dtype, # pylint: disable=E1123
+ weight_dtype=layer.weight.dtype,
)
new_layer.device = device
diff --git a/auto_round/export/export_to_autoround/export_to_autoround.py b/auto_round/export/export_to_autoround/export_to_autoround.py
index fab55a26c..34d0ac132 100644
--- a/auto_round/export/export_to_autoround/export_to_autoround.py
+++ b/auto_round/export/export_to_autoround/export_to_autoround.py
@@ -140,8 +140,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="gptq:exllamav
out_features = layer.weight.shape[1]
bias = layer.bias is not None and torch.any(layer.bias)
- new_layer = QuantLinear(
- bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype ##pylint: disable=E1123
+ new_layer = QuantLinear( ##pylint: disable=E1123
+ bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype
)
new_layer.device = device
diff --git a/docs/gemma-2b-acc.md b/docs/gemma-2b-acc.md
index 033d37dca..7ba60de03 100644
--- a/docs/gemma-2b-acc.md
+++ b/docs/gemma-2b-acc.md
@@ -1,27 +1,29 @@
### Evaluate the model
-Install [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness.git) from source, and the git id we used is 96d185fa6232a5ab685ba7c43e45d1dbb3bb906d
+Install [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness.git) from source, lm-eval 0.4.2 is used
+
+pip install auto-gptq
-pip install auto-gptq[triton]
-pip install triton==2.2.0
Please note that there is a discrepancy between the baseline result and the official data, which is a known issue within the official model card community.
+Given that the Gemma model family exhibits inconsistent results between FP16 and BF16 on lm-eval, we recommend converting to FP16 for both tuning and evaluation.
```bash
-lm_eval --model hf --model_args pretrained="Intel/gemma-2b-int4-inc",autogptq=True,gptq_use_triton=True --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 16
+lm_eval --model hf --model_args pretrained="Intel/gemma-2b-int4-inc",autogptq=True,gptq_use_triton=True,dtype=float16 --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 16
```
-| Metric | BF16 | INT4 |
-| -------------- |--------|--------|
-| Avg. | 0.5383 | 0.5338 |
-| mmlu | 0.3337 | 0.3276 |
-| lambada_openai | 0.6398 | 0.6319 |
-| hellaswag | 0.5271 | 0.5161 |
-| winogrande | 0.6472 | 0.6472 |
-| piqa | 0.7699 | 0.7622 |
-| truthfulqa_mc1 | 0.2203 | 0.2191 |
-| openbookqa | 0.3020 | 0.2980 |
-| boolq | 0.6939 | 0.6939 |
-| rte | 0.6426 | 0.6498 |
-| arc_easy | 0.7424 | 0.7348 |
-| arc_challenge | 0.4019 | 0.3908 |
+
+
+| Metric | BF16 | FP16 | AutoRound v0.1 | AutoRound v0.2 |
+| -------------- | ---- | ------ |----------------|----------------|
+| Avg.| 0.5263 | 0.5277 | 0.5235 | 0.5248 |
+| mmlu | 0.3287 | 0.3287 | 0.3297 | 0.3309 |
+| lambada_openai | 0.6344 | 0.6375 | 0.6307 | 0.6379 |
+| hellaswag | 0.5273 | 0.5281 | 0.5159 | 0.5184 |
+| winogrande | 0.6504 | 0.6488 | 0.6543 | 0.6575 |
+| piqa | 0.7671 | 0.7720 | 0.7612 | 0.7606 |
+| truthfulqa_mc1 | 0.2203 | 0.2203 | 0.2203 | 0.2191 |
+| openbookqa | 0.2980 | 0.3020 | 0.3000 | 0.3060 |
+| boolq | 0.6927 | 0.6936 | 0.6939 | 0.6966 |
+| arc_easy | 0.7420 | 0.7403 | 0.7353 | 0.7357 |
+| arc_challenge | 0.4019 | 0.4061 | 0.3933 | 0.3857 |
diff --git a/docs/gemma-7b-acc.md b/docs/gemma-7b-acc.md
index b371eadd9..aefcb8b9d 100644
--- a/docs/gemma-7b-acc.md
+++ b/docs/gemma-7b-acc.md
@@ -1,22 +1,23 @@
-Install [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness.git) from source, and the git id 96d185fa6232a5ab685ba7c43e45d1dbb3bb906d, Install the latest [AutoGPTQ](https://github.com/AutoGPTQ/AutoGPTQ) from source first
+pip install lm-eval==0.4.2
+pip install auto-gptq
Please note that there is a discrepancy between the baseline result and the official data, which is a known issue within the official model card community.
+Given that the Gemma model family exhibits inconsistent results between FP16 and BF16 on lm-eval, we recommend converting to FP16 for both tuning and evaluation.
```bash
-lm_eval --model hf --model_args pretrained="Intel/gemma-7b-int4-inc",autogptq=True,gptq_use_triton=True --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 32
+lm_eval --model hf --model_args pretrained="Intel/gemma-7b-int4-inc",autogptq=True,gptq_use_triton=True,dtype=float16 --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 32
```
-
-| Metric | BF16 | int4 |
-| -------------- |--------| ------ |
-| Avg. | 0.6239 | 0.6307 |
-| mmlu | 0.6162 | 0.6147 |
-| lambada_openai | 0.6751 | 0.7204 |
-| hellaswag | 0.6047 | 0.5903 |
-| winogrande | 0.7324 | 0.7514 |
-| piqa | 0.7943 | 0.7949 |
-| truthfulqa_mc1 | 0.3097 | 0.3011 |
-| openbookqa | 0.3320 | 0.3400 |
-| boolq | 0.8278 | 0.8269 |
-| rte | 0.6534 | 0.7076 |
-| arc_easy | 0.8178 | 0.7959 |
-| arc_challenge | 0.4991 | 0.4940 |
+| Metric | BF16 | FP16 | AutoRound v0.1 | AutoRound V0.2 |
+| -------------- | ---- | ------ |----------------|----------------|
+| Avg. | 0.6208 | 0.6302 | 0.6242 | 0.6254 |
+| mmlu | 0.6126 | 0.6189 | 0.6085 | 0.6147 |
+| lambada_openai | 0.6707 | 0.7308 | 0.7165 | 0.7270 |
+| hellaswag | 0.6039 | 0.6063 | 0.6017 | 0.6017 |
+| winogrande | 0.7356 | 0.7506 | 0.7482 | 0.7490 |
+| piqa | 0.8014 | 0.8025 | 0.7976 | 0.7982 |
+| truthfulqa_mc1 | 0.3121 | 0.3121 | 0.3060 | 0.2840 |
+| openbookqa | 0.3300 | 0.3220 | 0.3340 | 0.3240 |
+| boolq | 0.8254 | 0.8324 | 0.8300 | 0.8407 |
+| rte | 0.6643 | 0.6859 | 0.6787 | 0.6968 |
+| arc_easy | 0.8068 | 0.8262 | 0.8089 | 0.8194 |
+| arc_challenge | 0.5043 | 0.5000 | 0.4915 | 0.4949 |
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index df856236c..078329244 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -121,6 +121,9 @@ if __name__ == '__main__':
parser.add_argument("--quant_lm_head", action='store_true',
help="quant_lm_head")
+ parser.add_argument("--model_dtype", default=None, type=str,
+ help="force to convert the dtype, some backends supports fp16 dtype better")
+
args = parser.parse_args()
if args.low_gpu_mem_usage:
print(
@@ -212,6 +215,12 @@ if __name__ == '__main__':
model.seqlen = seqlen
seqlen = args.seqlen
+ if args.model_dtype != None:
+ if args.model_dtype == "float16" or args.model_dtype == "fp16":
+ model = model.to(torch.float16)
+ if args.model_dtype == "bfloat16" or args.model_dtype == "bfp16":
+ model = model.to(torch.bfloat16)
+
# if "llama" in model_name:
# from transformers import LlamaTokenizer
#
@@ -312,7 +321,7 @@ if __name__ == '__main__':
inplace = True if len(deployment_device) < 2 else False
if 'gpu' in deployment_device:
- autoround.save_quantized(f'{export_dir}-gpu', format=gpu_format, use_triton=True, inplace=inplace)
+ autoround.save_quantized(f'{export_dir}-gpu', format=gpu_format, use_triton=True, inplace=inplace)
if 'xpu' in deployment_device:
autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,
compression_dtype=torch.int8, compression_dim=0, use_optimum_format=False,
diff --git a/examples/language-modeling/scripts/gemma-2b.sh b/examples/language-modeling/scripts/gemma-2b.sh
index e562d55c1..b7136bfb7 100644
--- a/examples/language-modeling/scripts/gemma-2b.sh
+++ b/examples/language-modeling/scripts/gemma-2b.sh
@@ -7,5 +7,7 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 400 \
+--model_dtype "float16" \
+--disable_low_gpu_mem_usage \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/gemma-7b-it.sh b/examples/language-modeling/scripts/gemma-7b-it.sh
index 4e19ef55d..5b13c99bd 100644
--- a/examples/language-modeling/scripts/gemma-7b-it.sh
+++ b/examples/language-modeling/scripts/gemma-7b-it.sh
@@ -5,5 +5,6 @@ python3 main.py \
--bits 4 \
--iters 1000 \
--minmax_lr 2e-3 \
+--model_dtype "float16" \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/gemma-7b.sh b/examples/language-modeling/scripts/gemma-7b.sh
index 0ad3a3f16..bf89b0704 100644
--- a/examples/language-modeling/scripts/gemma-7b.sh
+++ b/examples/language-modeling/scripts/gemma-7b.sh
@@ -6,4 +6,6 @@ python3 main.py \
--iters 1000 \
--minmax_lr 2e-3 \
--deployment_device 'gpu' \
+--model_dtype "float16" \
+--disable_low_gpu_mem_usage \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh b/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh
index 208694e0b..5bc14d124 100644
--- a/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh
+++ b/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh
@@ -5,7 +5,7 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
---minmax_lr 0.0002 \
+--minmax_lr 2e-3 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround" \
--disable_quanted_input
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh b/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh
index c7db8b1f4..15a3d4b71 100644
--- a/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh
+++ b/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh
@@ -4,7 +4,7 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
---minmax_lr 0.0002 \
+--minmax_lr 2e-3 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround" \
--disable_quanted_input
\ No newline at end of file