Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix autogptq exporting issue and update readme

wenhuach21b99431b05cfa
diff --git a/README.md b/README.md
index 1439c42f7..5e5a4b3c9 100644
--- a/README.md
+++ b/README.md
@@ -175,7 +175,7 @@ For wikitext2/ptb-new/c4-new ppl, we follow the code of gptq and set the sequenc
</tr>
</tr>
- <th>Ours</th>
+ <th>Ours </th>
<td>66.90</td> <! acc avg -->
<td>60.56</td> <! MMLU -->
<td>72.19</td> <! Lambada_openai -->
@@ -195,7 +195,7 @@ For wikitext2/ptb-new/c4-new ppl, we follow the code of gptq and set the sequenc
</tr>
</tr>
- <th>Ours iters=1K,use_quant_input=False, minmax_lr=0.002</th>
+ <th>Ours <a href=https://huggingface.co/Intel/neural-chat-v3-3-int4-inc> hf_model_card</a> iters=1K,use_quant_input=False, minmax_lr=0.002</th>
<td>67.70</td> <! acc avg -->
<td>60.57</td> <! MMLU -->
<td>73.74</td> <! Lambada_openai -->
@@ -344,7 +344,7 @@ We provide a [comprehensive analysis](docs/README.md) with other methods in our
1 Consider increasing tuning steps to achieve better results, albeit with increased tuning time. Additionally, setting 'use_quant_input' to False or adjusting 'minmax_lr' to 2.0/iters has been observed to occasionally yield improved results.
2 Leverage AutoGPTQ to run the model on GPU
-
+please install auto-gptq https://github.com/AutoGPTQ/ from source first
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
@@ -357,12 +357,15 @@ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
autoround = AutoRound(model, tokenizer, bits=4, group_size=128, scheme="asym")
autoround.quantize()
-
-## export to autogptq
-# please install auto-gptq https://github.com/AutoGPTQ/
-output_dir = "/path/to/quantized_model"
+output_dir = "./tmp_autoround"
autoround.export(output_dir, target="auto_gptq", use_triton=True)
-# then follow auto-gptq to load the model and inference
+
+# then follow auto-gptq or transformers to load the model and inference
+model = AutoModelForCausalLM.from_pretrained(output_dir, device_map="auto",
+ trust_remote_code=False)
+print(tokenizer.decode(
+ model.generate(**tokenizer("There is a girl who likes adventure,", return_tensors="pt").to(model.device),
+ max_new_tokens=50)[0]))
```
## Examples
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 357614ec9..d436dc8e9 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -962,6 +962,8 @@ class AutoRound(object):
if not self.quantized:
logger.warning("please run autoround.quantize first")
return
+ if self.tokenizer is not None:
+ self.tokenizer.save_pretrained(output_dir)
model = copy.deepcopy(self.model.to("cpu")) ##TODO avoid this deepcopy
from auto_gptq.modeling._utils import pack_model
@@ -1068,7 +1070,6 @@ class AutoRound(object):
cost_time = end_time - start_time
logger.info(f"quantization tuning time {cost_time}")
self.quantized = True
- ##self.export_to_autogptq("test_export")
return self.model, self.weight_config
diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.py
index f7393318c..571048c1c 100644
--- a/auto_round/export/export_to_autogptq.py
+++ b/auto_round/export/export_to_autogptq.py
@@ -42,7 +42,7 @@ import os
from typing import Dict, List, Optional, Union
from safetensors.torch import save_file as safe_save
from os.path import join, isfile, isdir
-import copy
+import json
def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym=False, iters=200, lr=5e-3,
minmax_lr=5e-3,
@@ -52,7 +52,7 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym
os.makedirs(save_dir, exist_ok=True)
model.to("cpu")
- model_base_name = f"autoround-model-{str(bits)}bit-{str(group_size)}g"
+ model_base_name = f"model"
if use_safetensors:
model_save_name = model_base_name + ".safetensors"
state_dict = model.state_dict()
@@ -89,7 +89,7 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym
# Store the quantization configuration as safetensors metadata
from auto_round import __version__
- safetensors_metadata['auto_round_version'] = str(__version__)
+ safetensors_metadata['version'] = str(__version__)
safetensors_metadata['bits'] = str(bits)
safetensors_metadata['group_size'] = str(group_size)
safetensors_metadata['iters'] = str(iters)
@@ -104,10 +104,24 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym
model.config.save_pretrained(save_dir)
from auto_gptq.modeling._base import BaseQuantizeConfig
+
quantization_config = BaseQuantizeConfig(bits=bits, group_size=group_size, desc_act=False, sym=sym,
true_sequential=False, static_groups=False,
model_file_base_name=model_base_name)
quantization_config.model_file_base_name = model_base_name
- quantization_config.save_pretrained(save_dir)
+ config_dict = quantization_config.to_dict()
+ config_dict["quant_method"] = "intel/auto-round"
+ config_dict['version'] = __version__
+ config_dict['iters'] = iters
+ config_dict['lr'] = lr
+ config_dict['minmax_lr'] = minmax_lr
+ config_dict['enable_minmax_tuning'] = enable_minmax_tuning
+ config_dict['use_quant_input'] = use_quant_input
+
+ with open(join(save_dir, "quantize_config.json"), "w", encoding="utf-8") as f:
+ json.dump(config_dict, f, indent=2)
+
+
+
diff --git a/examples/language-modeling/requirements.txt b/examples/language-modeling/requirements.txt
index e319bb19c..09bebc4ec 100644
--- a/examples/language-modeling/requirements.txt
+++ b/examples/language-modeling/requirements.txt
@@ -1,4 +1,4 @@
-transformers==4.36.0
+transformers
torch
git+https://github.com/EleutherAI/lm-evaluation-harness.git@008fc2a23245c40384f2312718433eeb1e0f87a9
tiktoken
diff --git a/requirements.txt b/requirements.txt
index 3e64679d4..f7b9766e8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-transformers==4.36.0
+transformers
torch
sentencepiece
accelerate