Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

rename export to save quantized (#15)

wenhuach212fb6fd11f267
diff --git a/README.md b/README.md
index 928184a44..9ad6ea505 100644
--- a/README.md
+++ b/README.md
@@ -39,11 +39,12 @@ autoround.quantize()
output_dir = "./tmp_autoround"
deployment_device = "cpu" ## or gpu
if deployment_device=="cpu":
- autoround.export(output_dir, target="itrex") ## export to itrex format
+ autoround.save_quantized(output_dir, format="itrex") ## export to itrex format
else:
- autoround.export(output_dir, target="auto_gptq", use_triton=True) ##export to autogptq format
+ autoround.save_quantized(output_dir, format="auto_gptq", use_triton=True) ##export to autogptq format
```
+
### On Intel Gaudi2
```python
@@ -51,14 +52,14 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
model_name = "meta-llama/Llama-2-7b-hf"
model = AutoModelForCausalLM.from_pretrained(
- model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True)
+ model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
bits, group_size, scheme = 4, 128, "asym"
# need to load model first, then import
from auto_round import AutoRound
autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme,
- device="hpu", scale_dtype="bf16", amp=False)
+ device="hpu", amp=False)
autoround.quantize()
```
@@ -119,31 +120,38 @@ Please run the tuning code first
### Intel CPU
```python
+# save_quantized to itrex format first
# Please read ITREX(https://github.com/intel/intel-extension-for-transformers/tree/main/intel_extension_for_transformers/llm/runtime/neural_speed) to understand the details
# currently please install neural-speed (https://github.com/intel/neural-speed) from source
from intel_extension_for_transformers.transformers import AutoModelForCausalLM, WeightOnlyQuantConfig
-
+from transformers import AutoTokenizer
quantized_model_path = "./tmp_autoround"
-woq_config = WeightOnlyQuantConfig(group_size=group_size, scheme=scheme, use_autoround=True) ##only supports 4 bits currently
+woq_config = WeightOnlyQuantConfig(group_size=group_size, scheme=scheme,
+ use_autoround=True) ##only supports 4 bits currently
prompt = "There is a girl who likes adventure,"
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, trust_remote_code=True)
inputs = tokenizer(prompt, return_tensors="pt").input_ids
-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, quantization_config=woq_config, trust_remote_code=True,device="cpu")
+model = AutoModelForCausalLM.from_pretrained(quantized_model_path, quantization_config=woq_config,
+ trust_remote_code=True, device="cpu")
outputs = model.generate(inputs, max_new_tokens=50)
-
```
### GPU
```python
-# follow auto-gptq or transformers to load the model and inference
+# save_quantized to autogptq format first and then follow transformers or auto-gptq to load the model and inference
+from transformers import AutoModelForCausalLM, AutoTokenizer
quantized_model_path = "./tmp_autoround"
-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",
- trust_remote_code=False)
-prompt = "There is a girl who likes adventure,"
-inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-print(tokenizer.decode(
- model.generate(**tokenizer(prompt, return_tensors="pt").to(model.device),max_new_tokens=50)[0]))
+model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
+ device_map="auto",
+ trust_remote_code=True
+ )
+tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, use_fast=True)
+text = "There is a girl who likes adventure,"
+inputs = tokenizer(text, return_tensors="pt").to(model.device)
+print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
```
## Huggingface Model cards
+We fine-tuned the hyperparameters for each model with an iteration of 1K and successfully achieved near-lossless quantized models in the majority of scenarios. Some of these models has been uploaded to the Huggingface Hub.
+
### AutoGPTQ format
[Intel/neural-chat-7b-v3-3-int4-inc](https://huggingface.co/Intel/neural-chat-7b-v3-3-int4-inc)
@@ -155,222 +163,13 @@ print(tokenizer.decode(
[Intel/Mixtral-8x7B-v0.1-int4-inc](https://huggingface.co/Intel/Mixtral-8x7B-v0.1-int4-inc) coming soon
+[Intel/phi-2-int4-inc](https://huggingface.co/Intel/phi-2-int4-inc) coming soon
+
### Itrex format
Please stay tuned
-## Validated Models
-For wikitext2/ptb-new/c4-new ppl, we follow the code of gptq and set the sequence length to 2048. For lm-eval wikitext ppl, we adopt lm-eval. The quantization configure is W4G128.
-
-The lm-eval-harness git id we used in the following is 008fc2a23245c40384f2312718433eeb1e0f87a9 and we evaluated on qdq fake model.
-
-<table border="1">
- <tr>
- <th>Model</th>
- <th>Method </th>
- <th>Acc AVG.</th>
- <th>MMLU</th>
- <th>Lamb.</th>
- <th>Hella.</th>
- <th>Wino.</th>
- <th>Piqa</th>
- <th>Truth.</th>
- <th>Open.</th>
- <th>Boolq</th>
- <th>RTE</th>
- <th>ARC-e</th>
- <th>ARC-c.</th>
- <th>wikitext2 ppl
- <th>ptb_new ppl</th>
- <th>c4_new ppl</th>
- <th>lm_eval wikitext ppl</th>
-
- </tr>
-
- <tr>
- <td rowspan="3">Intel/neural-chat-7b-v3-3 </td>
- <th>FP16</th>
- <td>67.92</td> <! acc avg -->
- <td>61.13</td> <! MMLU -->
- <td>73.03</td> <! Lambada_openai -->
- <td>66.39</td> <! Hellsaswag -->
- <td>76.40</td> <! Winogrande -->
- <td>81.01</td> <! Piqa -->
- <td>47.37</td> <! Truthfulqa -->
- <td>38.8</td> <! Openbookqa -->
- <td>86.97</td> <! Boolq -->
- <td>75.81</td> <! RTE -->
- <td>82.66</td> <! Arc easy -->
- <td>57.51</td> <! Arc Challenge -->
- <td>6.00</td> <! wikitext2 ppl -->
- <td>48.96</td> <! ptb_new ppl -->
- <td>9.65</td> <! c4_new ppl -->
- <td>-</td> <! lm-eval wikitext ppl -->
- </tr>
-
- </tr>
- <th>Ours </th>
- <td>66.90</td> <! acc avg -->
- <td>60.56</td> <! MMLU -->
- <td>72.19</td> <! Lambada_openai -->
- <td>65.28</td> <! Hellsaswag -->
- <td>75.37</td> <! Winogrande -->
- <td>81.18</td> <! Piqa -->
- <td>46.76</td> <! Truthfulqa -->
- <td>36.0</td> <! Openbookqa -->
- <td>86.91</td> <! Boolq -->
- <td>73.29</td> <! RTE -->
- <td>81.73</td> <! Arc easy -->
- <td>56.66</td> <! Arc Challenge -->
- <td>6.21</td> <! wikitext2 ppl -->
- <td>59.78</td> <! ptb_new ppl -->
- <td>10.01</td> <! c4_new ppl -->
- <td>-</td> <! lm-eval wikitext ppl -->
- </tr>
-
- </tr>
- <th>Ours iters=1K,use_quant_input=False, minmax_lr=0.002</th>
- <td>67.70</td> <! acc avg -->
- <td>60.57</td> <! MMLU -->
- <td>73.74</td> <! Lambada_openai -->
- <td>65.62</td> <! Hellsaswag -->
- <td>77.43</td> <! Winogrande -->
- <td>80.85</td> <! Piqa -->
- <td>47.61</td> <! Truthfulqa -->
- <td>36.8</td> <! Openbookqa -->
- <td>86.94</td> <! Boolq -->
- <td>75.09</td> <! RTE -->
- <td>82.66</td> <! Arc easy -->
- <td>57.34</td> <! Arc Challenge -->
- <td>6.17</td> <! wikitext2 ppl -->
- <td>59.12</td> <! ptb_new ppl -->
- <td>9.83</td> <! c4_new ppl -->
- <td>-</td> <! lm-eval wikitext ppl -->
- </tr>
-
-
- <tr>
- <td rowspan="3">mistralai/Mixtral-8x7B-v0.1 </td>
- <th>BF16</th>
- <td>67.16</td>
- <td>69.83</td>
- <td>78.44</td>
- <td>64.89</td>
- <td>76.40</td>
- <td>82.43</td>
- <td>34.15</td>
- <td>35.40</td>
- <td>84.98</td>
- <td>71.12</td>
- <td>84.22</td>
- <td>56.91</td>
- <td>3.84</td>
- <td>19.22</td>
- <td>7.41</td>
- <td>-</td>
-
- </tr>
- <tr>
- <th>Ours</th>
- <td>65.98</td>
- <td>68.90</td>
- <td>78.11</td>
- <td>64.31</td>
- <td>74.27</td>
- <td>82.10</td>
- <td>30.97</td>
- <td>34.20</td>
- <td>84.57</td>
- <td>67.87</td>
- <td>83.96</td>
- <td>56.57</td>
- <td>4.08</td>
- <td>354</td>
- <td>7.56</td>
- <td>-</td>
- </tr>
- <tr>
- <th>Ours iters=1K,use_quant_input=False
- <td>66.78</td>
- <td>68.68</td>
- <td>78.61</td>
- <td>64.40</td>
- <td>76.56</td>
- <td>81.99</td>
- <td>32.56</td>
- <td>34.80</td>
- <td>85.96</td>
- <td>70.76</td>
- <td>83.96</td>
- <td>56.31</td>
- <td>3.99</td>
- <td>17.65</td>
- <td>7.52</td>
- <td>-</td>
-
- </tr>
- <tr>
- <td rowspan="3">microsoft/phi-2 </td>
- <th>FP16</th>
- <td>61.80</td>
- <td>56.40</td>
- <td>62.78</td>
- <td>55.83</td>
- <td>75.77</td>
- <td>78.67</td>
- <td>31.21</td>
- <td>40.40</td>
- <td>83.36</td>
- <td>62.45</td>
- <td>80.05</td>
- <td>52.90</td>
- <td>9.71</td>
- <td>18.16</td>
- <td>14.12</td>
- <td>11.05</td>
-
- </tr>
- <tr>
- <th>Ours</th>
- <td>61.67</td>
- <td>54.57</td>
- <td>61.32</td>
- <td>55.04</td>
- <td>76.48</td>
- <td>78.89</td>
- <td>29.74</td>
- <td>40.60</td>
- <td>83.24</td>
- <td>66.43</td>
- <td>79.76</td>
- <td>52.30</td>
- <td>9.98</td>
- <td>18.67</td>
- <td>14.39</td>
- <td>11.37</td>
-
- </tr>
-
- </tr>
- <th>Ours iters=1K,use_quant_input=False </th>
- <td>61.47</td> <! acc avg -->
- <td>55.41</td> <! MMLU -->
- <td>61.77</td> <! Lambada_openai -->
- <td>54.92</td> <! Hellsaswag -->
- <td>76.40</td> <! Winogrande -->
- <td>78.29</td> <! Piqa -->
- <td>31.09</td> <! Truthfulqa -->
- <td>40.0</td> <! Openbookqa -->
- <td>83.24</td> <! Boolq -->
- <td>63.54</td> <! RTE -->
- <td>79.29</td> <! Arc easy -->
- <td>52.22</td> <! Arc Challenge -->
- <td>9.97</td> <! wikitext2 ppl -->
- <td>18.63</td> <! ptb_new ppl -->
- <td>14.37</td> <! c4_new ppl -->
- <td>11.35</td> <! lm-eval wikitext ppl -->
- </tr>
-</table>
+## Comparison with other methods
We provide a [comprehensive analysis](docs/README.md) with other methods in our accuracy data section. Notably, our approach has outperformed GPTQ with a score of 30/32 and AWQ with a score of 27/32 across llamv1/llamav2/mistral-7b on W4G-1, W4G128, W3G128, W2G128. And the tuning costs are comparable.
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 5f509d75a..efed7988b 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -944,15 +944,15 @@ class AutoRound(object):
torch.cuda.empty_cache()
- def export(self, output_dir, target="itrex", **kwargs):
- if target == "itrex":
- self.export_to_itrex(output_dir)
- elif target == "auto_gptq":
- self.export_to_autogptq(output_dir, **kwargs)
+ def save_quantized(self, output_dir, format="itrex", **kwargs):
+ if format == "itrex":
+ self.save_quantized_as_itrex(output_dir)
+ elif format == "auto_gptq":
+ self.save_quantized_as_autogptq(output_dir, **kwargs)
else:
logger.error("export only supports itrex and auto_gptq now")
- def export_to_autogptq(self, output_dir, use_triton=False):
+ def save_quantized_as_autogptq(self, output_dir, use_triton=False):
"""
Export the model to autogptq format to easily leverage cuda kernel
"""
@@ -1012,9 +1012,10 @@ class AutoRound(object):
save_quantized_to_autogptq(model, output_dir, bits=self.bits, group_size=self.group_size, sym=sym,
iters=self.iters, lr=self.lr, minmax_lr=self.minmax_lr,
enable_minmax_tuning=self.enable_minmax_tuning, use_quant_input=self.use_quant_input,
+ scale_dtype=self.scale_dtype,
use_safetensors=True, modules_in_block_to_quantize=modules_in_block_to_quantize)
- def export_to_itrex(self, output_dir):
+ def save_quantized_as_itrex(self, output_dir):
"""Save configure file and weights for CPU backend inference."""
from auto_round.export.export_to_itrex import compress_model
compressed_model, quantize_config = compress_model(self.model, self.weight_config)
diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.py
index 289b83055..a81c8b2b8 100644
--- a/auto_round/export/export_to_autogptq.py
+++ b/auto_round/export/export_to_autogptq.py
@@ -48,7 +48,9 @@ import json
def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym=False, iters=200, lr=5e-3,
minmax_lr=5e-3,
enable_minmax_tuning=True, use_quant_input=True, use_safetensors: bool = True,
- safetensors_metadata: Optional[Dict[str, str]] = None, modules_in_block_to_quantize=None):
+ scale_dtype=torch.float32,
+ safetensors_metadata: Optional[Dict[str, str]] = None,
+ modules_in_block_to_quantize=None):
"""save quantized model and configs to local disk for cuda """
os.makedirs(save_dir, exist_ok=True)
model.to("cpu")
@@ -98,6 +100,7 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym
safetensors_metadata['minmax_lr'] = str(minmax_lr)
safetensors_metadata['enable_minmax_tuning'] = str(enable_minmax_tuning)
safetensors_metadata['use_quant_input'] = str(use_quant_input)
+ safetensors_metadata['scale_dtype'] = str(scale_dtype)
safe_save(state_dict, join(save_dir, model_save_name), safetensors_metadata)
else:
model_save_name = model_base_name + ".bin"
@@ -118,6 +121,7 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym
config_dict['minmax_lr'] = minmax_lr
config_dict['enable_minmax_tuning'] = enable_minmax_tuning
config_dict['use_quant_input'] = use_quant_input
+ config_dict['scale_dtype'] = str(scale_dtype)
if modules_in_block_to_quantize is not None:
config_dict["modules_in_block_to_quantize"] = modules_in_block_to_quantize
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py
index 5af0528c6..60d529a01 100644
--- a/auto_round/export/export_to_itrex/export.py
+++ b/auto_round/export/export_to_itrex/export.py
@@ -29,13 +29,13 @@ from auto_round.utils import quant_weight_w_scale, get_module, set_module
def compress_model(
model,
- weight_config:Union[str, dict],
+ weight_config: Union[str, dict],
enable_full_range=False,
compression_dtype=torch.int32,
compression_dim=1,
device="cpu",
use_optimum_format=True,
- ):
+):
"""Convert Linear to WeightOnlyLinear for low memory inference.
Args:
@@ -73,7 +73,7 @@ def compress_model(
fp_weight = m.weight.data
# scale = torch.tensor(v["scale"], dtype=scale_dtype)
scale, zp = v["scale"], v["zp"]
- convert_dtype=torch.float32 if fp_weight.device.type == "cpu" else scale_dtype
+ convert_dtype = torch.float32 if fp_weight.device.type == "cpu" else scale_dtype
if not isinstance(scale, torch.Tensor):
scale = torch.tensor(scale, dtype=convert_dtype)
zp = None if scheme == "sym" else torch.tensor(zp, dtype=torch.int32)
@@ -96,6 +96,6 @@ def compress_model(
)
new_module.pack(int_weight, scale, zp, m.bias)
set_module(compressed_model, k, new_module)
-
- quantize_config = QuantConfig(bits=num_bits, sym=(scheme=="sym"), group_size=group_size)
+
+ quantize_config = QuantConfig(bits=num_bits, sym=(scheme == "sym"), group_size=group_size)
return compressed_model, quantize_config
diff --git a/docs/acc.md b/docs/acc.md
index 6d7a65d80..542152d0b 100644
--- a/docs/acc.md
+++ b/docs/acc.md
@@ -1,3 +1,8 @@
+### Comparison with other methods
+
+For wikitext2/ptb-new/c4-new ppl, we follow the code of gptq and set the sequence length to 2048. For lm-eval wikitext ppl, we adopt lm-eval. The lm-eval-harness git id we used in the following is 008fc2a23245c40384f2312718433eeb1e0f87a9 and we evaluated on qdq fake model.
+
+
To ensure an equitable comparison and alleviate overfitting in perplexity evaluation on Wikitext or C4, we utilized 512 samples from NeelNanda/pile-10k for all methods during calibration. Due to memory constraints, we maintained the original sequence length of 512 for AWQ, while for GPTQ and our approach, a sequence length of 2048 is used. We have enalbed act-order and true-seqential in GPTQ, and also activated static group in scenarios where group_size!=-1. The notation GPTQ* indicates that we adjusted the random seed or data preprocessing to address issues related to the non-positive definite Hessian matrix or other issues. With these configurations, the tuning costs for GPTQ, AWQ, and ours are comparable.
</br>
@@ -1894,7 +1899,7 @@ To ensure an equitable comparison and alleviate overfitting in perplexity evalua
</br>
-### 4. Accuracies $\uparrow$ across 11 tasks(0-shot) of LLaMA and Mistral models at W2G128.
+### Others. Accuracies $\uparrow$ across 11 tasks(0-shot) of LLaMA and Mistral models at W2G128.
<table border="1">
<tr>
@@ -2524,3 +2529,213 @@ To ensure an equitable comparison and alleviate overfitting in perplexity evalua
</table>
+
+
+
+<table border="1">
+ <tr>
+ <th>Model</th>
+ <th>Method </th>
+ <th>Acc AVG.</th>
+ <th>MMLU</th>
+ <th>Lamb.</th>
+ <th>Hella.</th>
+ <th>Wino.</th>
+ <th>Piqa</th>
+ <th>Truth.</th>
+ <th>Open.</th>
+ <th>Boolq</th>
+ <th>RTE</th>
+ <th>ARC-e</th>
+ <th>ARC-c.</th>
+ <th>wikitext2 ppl
+ <th>ptb_new ppl</th>
+ <th>c4_new ppl</th>
+ <th>lm_eval wikitext ppl</th>
+
+ </tr>
+
+ <tr>
+ <td rowspan="3">Intel/neural-chat-7b-v3-3 </td>
+ <th>FP16</th>
+ <td>67.92</td> <! acc avg -->
+ <td>61.13</td> <! MMLU -->
+ <td>73.03</td> <! Lambada_openai -->
+ <td>66.39</td> <! Hellsaswag -->
+ <td>76.40</td> <! Winogrande -->
+ <td>81.01</td> <! Piqa -->
+ <td>47.37</td> <! Truthfulqa -->
+ <td>38.8</td> <! Openbookqa -->
+ <td>86.97</td> <! Boolq -->
+ <td>75.81</td> <! RTE -->
+ <td>82.66</td> <! Arc easy -->
+ <td>57.51</td> <! Arc Challenge -->
+ <td>6.00</td> <! wikitext2 ppl -->
+ <td>48.96</td> <! ptb_new ppl -->
+ <td>9.65</td> <! c4_new ppl -->
+ <td>-</td> <! lm-eval wikitext ppl -->
+ </tr>
+
+ </tr>
+ <th>Ours </th>
+ <td>66.90</td> <! acc avg -->
+ <td>60.56</td> <! MMLU -->
+ <td>72.19</td> <! Lambada_openai -->
+ <td>65.28</td> <! Hellsaswag -->
+ <td>75.37</td> <! Winogrande -->
+ <td>81.18</td> <! Piqa -->
+ <td>46.76</td> <! Truthfulqa -->
+ <td>36.0</td> <! Openbookqa -->
+ <td>86.91</td> <! Boolq -->
+ <td>73.29</td> <! RTE -->
+ <td>81.73</td> <! Arc easy -->
+ <td>56.66</td> <! Arc Challenge -->
+ <td>6.21</td> <! wikitext2 ppl -->
+ <td>59.78</td> <! ptb_new ppl -->
+ <td>10.01</td> <! c4_new ppl -->
+ <td>-</td> <! lm-eval wikitext ppl -->
+ </tr>
+
+ </tr>
+ <th>Ours iters=1K,use_quant_input=False, minmax_lr=0.002</th>
+ <td>67.70</td> <! acc avg -->
+ <td>60.57</td> <! MMLU -->
+ <td>73.74</td> <! Lambada_openai -->
+ <td>65.62</td> <! Hellsaswag -->
+ <td>77.43</td> <! Winogrande -->
+ <td>80.85</td> <! Piqa -->
+ <td>47.61</td> <! Truthfulqa -->
+ <td>36.8</td> <! Openbookqa -->
+ <td>86.94</td> <! Boolq -->
+ <td>75.09</td> <! RTE -->
+ <td>82.66</td> <! Arc easy -->
+ <td>57.34</td> <! Arc Challenge -->
+ <td>6.17</td> <! wikitext2 ppl -->
+ <td>59.12</td> <! ptb_new ppl -->
+ <td>9.83</td> <! c4_new ppl -->
+ <td>-</td> <! lm-eval wikitext ppl -->
+ </tr>
+
+
+ <tr>
+ <td rowspan="3">mistralai/Mixtral-8x7B-v0.1 </td>
+ <th>BF16</th>
+ <td>67.16</td>
+ <td>69.83</td>
+ <td>78.44</td>
+ <td>64.89</td>
+ <td>76.40</td>
+ <td>82.43</td>
+ <td>34.15</td>
+ <td>35.40</td>
+ <td>84.98</td>
+ <td>71.12</td>
+ <td>84.22</td>
+ <td>56.91</td>
+ <td>3.84</td>
+ <td>19.22</td>
+ <td>7.41</td>
+ <td>-</td>
+
+ </tr>
+ <tr>
+ <th>Ours</th>
+ <td>65.98</td>
+ <td>68.90</td>
+ <td>78.11</td>
+ <td>64.31</td>
+ <td>74.27</td>
+ <td>82.10</td>
+ <td>30.97</td>
+ <td>34.20</td>
+ <td>84.57</td>
+ <td>67.87</td>
+ <td>83.96</td>
+ <td>56.57</td>
+ <td>4.08</td>
+ <td>354</td>
+ <td>7.56</td>
+ <td>-</td>
+ </tr>
+ <tr>
+ <th>Ours iters=1K,use_quant_input=False
+ <td>66.78</td>
+ <td>68.68</td>
+ <td>78.61</td>
+ <td>64.40</td>
+ <td>76.56</td>
+ <td>81.99</td>
+ <td>32.56</td>
+ <td>34.80</td>
+ <td>85.96</td>
+ <td>70.76</td>
+ <td>83.96</td>
+ <td>56.31</td>
+ <td>3.99</td>
+ <td>17.65</td>
+ <td>7.52</td>
+ <td>-</td>
+
+ </tr>
+ <tr>
+ <td rowspan="3">microsoft/phi-2 </td>
+ <th>FP16</th>
+ <td>61.80</td>
+ <td>56.40</td>
+ <td>62.78</td>
+ <td>55.83</td>
+ <td>75.77</td>
+ <td>78.67</td>
+ <td>31.21</td>
+ <td>40.40</td>
+ <td>83.36</td>
+ <td>62.45</td>
+ <td>80.05</td>
+ <td>52.90</td>
+ <td>9.71</td>
+ <td>18.16</td>
+ <td>14.12</td>
+ <td>11.05</td>
+
+ </tr>
+ <tr>
+ <th>Ours</th>
+ <td>61.67</td>
+ <td>54.57</td>
+ <td>61.32</td>
+ <td>55.04</td>
+ <td>76.48</td>
+ <td>78.89</td>
+ <td>29.74</td>
+ <td>40.60</td>
+ <td>83.24</td>
+ <td>66.43</td>
+ <td>79.76</td>
+ <td>52.30</td>
+ <td>9.98</td>
+ <td>18.67</td>
+ <td>14.39</td>
+ <td>11.37</td>
+
+ </tr>
+
+ </tr>
+ <th>Ours iters=1K,use_quant_input=False </th>
+ <td>61.47</td> <! acc avg -->
+ <td>55.41</td> <! MMLU -->
+ <td>61.77</td> <! Lambada_openai -->
+ <td>54.92</td> <! Hellsaswag -->
+ <td>76.40</td> <! Winogrande -->
+ <td>78.29</td> <! Piqa -->
+ <td>31.09</td> <! Truthfulqa -->
+ <td>40.0</td> <! Openbookqa -->
+ <td>83.24</td> <! Boolq -->
+ <td>63.54</td> <! RTE -->
+ <td>79.29</td> <! Arc easy -->
+ <td>52.22</td> <! Arc Challenge -->
+ <td>9.97</td> <! wikitext2 ppl -->
+ <td>18.63</td> <! ptb_new ppl -->
+ <td>14.37</td> <! c4_new ppl -->
+ <td>11.35</td> <! lm-eval wikitext ppl -->
+ </tr>
+</table>
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index d0f4c5804..506d3f006 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -190,11 +190,11 @@ if __name__ == '__main__':
if args.deployment_device == 'cpu':
export_dir += "-cpu"
- autoround.export(output_dir=export_dir)
+ autoround.save_quantized(output_dir=export_dir)
del q_config
elif args.deployment_device == 'gpu':
export_dir += "-gpu"
- autoround.export(export_dir, target="auto_gptq", use_triton=True)
+ autoround.save_quantized(export_dir, format="auto_gptq", use_triton=True)
model = model.eval()
if args.device != "cpu":