PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix autogptq exporting issue and update readme
wenhuach21b99431b05cfa
diff --git a/README.md b/README.mdindex 1439c42f7..5e5a4b3c9 100644--- a/README.md+++ b/README.md@@ -175,7 +175,7 @@ For wikitext2/ptb-new/c4-new ppl, we follow the code of gptq and set the sequenc</tr></tr>- <th>Ours</th>+ <th>Ours </th><td>66.90</td> <! acc avg --><td>60.56</td> <! MMLU --><td>72.19</td> <! Lambada_openai -->@@ -195,7 +195,7 @@ For wikitext2/ptb-new/c4-new ppl, we follow the code of gptq and set the sequenc</tr></tr>- <th>Ours iters=1K,use_quant_input=False, minmax_lr=0.002</th>+ <th>Ours <a href=https://huggingface.co/Intel/neural-chat-v3-3-int4-inc> hf_model_card</a> iters=1K,use_quant_input=False, minmax_lr=0.002</th><td>67.70</td> <! acc avg --><td>60.57</td> <! MMLU --><td>73.74</td> <! Lambada_openai -->@@ -344,7 +344,7 @@ We provide a [comprehensive analysis](docs/README.md) with other methods in our1 Consider increasing tuning steps to achieve better results, albeit with increased tuning time. Additionally, setting 'use_quant_input' to False or adjusting 'minmax_lr' to 2.0/iters has been observed to occasionally yield improved results.2 Leverage AutoGPTQ to run the model on GPU-+please install auto-gptq https://github.com/AutoGPTQ/ from source first```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound@@ -357,12 +357,15 @@ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)autoround = AutoRound(model, tokenizer, bits=4, group_size=128, scheme="asym")autoround.quantize()--## export to autogptq-# please install auto-gptq https://github.com/AutoGPTQ/-output_dir = "/path/to/quantized_model"+output_dir = "./tmp_autoround"autoround.export(output_dir, target="auto_gptq", use_triton=True)-# then follow auto-gptq to load the model and inference++# then follow auto-gptq or transformers to load the model and inference+model = AutoModelForCausalLM.from_pretrained(output_dir, device_map="auto",+ trust_remote_code=False)+print(tokenizer.decode(+ model.generate(**tokenizer("There is a girl who likes adventure,", return_tensors="pt").to(model.device),+ max_new_tokens=50)[0]))```## Examplesdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 357614ec9..d436dc8e9 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -962,6 +962,8 @@ class AutoRound(object):if not self.quantized:logger.warning("please run autoround.quantize first")return+ if self.tokenizer is not None:+ self.tokenizer.save_pretrained(output_dir)model = copy.deepcopy(self.model.to("cpu")) ##TODO avoid this deepcopyfrom auto_gptq.modeling._utils import pack_model@@ -1068,7 +1070,6 @@ class AutoRound(object):cost_time = end_time - start_timelogger.info(f"quantization tuning time {cost_time}")self.quantized = True- ##self.export_to_autogptq("test_export")return self.model, self.weight_configdiff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.pyindex f7393318c..571048c1c 100644--- a/auto_round/export/export_to_autogptq.py+++ b/auto_round/export/export_to_autogptq.py@@ -42,7 +42,7 @@ import osfrom typing import Dict, List, Optional, Unionfrom safetensors.torch import save_file as safe_savefrom os.path import join, isfile, isdir-import copy+import jsondef save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym=False, iters=200, lr=5e-3,minmax_lr=5e-3,@@ -52,7 +52,7 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, symos.makedirs(save_dir, exist_ok=True)model.to("cpu")- model_base_name = f"autoround-model-{str(bits)}bit-{str(group_size)}g"+ model_base_name = f"model"if use_safetensors:model_save_name = model_base_name + ".safetensors"state_dict = model.state_dict()@@ -89,7 +89,7 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym# Store the quantization configuration as safetensors metadatafrom auto_round import __version__- safetensors_metadata['auto_round_version'] = str(__version__)+ safetensors_metadata['version'] = str(__version__)safetensors_metadata['bits'] = str(bits)safetensors_metadata['group_size'] = str(group_size)safetensors_metadata['iters'] = str(iters)@@ -104,10 +104,24 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, symmodel.config.save_pretrained(save_dir)from auto_gptq.modeling._base import BaseQuantizeConfig+quantization_config = BaseQuantizeConfig(bits=bits, group_size=group_size, desc_act=False, sym=sym,true_sequential=False, static_groups=False,model_file_base_name=model_base_name)quantization_config.model_file_base_name = model_base_name- quantization_config.save_pretrained(save_dir)+ config_dict = quantization_config.to_dict()+ config_dict["quant_method"] = "intel/auto-round"+ config_dict['version'] = __version__+ config_dict['iters'] = iters+ config_dict['lr'] = lr+ config_dict['minmax_lr'] = minmax_lr+ config_dict['enable_minmax_tuning'] = enable_minmax_tuning+ config_dict['use_quant_input'] = use_quant_input++ with open(join(save_dir, "quantize_config.json"), "w", encoding="utf-8") as f:+ json.dump(config_dict, f, indent=2)+++diff --git a/examples/language-modeling/requirements.txt b/examples/language-modeling/requirements.txtindex e319bb19c..09bebc4ec 100644--- a/examples/language-modeling/requirements.txt+++ b/examples/language-modeling/requirements.txt@@ -1,4 +1,4 @@-transformers==4.36.0+transformerstorchgit+https://github.com/EleutherAI/lm-evaluation-harness.git@008fc2a23245c40384f2312718433eeb1e0f87a9tiktokendiff --git a/requirements.txt b/requirements.txtindex 3e64679d4..f7b9766e8 100644--- a/requirements.txt+++ b/requirements.txt@@ -1,4 +1,4 @@-transformers==4.36.0+transformerstorchsentencepieceaccelerate