PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
export to autogptq
wenhuach21e20dcc182cba
diff --git a/README.md b/README.mdindex 51014e270..bac07de97 100644--- a/README.md+++ b/README.md@@ -83,6 +83,10 @@ LLaMAV1 doneLLaMAv2 done+PI2 done++mixstral-7Bx8 done+LaMini-GPT-124M doneQWEN1-8B done,but has random issue@@ -105,7 +109,7 @@ gpt-j-6b smoke test donechatglm2-6b smoke test done-mixstral-7Bx8 smoke test done+## ReferenceIf you find SignRound useful for your research, please cite our paper:diff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex b3aec764e..1b2514237 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -12,3 +12,4 @@# See the License for the specific language governing permissions and# limitations under the License.from .autoround import AutoRound, AutoAdamRound, AutoOPTRound+from .version import __version__diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 24e9b0c0e..8d314bde7 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -146,6 +146,9 @@ def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scalweight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale)weight = weight.reshape(orig_shape)+ scale = scale.reshape(weight.shape[0], -1) ##only for linear, conv1d+ if zp is not None:+ zp = zp.reshape(weight.shape[0], -1)return weight, scale, zpelse:@@ -161,6 +164,9 @@ def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scalweight_new = weight_new.reshape(orig_shape[0], -1)weight_new = weight_new[:, :-pad_len]+ scale = scale.reshape(weight_new.shape[0], -1) ##only for linear, conv1d+ if zp is not None:+ zp = zp.reshape(weight_new.shape[0], -1)return weight_new, scale, zp@@ -499,7 +505,7 @@ class WrapperTransformerConv1d(torch.nn.Module):self.max_scale = torch.tensor(0, device=device)def unwrapper(self, v, min_scale, max_scale):- """Unwrapper the layer to the original conv1d layer..+ """Unwrapper the layer to the original conv1d layer.Args:- v (torch.Tensor): The scaling parameter for quantization.@@ -1355,6 +1361,31 @@ class AutoRound(object):torch.cuda.empty_cache()+ def export_to_autogptq(self, output_dir):+ """+ Export the model to autogptq format to easily leverage cuda kernel+ """+ model = copy.deepcopy(self.model.to("cpu")) ##TODO avoid this deepcopy+ quantizers = {}+ for key in self.weight_config:+ info = self.weight_config[key]+ if info["bits"] > 8:+ continue+ quantizers[key] = (None, info['scale'], info['zp'], info['g_idx'])+ from auto_gptq.modeling._utils import pack_model+ if self.bits == 3:+ pack_model(model, quantizers, self.bits, self.group_size, use_cuda_fp16=True, desc_act=False,+ force_layer_back_to_cpu=True)+ else:+ pack_model(model, quantizers, self.bits, self.group_size, use_cuda_fp16=True, desc_act=False,+ force_layer_back_to_cpu=True, use_triton=True)+ from auto_round.export import save_quantized_to_autogptq+ sym = self.scheme == "sym"+ save_quantized_to_autogptq(model, output_dir, bits=self.bits, group_size=self.group_size, sym=sym,+ iters=self.iters, lr=self.lr, minmax_lr=self.minmax_lr,+ enable_minmax_tuning=self.enable_minmax_tuning, use_quant_input=self.use_quant_input,+ use_safetensors=True)+def quantize(self):"""Quantize the model and return the quantized model along with weight configurations.@@ -1395,6 +1426,8 @@ class AutoRound(object):if hasattr(m, "scale"):self.weight_config[n]["scale"] = m.scaleself.weight_config[n]["zp"] = m.zp+ self.weight_config[n]["g_idx"] = torch.tensor(+ [i // self.group_size for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu")delattr(m, "scale")delattr(m, "zp")else:@@ -1407,7 +1440,8 @@ class AutoRound(object):end_time = time.time()cost_time = end_time - start_time- logger.info(f"quantization runtime {cost_time}")+ logger.info(f"quantization tuning time {cost_time}")+ self.export_to_autogptq("test_export")return self.model, self.weight_configdiff --git a/auto_round/export.py b/auto_round/export.pynew file mode 100644index 000000000..f73948b61--- /dev/null+++ b/auto_round/export.py@@ -0,0 +1,110 @@+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++# MIT License+#+# Copyright (c) 2023 潘其威(William)+#+# Permission is hereby granted, free of charge, to any person obtaining a copy+# of this software and associated documentation files (the "Software"), to deal+# in the Software without restriction, including without limitation the rights+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+# copies of the Software, and to permit persons to whom the Software is+# furnished to do so, subject to the following conditions:+#+# The above copyright notice and this permission notice shall be included in all+# copies or substantial portions of the Software.+#+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+# SOFTWARE.+import torch+from logging import getLogger++logger = getLogger(__name__)+import os+from typing import Dict, List, Optional, Union+from safetensors.torch import save_file as safe_save+from os.path import join, isfile, isdir+++def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym=False, iters=200, lr=5e-3, minmax_lr=5e-3,+ enable_minmax_tuning=True, use_quant_input=True, use_safetensors: bool = True,+ safetensors_metadata: Optional[Dict[str, str]] = None):+ """save quantized model and configs to local disk"""+ os.makedirs(save_dir, exist_ok=True)+ model.to("cpu")++ model_base_name = f"autoround-model-{str(bits)}bit-{str(group_size)}g"+ if use_safetensors:+ model_save_name = model_base_name + ".safetensors"+ state_dict = model.state_dict()+ state_dict = {k: v.clone().contiguous() for k, v in state_dict.items()}+ if safetensors_metadata is None:+ safetensors_metadata = {}+ elif not isinstance(safetensors_metadata, dict):+ raise TypeError("safetensors_metadata must be a dictionary.")+ else:+ logger.debug(f"Received safetensors_metadata: {safetensors_metadata}")+ new_safetensors_metadata = {}+ converted_keys = False+ for key, value in safetensors_metadata.items():+ if not isinstance(key, str) or not isinstance(value, str):+ converted_keys = True+ try:+ new_key = str(key)+ new_value = str(value)+ except Exception as e:+ raise TypeError(+ f"safetensors_metadata: both keys and values must be strings and an error occured when trying to convert them: {e}")+ if new_key in new_safetensors_metadata:+ logger.warning(+ f"After converting safetensors_metadata keys to strings, the key '{new_key}' is duplicated. Ensure that all your metadata keys are strings to avoid overwriting.")+ new_safetensors_metadata[new_key] = new_value+ safetensors_metadata = new_safetensors_metadata+ if converted_keys:+ logger.debug(+ f"One or more safetensors_metadata keys or values had to be converted to str(). Final safetensors_metadata: {safetensors_metadata}")++ # Format is required to enable Accelerate to load the metadata+ # otherwise it raises an OSError+ safetensors_metadata['format'] = "pt"++ # Store the quantization configuration as safetensors metadata+ from auto_round import __version__+ safetensors_metadata['auto_round_version'] = str(__version__)+ safetensors_metadata['bits'] = str(bits)+ safetensors_metadata['group_size'] = str(group_size)+ safetensors_metadata['iters'] = str(iters)+ safetensors_metadata['lr'] = str(lr)+ safetensors_metadata['minmax_lr'] = str(minmax_lr)+ safetensors_metadata['enable_minmax_tuning'] = str(enable_minmax_tuning)+ safetensors_metadata['use_quant_input'] = str(use_quant_input)+ safe_save(state_dict, join(save_dir, model_save_name), safetensors_metadata)+ else:+ model_save_name = model_base_name + ".bin"+ torch.save(model.state_dict(), join(save_dir, model_save_name))++ model.config.save_pretrained(save_dir)+ from auto_gptq.modeling._base import BaseQuantizeConfig+ quantization_config = BaseQuantizeConfig(bits=bits,group_size=group_size, desc_act=False, sym=sym, true_sequential=False, static_groups=False,+ model_file_base_name=model_base_name)+ quantization_config.model_file_base_name = model_base_name++ quantization_config.save_pretrained(save_dir)diff --git a/examples/eval/evaluation.py b/examples/eval/evaluation.pyindex c60d15914..443423844 100644--- a/examples/eval/evaluation.py+++ b/examples/eval/evaluation.py@@ -346,7 +346,7 @@ if __name__ == "__main__":import argparseparser = argparse.ArgumentParser()parser.add_argument(- "--model_name", default="/models/opt-125m"+ "--model_name", default="/models/opt-125m/")parser.add_argument("--eval_bs", default=1,@@ -358,6 +358,7 @@ if __name__ == "__main__":test_tasks = ['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',"hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte", "arc_easy","arc_challenge"]+ model_name = args.model_name.rstrip('/')excel_name = (args.model_name).split('/')[-1] + ".xlsx"eval_model(output_dir=args.model_name,tasks=test_tasks,diff --git a/examples/eval/utils.py b/examples/eval/utils.pyindex 269b8e676..783010ea3 100644--- a/examples/eval/utils.py+++ b/examples/eval/utils.py@@ -3,6 +3,7 @@ import numpy as npimport torchimport torch.nn as nn+import datasetsdef set_seed(seed):np.random.seed(seed)torch.random.manual_seed(seed)@@ -50,13 +51,21 @@ def get_ptb(nsamples, seed, seqlen, tokenizer):def get_c4(nsamples, seed, seqlen, tokenizer):from datasets import load_dataset- traindata = load_dataset(- 'allenai/c4', 'allenai--c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'- )- valdata = load_dataset(- 'allenai/c4', 'allenai--c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'- )-+ from packaging import version+ if version.parse(datasets.__version__)>=version.parse("2.16.1"):+ traindata = load_dataset(+ 'allenai/c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'+ )+ valdata = load_dataset(+ 'allenai/c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'+ )+ else:+ traindata = load_dataset(+ 'allenai/c4', 'allenai--c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'+ )+ valdata = load_dataset(+ 'allenai/c4', 'allenai--c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'+ )import randomrandom.seed(seed)@@ -115,13 +124,23 @@ def get_ptb_new(nsamples, seed, seqlen, tokenizer):return trainloader, testencdef get_c4_new(nsamples, seed, seqlen, tokenizer):+from datasets import load_dataset- traindata = load_dataset(- 'allenai/c4', 'allenai--c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'- )- valdata = load_dataset(- 'allenai/c4', 'allenai--c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'- )+ from packaging import version+ if version.parse(datasets.__version__)>=version.parse("2.16.1"):+ traindata = load_dataset(+ 'allenai/c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'+ )+ valdata = load_dataset(+ 'allenai/c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'+ )+ else:+ traindata = load_dataset(+ 'allenai/c4', 'allenai--c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'+ )+ valdata = load_dataset(+ 'allenai/c4', 'allenai--c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'+ )import randomrandom.seed(seed)