Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

export to autogptq

wenhuach21e20dcc182cba
diff --git a/README.md b/README.md
index 51014e270..bac07de97 100644
--- a/README.md
+++ b/README.md
@@ -83,6 +83,10 @@ LLaMAV1 done
LLaMAv2 done
+PI2 done
+
+mixstral-7Bx8 done
+
LaMini-GPT-124M done
QWEN1-8B done,but has random issue
@@ -105,7 +109,7 @@ gpt-j-6b smoke test done
chatglm2-6b smoke test done
-mixstral-7Bx8 smoke test done
+
## Reference
If you find SignRound useful for your research, please cite our paper:
diff --git a/auto_round/__init__.py b/auto_round/__init__.py
index b3aec764e..1b2514237 100644
--- a/auto_round/__init__.py
+++ b/auto_round/__init__.py
@@ -12,3 +12,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from .autoround import AutoRound, AutoAdamRound, AutoOPTRound
+from .version import __version__
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 24e9b0c0e..8d314bde7 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -146,6 +146,9 @@ def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scal
weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale
)
weight = weight.reshape(orig_shape)
+ scale = scale.reshape(weight.shape[0], -1) ##only for linear, conv1d
+ if zp is not None:
+ zp = zp.reshape(weight.shape[0], -1)
return weight, scale, zp
else:
@@ -161,6 +164,9 @@ def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scal
weight_new = weight_new.reshape(orig_shape[0], -1)
weight_new = weight_new[:, :-pad_len]
+ scale = scale.reshape(weight_new.shape[0], -1) ##only for linear, conv1d
+ if zp is not None:
+ zp = zp.reshape(weight_new.shape[0], -1)
return weight_new, scale, zp
@@ -499,7 +505,7 @@ class WrapperTransformerConv1d(torch.nn.Module):
self.max_scale = torch.tensor(0, device=device)
def unwrapper(self, v, min_scale, max_scale):
- """Unwrapper the layer to the original conv1d layer..
+ """Unwrapper the layer to the original conv1d layer.
Args:
- v (torch.Tensor): The scaling parameter for quantization.
@@ -1355,6 +1361,31 @@ class AutoRound(object):
torch.cuda.empty_cache()
+ def export_to_autogptq(self, output_dir):
+ """
+ Export the model to autogptq format to easily leverage cuda kernel
+ """
+ model = copy.deepcopy(self.model.to("cpu")) ##TODO avoid this deepcopy
+ quantizers = {}
+ for key in self.weight_config:
+ info = self.weight_config[key]
+ if info["bits"] > 8:
+ continue
+ quantizers[key] = (None, info['scale'], info['zp'], info['g_idx'])
+ from auto_gptq.modeling._utils import pack_model
+ if self.bits == 3:
+ pack_model(model, quantizers, self.bits, self.group_size, use_cuda_fp16=True, desc_act=False,
+ force_layer_back_to_cpu=True)
+ else:
+ pack_model(model, quantizers, self.bits, self.group_size, use_cuda_fp16=True, desc_act=False,
+ force_layer_back_to_cpu=True, use_triton=True)
+ from auto_round.export import save_quantized_to_autogptq
+ sym = self.scheme == "sym"
+ save_quantized_to_autogptq(model, output_dir, bits=self.bits, group_size=self.group_size, sym=sym,
+ iters=self.iters, lr=self.lr, minmax_lr=self.minmax_lr,
+ enable_minmax_tuning=self.enable_minmax_tuning, use_quant_input=self.use_quant_input,
+ use_safetensors=True)
+
def quantize(self):
"""Quantize the model and return the quantized model along with weight configurations.
@@ -1395,6 +1426,8 @@ class AutoRound(object):
if hasattr(m, "scale"):
self.weight_config[n]["scale"] = m.scale
self.weight_config[n]["zp"] = m.zp
+ self.weight_config[n]["g_idx"] = torch.tensor(
+ [i // self.group_size for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu")
delattr(m, "scale")
delattr(m, "zp")
else:
@@ -1407,7 +1440,8 @@ class AutoRound(object):
end_time = time.time()
cost_time = end_time - start_time
- logger.info(f"quantization runtime {cost_time}")
+ logger.info(f"quantization tuning time {cost_time}")
+ self.export_to_autogptq("test_export")
return self.model, self.weight_config
diff --git a/auto_round/export.py b/auto_round/export.py
new file mode 100644
index 000000000..f73948b61
--- /dev/null
+++ b/auto_round/export.py
@@ -0,0 +1,110 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# MIT License
+#
+# Copyright (c) 2023 潘其威(William)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import torch
+from logging import getLogger
+
+logger = getLogger(__name__)
+import os
+from typing import Dict, List, Optional, Union
+from safetensors.torch import save_file as safe_save
+from os.path import join, isfile, isdir
+
+
+def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym=False, iters=200, lr=5e-3, minmax_lr=5e-3,
+ enable_minmax_tuning=True, use_quant_input=True, use_safetensors: bool = True,
+ safetensors_metadata: Optional[Dict[str, str]] = None):
+ """save quantized model and configs to local disk"""
+ os.makedirs(save_dir, exist_ok=True)
+ model.to("cpu")
+
+ model_base_name = f"autoround-model-{str(bits)}bit-{str(group_size)}g"
+ if use_safetensors:
+ model_save_name = model_base_name + ".safetensors"
+ state_dict = model.state_dict()
+ state_dict = {k: v.clone().contiguous() for k, v in state_dict.items()}
+ if safetensors_metadata is None:
+ safetensors_metadata = {}
+ elif not isinstance(safetensors_metadata, dict):
+ raise TypeError("safetensors_metadata must be a dictionary.")
+ else:
+ logger.debug(f"Received safetensors_metadata: {safetensors_metadata}")
+ new_safetensors_metadata = {}
+ converted_keys = False
+ for key, value in safetensors_metadata.items():
+ if not isinstance(key, str) or not isinstance(value, str):
+ converted_keys = True
+ try:
+ new_key = str(key)
+ new_value = str(value)
+ except Exception as e:
+ raise TypeError(
+ f"safetensors_metadata: both keys and values must be strings and an error occured when trying to convert them: {e}")
+ if new_key in new_safetensors_metadata:
+ logger.warning(
+ f"After converting safetensors_metadata keys to strings, the key '{new_key}' is duplicated. Ensure that all your metadata keys are strings to avoid overwriting.")
+ new_safetensors_metadata[new_key] = new_value
+ safetensors_metadata = new_safetensors_metadata
+ if converted_keys:
+ logger.debug(
+ f"One or more safetensors_metadata keys or values had to be converted to str(). Final safetensors_metadata: {safetensors_metadata}")
+
+ # Format is required to enable Accelerate to load the metadata
+ # otherwise it raises an OSError
+ safetensors_metadata['format'] = "pt"
+
+ # Store the quantization configuration as safetensors metadata
+ from auto_round import __version__
+ safetensors_metadata['auto_round_version'] = str(__version__)
+ safetensors_metadata['bits'] = str(bits)
+ safetensors_metadata['group_size'] = str(group_size)
+ safetensors_metadata['iters'] = str(iters)
+ safetensors_metadata['lr'] = str(lr)
+ safetensors_metadata['minmax_lr'] = str(minmax_lr)
+ safetensors_metadata['enable_minmax_tuning'] = str(enable_minmax_tuning)
+ safetensors_metadata['use_quant_input'] = str(use_quant_input)
+ safe_save(state_dict, join(save_dir, model_save_name), safetensors_metadata)
+ else:
+ model_save_name = model_base_name + ".bin"
+ torch.save(model.state_dict(), join(save_dir, model_save_name))
+
+ model.config.save_pretrained(save_dir)
+ from auto_gptq.modeling._base import BaseQuantizeConfig
+ quantization_config = BaseQuantizeConfig(bits=bits,group_size=group_size, desc_act=False, sym=sym, true_sequential=False, static_groups=False,
+ model_file_base_name=model_base_name)
+ quantization_config.model_file_base_name = model_base_name
+
+ quantization_config.save_pretrained(save_dir)
diff --git a/examples/eval/evaluation.py b/examples/eval/evaluation.py
index c60d15914..443423844 100644
--- a/examples/eval/evaluation.py
+++ b/examples/eval/evaluation.py
@@ -346,7 +346,7 @@ if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
- "--model_name", default="/models/opt-125m"
+ "--model_name", default="/models/opt-125m/"
)
parser.add_argument(
"--eval_bs", default=1,
@@ -358,6 +358,7 @@ if __name__ == "__main__":
test_tasks = ['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
"hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte", "arc_easy",
"arc_challenge"]
+ model_name = args.model_name.rstrip('/')
excel_name = (args.model_name).split('/')[-1] + ".xlsx"
eval_model(output_dir=args.model_name,
tasks=test_tasks,
diff --git a/examples/eval/utils.py b/examples/eval/utils.py
index 269b8e676..783010ea3 100644
--- a/examples/eval/utils.py
+++ b/examples/eval/utils.py
@@ -3,6 +3,7 @@ import numpy as np
import torch
import torch.nn as nn
+import datasets
def set_seed(seed):
np.random.seed(seed)
torch.random.manual_seed(seed)
@@ -50,13 +51,21 @@ def get_ptb(nsamples, seed, seqlen, tokenizer):
def get_c4(nsamples, seed, seqlen, tokenizer):
from datasets import load_dataset
- traindata = load_dataset(
- 'allenai/c4', 'allenai--c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'
- )
- valdata = load_dataset(
- 'allenai/c4', 'allenai--c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'
- )
-
+ from packaging import version
+ if version.parse(datasets.__version__)>=version.parse("2.16.1"):
+ traindata = load_dataset(
+ 'allenai/c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'
+ )
+ valdata = load_dataset(
+ 'allenai/c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'
+ )
+ else:
+ traindata = load_dataset(
+ 'allenai/c4', 'allenai--c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'
+ )
+ valdata = load_dataset(
+ 'allenai/c4', 'allenai--c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'
+ )
import random
random.seed(seed)
@@ -115,13 +124,23 @@ def get_ptb_new(nsamples, seed, seqlen, tokenizer):
return trainloader, testenc
def get_c4_new(nsamples, seed, seqlen, tokenizer):
+
from datasets import load_dataset
- traindata = load_dataset(
- 'allenai/c4', 'allenai--c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'
- )
- valdata = load_dataset(
- 'allenai/c4', 'allenai--c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'
- )
+ from packaging import version
+ if version.parse(datasets.__version__)>=version.parse("2.16.1"):
+ traindata = load_dataset(
+ 'allenai/c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'
+ )
+ valdata = load_dataset(
+ 'allenai/c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'
+ )
+ else:
+ traindata = load_dataset(
+ 'allenai/c4', 'allenai--c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'
+ )
+ valdata = load_dataset(
+ 'allenai/c4', 'allenai--c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'
+ )
import random
random.seed(seed)