Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix autoround format mixed precision issue and refine gptq format code (#183)

wenhuach21e28149969e09
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 359af0b85..241196ebd 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -300,20 +300,6 @@ class AutoRound(object):
if hasattr(m, "scale"):
self.layer_config[n]["scale"] = m.scale
self.layer_config[n]["zp"] = m.zp
- if isinstance(m, transformers.modeling_utils.Conv1D):
- weight = m.weight.t()
- else:
- weight = m.weight
-
- if self.group_size <= 0:
-
- self.layer_config[n]["g_idx"] = torch.tensor(
- [0 for i in range(weight.shape[1])], dtype=torch.int32, device="cpu"
- )
- else:
- self.layer_config[n]["g_idx"] = torch.tensor(
- [i // self.group_size for i in range(weight.shape[1])], dtype=torch.int32, device="cpu"
- )
delattr(m, "scale")
delattr(m, "zp")
else:
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index 9231b6e63..7249224c5 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -13,12 +13,6 @@
# limitations under the License.
-import copy
-import json
-import os
-from os.path import isdir, isfile, join
-from typing import Dict, List, Optional, Union
-
# MIT License
#
# Copyright (c) 2023 潘其威(William)
@@ -41,31 +35,76 @@ from typing import Dict, List, Optional, Union
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import torch
-from safetensors.torch import save_file as safe_save
+
+from auto_round.utils import check_to_quantized, get_block_names, get_module, logger, get_layer_names_in_block, \
+ set_module
+import copy
+import json
+import os
+
+import torch.nn as nn
+import transformers
from auto_round.export.register import register_format
-from auto_round.utils import check_to_quantized, get_block_names, get_module, logger, convert_dtype_torch2str_hf
import threadpoolctl as tctl
+import inspect
+
+
+def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):
+ """
+ Configures and returns a QuantLinear class based on the specified backend and parameters.
+
+ Args:
+ backend (str): The backend to be used for quantization. Supported values include "qigen", "triton", "marlin",
+ "exllama", and "cuda".
+ bits (int, optional): The number of bits for quantization. Default is 4.
+ group_size (int, optional): The group size for quantization. Default is 128.
+ sym (bool, optional): Flag indicating whether to use symmetric quantization. Default is False.
+
+ Returns:
+ class: The dynamically imported QuantLinear class configured according to the specified parameters.
+ """
+ use_triton = True
+ disable_exllamav2 = True
+ disable_exllamav1 = False
+ disable_marlin = True
+ use_qigen = False
+ if "qigen" in backend:
+ use_triton = False
+ use_qigen = True
+ elif "marlin" in backend and sym:
+ use_triton = False
+ disable_marlin = False
+ else:
+ ##we all use triton for others, ##TODO may have bugs for some backends
+ from auto_round.export.export_to_autogptq.qlinear_triton import QuantLinear
+ return QuantLinear
+ try:
+ import auto_gptq # pylint: disable=E0401
+ except:
+ logger.warning_once(f"please install auto_gptq via 'pip install auto-gptq' to support exporting to {backend}")
+ exit()
+
+ from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401
+ QuantLinear = dynamically_import_QuantLinear(
+ use_triton=use_triton,
+ desc_act=False,
+ group_size=group_size,
+ bits=bits,
+ disable_exllama=disable_exllamav1,
+ disable_exllamav2=disable_exllamav2,
+ use_qigen=use_qigen,
+ disable_marlin=disable_marlin,
+ )
+ return QuantLinear
@register_format("auto_gptq")
-def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwargs): ##TODO align with autoround format
+def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllamav2",
+ **kwargs):
"""Export the model to autogptq format to easily leverage cuda kernel."""
- try:
- import auto_gptq
- except ImportError:
- raise ImportError("export to autogptq requires autogptq library. Please run 'pip install auto-gptq'")
+
model = kwargs["model"]
- layer_config = kwargs["layer_config"]
- sym = kwargs["sym"]
- bits = kwargs["bits"]
- group_size = kwargs["group_size"]
- iters = kwargs["iters"]
- lr = kwargs["lr"]
- minmax_lr = kwargs["minmax_lr"]
- enable_minmax_tuning = kwargs["enable_minmax_tuning"]
- enable_quanted_input = kwargs["enable_quanted_input"]
- scale_dtype = kwargs["scale_dtype"]
tokenizer = kwargs["tokenizer"]
supported_types = kwargs["supported_types"]
@@ -93,163 +132,102 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwar
if all_to_quantized:
modules_in_block_to_quantize = None
- if inplace:
- compressed_model = model.to("cpu")
- else:
- compressed_model = copy.deepcopy(model.to("cpu"))
+ model = model.to(torch.float16) ##force to fp16
+ if not inplace:
+ model = copy.deepcopy(model.to("cpu"))
- from auto_gptq.modeling._utils import pack_model # pylint: disable=E0401
+ layer_config = kwargs["layer_config"]
- if bits == 3 or use_triton is False:
- if bits == 3 and use_triton is True:
- logger.warning("triton does not support 3 bits, reset it to False")
- use_triton = False
- quantizers = {}
- for key in layer_config:
- if key == "lm_head": ##TODO remove this after pr 87 is merged
- continue
- info = layer_config[key]
- if not check_to_quantized(info):
- continue
- ##force to float32 to be compatible with torch 2.0
- quantizers[key] = (None, info["scale"], info["zp"].to(torch.float32), info["g_idx"])
with tctl.threadpool_limits(limits=1):
- pack_model(
- compressed_model,
- quantizers,
- bits,
- group_size,
- use_cuda_fp16=True,
- desc_act=False,
- force_layer_back_to_cpu=True,
- use_triton=use_triton,
- )
+ for name in layer_config.keys():
+ if name == "lm_head": ##dese not support lm-head
+ continue
+ config = kwargs["layer_config"][name]
+ if config["bits"] > 8:
+ continue
+ logger.info(f"packing {name}")
+
+ bits = config["bits"]
+ group_size = config["group_size"]
+ sym = config["sym"]
+
+ layer = get_module(model, name)
+ device = layer.weight.device
+
+ QuantLinear = get_autogptq_packing_qlinear(backend, bits, group_size, sym)
+
+ if isinstance(layer, nn.Linear):
+ in_features = layer.in_features
+ out_features = layer.out_features
+ elif isinstance(layer, nn.Conv2d):
+ in_features = layer.in_channels
+ out_features = layer.out_channels
+ elif isinstance(layer, transformers.pytorch_utils.Conv1D):
+ in_features = layer.weight.shape[0]
+ out_features = layer.weight.shape[1]
+
+ ##bias = layer.bias is not None and torch.any(layer.bias)
+ bias = True ## if using the above, llama3 lambada RTN will be NAN , TODO why?
+ new_layer = QuantLinear( ##pylint: disable=E1123
+ bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype
+ )
+
+ new_layer.device = device
+ set_module(model, name, new_layer)
+ qlayer = new_layer
+ scale = layer_config[name]["scale"]
+ zero = layer_config[name]["zp"]
+ # so far can only pack layer on CPU
+ qlayer.to("cpu")
+ ##force to float32 to be compatible with torch 2.0
+ layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)
+ sig = inspect.signature(qlayer.pack)
+ param_count = len(sig.parameters)
+ if param_count == 2:
+ qlayer.pack(layer, scale)
+ else:
+ qlayer.pack(layer, scale, zero, None)
+ qlayer.to(device)
if output_dir is None:
- return compressed_model
-
- _save_quantized_to_autogptq(
- compressed_model,
- output_dir,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=iters,
- lr=lr,
- minmax_lr=minmax_lr,
- enable_minmax_tuning=enable_minmax_tuning,
- enable_quanted_input=enable_quanted_input,
- scale_dtype=scale_dtype,
- use_safetensors=True,
- modules_in_block_to_quantize=modules_in_block_to_quantize,
- )
-
-
-def _save_quantized_to_autogptq(
- model,
- save_dir: str,
- bits=4,
- group_size=128,
- sym=False,
- iters=200,
- lr=5e-3,
- minmax_lr=5e-3,
- enable_minmax_tuning=True,
- enable_quanted_input=True,
- use_safetensors: bool = True,
- scale_dtype=torch.float32,
- safetensors_metadata: Optional[Dict[str, str]] = None,
- modules_in_block_to_quantize=None,
-):
- """Save quantized model and configs to local disk for cuda."""
- os.makedirs(save_dir, exist_ok=True)
- model.to("cpu")
-
- model_base_name = "model"
- if use_safetensors:
- model_save_name = model_base_name + ".safetensors"
- state_dict = model.state_dict()
- state_dict = {k: v.clone().contiguous() for k, v in state_dict.items()}
- if safetensors_metadata is None:
- safetensors_metadata = {}
- elif not isinstance(safetensors_metadata, dict):
- raise TypeError("safetensors_metadata must be a dictionary.")
- else:
- logger.debug(f"Received safetensors_metadata: {safetensors_metadata}")
- new_safetensors_metadata = {}
- converted_keys = False
- for key, value in safetensors_metadata.items():
- if not isinstance(key, str) or not isinstance(value, str):
- converted_keys = True
- try:
- new_key = str(key)
- new_value = str(value)
- except Exception as e:
- raise TypeError(
- "safetensors_metadata: both keys and values must be strings"
- / f" and an error occurred when trying to convert them: {e}"
- )
- if new_key in new_safetensors_metadata:
- logger.warning(
- f"After converting safetensors_metadata keys to strings, the key '{new_key}' "
- / "is duplicated. Ensure that all your metadata keys are strings to avoid overwriting."
- )
- new_safetensors_metadata[new_key] = new_value
- safetensors_metadata = new_safetensors_metadata
- if converted_keys:
- logger.debug(
- "One or more safetensors_metadata keys or values had to be converted to str()."
- / f" Final safetensors_metadata: {safetensors_metadata}"
- )
-
- # Format is required to enable Accelerate to load the metadata
- # otherwise it raises an OSError
- safetensors_metadata["format"] = "pt"
-
- # Store the quantization configuration as safetensors metadata
- from auto_round import __version__
-
- safetensors_metadata["autoround_version"] = str(__version__)
- safetensors_metadata["bits"] = str(bits)
- safetensors_metadata["group_size"] = str(group_size)
- safetensors_metadata["iters"] = str(iters)
- safetensors_metadata["lr"] = str(lr)
- safetensors_metadata["minmax_lr"] = str(minmax_lr)
- safetensors_metadata["enable_minmax_tuning"] = str(enable_minmax_tuning)
- safetensors_metadata["enable_quanted_input"] = str(enable_quanted_input)
- safetensors_metadata["scale_dtype"] = convert_dtype_torch2str_hf(scale_dtype)
- safe_save(state_dict, join(save_dir, model_save_name), safetensors_metadata)
- else:
- model_save_name = model_base_name + ".bin"
- torch.save(model.state_dict(), join(save_dir, model_save_name))
-
- from auto_gptq.modeling._base import BaseQuantizeConfig # pylint: disable=E0401
-
- quantization_config = BaseQuantizeConfig(
- bits=bits,
- group_size=group_size,
- desc_act=False,
- sym=sym,
- true_sequential=False,
- static_groups=False,
- model_file_base_name=model_base_name,
- )
- quantization_config.model_file_base_name = model_base_name
-
- config_dict = quantization_config.to_dict()
- config_dict["quant_method"] = "gptq"
- config_dict["autoround_version"] = __version__
- config_dict["iters"] = iters
- config_dict["lr"] = lr
- config_dict["minmax_lr"] = minmax_lr
- config_dict["enable_minmax_tuning"] = enable_minmax_tuning
- config_dict["enable_quanted_input"] = enable_quanted_input
- config_dict["scale_dtype"] = convert_dtype_torch2str_hf(scale_dtype)
+ return model
+ quantization_config = kwargs["serialization_dict"]
+ quantization_config["quant_method"] = "gptq"
+ quantization_config.pop("dataset", None) ## pile-10k is not supported in gptq
+ quantization_config["desc_act"] = False ## for autogptq API
+ quantization_config["true_sequential"] = False
+ quantization_config["damp_percent"] = 0.01
if modules_in_block_to_quantize is not None:
- config_dict["modules_in_block_to_quantize"] = modules_in_block_to_quantize
-
- with open(join(save_dir, "quantize_config.json"), "w", encoding="utf-8") as f:
- json.dump(config_dict, f, indent=2)
-
- config_dict["quant_method"] = "gptq" ##hf transformers could only recognize this value
- model.config.quantization_config = config_dict
- model.config.save_pretrained(save_dir)
+ quantization_config["modules_in_block_to_quantize"] = modules_in_block_to_quantize
+ if hasattr(model, "config"):
+ model.config.quantization_config = quantization_config
+ save(model, output_dir)
+
+
+##
+def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):
+ """Save model state dict and configs.
+
+ Args:
+ model (`nn.Module`):
+ Model to be saved. The model can be wrapped or unwrapped.
+ save_dir (`str`):
+ Directory to which to save. Will be created if it doesn't exist.
+ max_shard_size (`str`, defaults to `"10GB"`):
+ The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size
+ lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).
+ <Tip warning={true}>
+
+ If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard
+ which will be bigger than `max_shard_size`.
+
+ </Tip>
+ safe_serialization (`bool`, defaults to `True`):
+ Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ """
+ max_shard_size = "10000GB" ## API of auto-gptq with marlin does not support shard size
+ os.makedirs(save_dir, exist_ok=True)
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ config_file = "quantize_config.json"
+ if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
+ with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
+ json.dump(model.config.quantization_config, f, indent=2)
diff --git a/auto_round/export/export_to_autogptq/qlinear_triton.py b/auto_round/export/export_to_autogptq/qlinear_triton.py
new file mode 100644
index 000000000..ea34c881a
--- /dev/null
+++ b/auto_round/export/export_to_autogptq/qlinear_triton.py
@@ -0,0 +1,142 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+import transformers
+
+class TritonModuleMixin:
+ @classmethod
+ def warmup(cls, model, transpose=False, seqlen=2048):
+ pass
+
+
+
+class QuantLinear(nn.Module, TritonModuleMixin):
+ QUANT_TYPE = "triton"
+
+ def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):
+ super().__init__()
+ if bits not in [2, 4, 8]:
+ raise NotImplementedError("Only 2,4,8 bits are supported.")
+ if infeatures % 32 != 0 or outfeatures % 32 != 0:
+ raise NotImplementedError("in_feature and out_feature must be divisible by 32.")
+ self.infeatures = infeatures
+ self.outfeatures = outfeatures
+ self.bits = bits
+ self.group_size = group_size if group_size != -1 else infeatures
+ self.maxq = 2**self.bits - 1
+
+ self.register_buffer(
+ "qweight",
+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
+ )
+ self.register_buffer(
+ "qzeros",
+ torch.zeros(
+ (
+ math.ceil(infeatures / self.group_size),
+ outfeatures // 32 * self.bits,
+ ),
+ dtype=torch.int32,
+ ),
+ )
+ self.register_buffer(
+ "scales",
+ torch.zeros(
+ (math.ceil(infeatures / self.group_size), outfeatures),
+ dtype=torch.float16,
+ ),
+ )
+ self.register_buffer(
+ "g_idx",
+ torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),
+ )
+ if bias:
+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))
+ else:
+ self.bias = None
+
+ self.trainable = trainable
+
+ def post_init(self):
+ pass
+
+ def pack(self, linear, scales, zeros, g_idx=None):
+ W = linear.weight.data.clone()
+ if isinstance(linear, nn.Conv2d):
+ W = W.flatten(1)
+ if isinstance(linear, transformers.pytorch_utils.Conv1D):
+ W = W.t()
+
+ self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
+
+ scales = scales.t().contiguous()
+ zeros = zeros.t().contiguous()
+ scale_zeros = zeros * scales
+ self.scales = scales.clone().half()
+ if linear.bias is not None:
+ self.bias = linear.bias.clone().half()
+
+ intweight = []
+ for idx in range(self.infeatures):
+ intweight.append(
+ torch.round((W[:, idx] + scale_zeros[self.g_idx[idx]]) / self.scales[self.g_idx[idx]]).to(torch.int)[
+ :, None
+ ]
+ )
+ intweight = torch.cat(intweight, dim=1)
+ intweight = intweight.t().contiguous()
+ intweight = intweight.numpy().astype(np.uint32)
+
+ i = 0
+ row = 0
+ qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)
+ while row < qweight.shape[0]:
+ if self.bits in [2, 4, 8]:
+ for j in range(i, i + (32 // self.bits)):
+ qweight[row] |= intweight[j] << (self.bits * (j - i))
+ i += 32 // self.bits
+ row += 1
+ else:
+ raise NotImplementedError("Only 2,4,8 bits are supported.")
+
+ qweight = qweight.astype(np.int32)
+ self.qweight = torch.from_numpy(qweight)
+
+ zeros -= 1
+ zeros = zeros.numpy().astype(np.uint32)
+ qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)
+ i = 0
+ col = 0
+ while col < qzeros.shape[1]:
+ if self.bits in [2, 4, 8]:
+ for j in range(i, i + (32 // self.bits)):
+ qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
+ i += 32 // self.bits
+ col += 1
+ else:
+ raise NotImplementedError("Only 2,4,8 bits are supported.")
+
+ qzeros = qzeros.astype(np.int32)
+ self.qzeros = torch.from_numpy(qzeros)
+
+
+
+
+
+__all__ = ["QuantLinear"]
\ No newline at end of file
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index cbb21e6da..c6c33e117 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -44,7 +44,6 @@ def check_neq_config(config, data_type, bits, group_size, sym):
res = []
if data_type != config["data_type"]:
res.append("data_type")
- return res
if bits != config["bits"]:
res.append("bits")
if group_size != config["group_size"]:
@@ -172,7 +171,31 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
layer_names_in_block = get_layer_names_in_block(model)
layer_config = kwargs["layer_config"]
-
+ quantization_config = kwargs["serialization_dict"]
+ quantization_config["quant_method"] = "intel/auto-round"
+ quantization_config["backend"] = backend
+ extra_config = {}
+ for layer_name in layer_config:
+ if layer_name not in layer_names_in_block and layer_config[layer_name]["bits"] <= 8: ##lm head
+ extra_config[layer_name] = {}
+ extra_config[layer_name]["bits"] = layer_config[layer_name]["bits"]
+ extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]
+ extra_config[layer_name]["group_size"] = layer_config[layer_name]["group_size"]
+ extra_config[layer_name]["sym"] = layer_config[layer_name]["sym"]
+ elif layer_name in layer_names_in_block:
+ neq_keys = check_neq_config(
+ layer_config[layer_name],
+ data_type=quantization_config["data_type"],
+ bits=quantization_config["bits"],
+ group_size=quantization_config["group_size"],
+ sym=quantization_config["sym"],
+ )
+ if len(neq_keys) > 0:
+ extra_config[layer_name] = {}
+ for key in neq_keys:
+ extra_config[layer_name][key] = layer_config[layer_name][key]
+ if len(extra_config) > 0:
+ quantization_config["extra_config"] = extra_config
with tctl.threadpool_limits(limits=1):
for name in layer_config.keys():
@@ -221,33 +244,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
else:
qlayer.pack(layer, scale, zero, None)
qlayer.to(device)
- quantization_config = kwargs["serialization_dict"]
- quantization_config["quant_method"] = "intel/auto-round"
- quantization_config["backend"] = backend
- extra_config = {}
- for layer_name in layer_config:
- if layer_config[layer_name]["bits"] > 8:
- continue
- if layer_name not in layer_names_in_block:
- extra_config[layer_name] = {}
- extra_config[layer_name]["bits"] = layer_config[layer_name]["bits"]
- extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]
- extra_config[layer_name]["group_size"] = layer_config[layer_name]["group_size"]
- extra_config[layer_name]["sym"] = layer_config[layer_name]["sym"]
- else:
- neq_keys = check_neq_config(
- layer_config[layer_name],
- data_type=quantization_config["data_type"],
- bits=quantization_config["bits"],
- group_size=quantization_config["group_size"],
- sym=quantization_config["sym"],
- )
- if len(neq_keys) > 0:
- extra_config[layer_name] = {}
- for key in neq_keys:
- extra_config[layer_name][key] = layer_config[layer_name][key]
- if len(extra_config) > 0:
- quantization_config["extra_config"] = extra_config
+
if hasattr(model, "config"):
model.config.quantization_config = quantization_config
tokenizer = kwargs["tokenizer"]
diff --git a/requirements.txt b/requirements.txt
index 66cbf2523..83ec1fe4e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,5 +6,4 @@ torch
transformers
triton
numpy < 2.0
-threadpoolctl
-auto-gptq
\ No newline at end of file
+threadpoolctl
\ No newline at end of file