PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix autoround format mixed precision issue and refine gptq format code (#183)
wenhuach21e28149969e09
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 359af0b85..241196ebd 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -300,20 +300,6 @@ class AutoRound(object):if hasattr(m, "scale"):self.layer_config[n]["scale"] = m.scaleself.layer_config[n]["zp"] = m.zp- if isinstance(m, transformers.modeling_utils.Conv1D):- weight = m.weight.t()- else:- weight = m.weight-- if self.group_size <= 0:-- self.layer_config[n]["g_idx"] = torch.tensor(- [0 for i in range(weight.shape[1])], dtype=torch.int32, device="cpu"- )- else:- self.layer_config[n]["g_idx"] = torch.tensor(- [i // self.group_size for i in range(weight.shape[1])], dtype=torch.int32, device="cpu"- )delattr(m, "scale")delattr(m, "zp")else:diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 9231b6e63..7249224c5 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -13,12 +13,6 @@# limitations under the License.-import copy-import json-import os-from os.path import isdir, isfile, join-from typing import Dict, List, Optional, Union-# MIT License## Copyright (c) 2023 潘其威(William)@@ -41,31 +35,76 @@ from typing import Dict, List, Optional, Union# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE# SOFTWARE.import torch-from safetensors.torch import save_file as safe_save++from auto_round.utils import check_to_quantized, get_block_names, get_module, logger, get_layer_names_in_block, \+ set_module+import copy+import json+import os++import torch.nn as nn+import transformersfrom auto_round.export.register import register_format-from auto_round.utils import check_to_quantized, get_block_names, get_module, logger, convert_dtype_torch2str_hfimport threadpoolctl as tctl+import inspect+++def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):+ """+ Configures and returns a QuantLinear class based on the specified backend and parameters.++ Args:+ backend (str): The backend to be used for quantization. Supported values include "qigen", "triton", "marlin",+ "exllama", and "cuda".+ bits (int, optional): The number of bits for quantization. Default is 4.+ group_size (int, optional): The group size for quantization. Default is 128.+ sym (bool, optional): Flag indicating whether to use symmetric quantization. Default is False.++ Returns:+ class: The dynamically imported QuantLinear class configured according to the specified parameters.+ """+ use_triton = True+ disable_exllamav2 = True+ disable_exllamav1 = False+ disable_marlin = True+ use_qigen = False+ if "qigen" in backend:+ use_triton = False+ use_qigen = True+ elif "marlin" in backend and sym:+ use_triton = False+ disable_marlin = False+ else:+ ##we all use triton for others, ##TODO may have bugs for some backends+ from auto_round.export.export_to_autogptq.qlinear_triton import QuantLinear+ return QuantLinear+ try:+ import auto_gptq # pylint: disable=E0401+ except:+ logger.warning_once(f"please install auto_gptq via 'pip install auto-gptq' to support exporting to {backend}")+ exit()++ from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401+ QuantLinear = dynamically_import_QuantLinear(+ use_triton=use_triton,+ desc_act=False,+ group_size=group_size,+ bits=bits,+ disable_exllama=disable_exllamav1,+ disable_exllamav2=disable_exllamav2,+ use_qigen=use_qigen,+ disable_marlin=disable_marlin,+ )+ return QuantLinear@register_format("auto_gptq")-def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwargs): ##TODO align with autoround format+def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllamav2",+ **kwargs):"""Export the model to autogptq format to easily leverage cuda kernel."""- try:- import auto_gptq- except ImportError:- raise ImportError("export to autogptq requires autogptq library. Please run 'pip install auto-gptq'")+model = kwargs["model"]- layer_config = kwargs["layer_config"]- sym = kwargs["sym"]- bits = kwargs["bits"]- group_size = kwargs["group_size"]- iters = kwargs["iters"]- lr = kwargs["lr"]- minmax_lr = kwargs["minmax_lr"]- enable_minmax_tuning = kwargs["enable_minmax_tuning"]- enable_quanted_input = kwargs["enable_quanted_input"]- scale_dtype = kwargs["scale_dtype"]tokenizer = kwargs["tokenizer"]supported_types = kwargs["supported_types"]@@ -93,163 +132,102 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwarif all_to_quantized:modules_in_block_to_quantize = None- if inplace:- compressed_model = model.to("cpu")- else:- compressed_model = copy.deepcopy(model.to("cpu"))+ model = model.to(torch.float16) ##force to fp16+ if not inplace:+ model = copy.deepcopy(model.to("cpu"))- from auto_gptq.modeling._utils import pack_model # pylint: disable=E0401+ layer_config = kwargs["layer_config"]- if bits == 3 or use_triton is False:- if bits == 3 and use_triton is True:- logger.warning("triton does not support 3 bits, reset it to False")- use_triton = False- quantizers = {}- for key in layer_config:- if key == "lm_head": ##TODO remove this after pr 87 is merged- continue- info = layer_config[key]- if not check_to_quantized(info):- continue- ##force to float32 to be compatible with torch 2.0- quantizers[key] = (None, info["scale"], info["zp"].to(torch.float32), info["g_idx"])with tctl.threadpool_limits(limits=1):- pack_model(- compressed_model,- quantizers,- bits,- group_size,- use_cuda_fp16=True,- desc_act=False,- force_layer_back_to_cpu=True,- use_triton=use_triton,- )+ for name in layer_config.keys():+ if name == "lm_head": ##dese not support lm-head+ continue+ config = kwargs["layer_config"][name]+ if config["bits"] > 8:+ continue+ logger.info(f"packing {name}")++ bits = config["bits"]+ group_size = config["group_size"]+ sym = config["sym"]++ layer = get_module(model, name)+ device = layer.weight.device++ QuantLinear = get_autogptq_packing_qlinear(backend, bits, group_size, sym)++ if isinstance(layer, nn.Linear):+ in_features = layer.in_features+ out_features = layer.out_features+ elif isinstance(layer, nn.Conv2d):+ in_features = layer.in_channels+ out_features = layer.out_channels+ elif isinstance(layer, transformers.pytorch_utils.Conv1D):+ in_features = layer.weight.shape[0]+ out_features = layer.weight.shape[1]++ ##bias = layer.bias is not None and torch.any(layer.bias)+ bias = True ## if using the above, llama3 lambada RTN will be NAN , TODO why?+ new_layer = QuantLinear( ##pylint: disable=E1123+ bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype+ )++ new_layer.device = device+ set_module(model, name, new_layer)+ qlayer = new_layer+ scale = layer_config[name]["scale"]+ zero = layer_config[name]["zp"]+ # so far can only pack layer on CPU+ qlayer.to("cpu")+ ##force to float32 to be compatible with torch 2.0+ layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)+ sig = inspect.signature(qlayer.pack)+ param_count = len(sig.parameters)+ if param_count == 2:+ qlayer.pack(layer, scale)+ else:+ qlayer.pack(layer, scale, zero, None)+ qlayer.to(device)if output_dir is None:- return compressed_model-- _save_quantized_to_autogptq(- compressed_model,- output_dir,- bits=bits,- group_size=group_size,- sym=sym,- iters=iters,- lr=lr,- minmax_lr=minmax_lr,- enable_minmax_tuning=enable_minmax_tuning,- enable_quanted_input=enable_quanted_input,- scale_dtype=scale_dtype,- use_safetensors=True,- modules_in_block_to_quantize=modules_in_block_to_quantize,- )---def _save_quantized_to_autogptq(- model,- save_dir: str,- bits=4,- group_size=128,- sym=False,- iters=200,- lr=5e-3,- minmax_lr=5e-3,- enable_minmax_tuning=True,- enable_quanted_input=True,- use_safetensors: bool = True,- scale_dtype=torch.float32,- safetensors_metadata: Optional[Dict[str, str]] = None,- modules_in_block_to_quantize=None,-):- """Save quantized model and configs to local disk for cuda."""- os.makedirs(save_dir, exist_ok=True)- model.to("cpu")-- model_base_name = "model"- if use_safetensors:- model_save_name = model_base_name + ".safetensors"- state_dict = model.state_dict()- state_dict = {k: v.clone().contiguous() for k, v in state_dict.items()}- if safetensors_metadata is None:- safetensors_metadata = {}- elif not isinstance(safetensors_metadata, dict):- raise TypeError("safetensors_metadata must be a dictionary.")- else:- logger.debug(f"Received safetensors_metadata: {safetensors_metadata}")- new_safetensors_metadata = {}- converted_keys = False- for key, value in safetensors_metadata.items():- if not isinstance(key, str) or not isinstance(value, str):- converted_keys = True- try:- new_key = str(key)- new_value = str(value)- except Exception as e:- raise TypeError(- "safetensors_metadata: both keys and values must be strings"- / f" and an error occurred when trying to convert them: {e}"- )- if new_key in new_safetensors_metadata:- logger.warning(- f"After converting safetensors_metadata keys to strings, the key '{new_key}' "- / "is duplicated. Ensure that all your metadata keys are strings to avoid overwriting."- )- new_safetensors_metadata[new_key] = new_value- safetensors_metadata = new_safetensors_metadata- if converted_keys:- logger.debug(- "One or more safetensors_metadata keys or values had to be converted to str()."- / f" Final safetensors_metadata: {safetensors_metadata}"- )-- # Format is required to enable Accelerate to load the metadata- # otherwise it raises an OSError- safetensors_metadata["format"] = "pt"-- # Store the quantization configuration as safetensors metadata- from auto_round import __version__-- safetensors_metadata["autoround_version"] = str(__version__)- safetensors_metadata["bits"] = str(bits)- safetensors_metadata["group_size"] = str(group_size)- safetensors_metadata["iters"] = str(iters)- safetensors_metadata["lr"] = str(lr)- safetensors_metadata["minmax_lr"] = str(minmax_lr)- safetensors_metadata["enable_minmax_tuning"] = str(enable_minmax_tuning)- safetensors_metadata["enable_quanted_input"] = str(enable_quanted_input)- safetensors_metadata["scale_dtype"] = convert_dtype_torch2str_hf(scale_dtype)- safe_save(state_dict, join(save_dir, model_save_name), safetensors_metadata)- else:- model_save_name = model_base_name + ".bin"- torch.save(model.state_dict(), join(save_dir, model_save_name))-- from auto_gptq.modeling._base import BaseQuantizeConfig # pylint: disable=E0401-- quantization_config = BaseQuantizeConfig(- bits=bits,- group_size=group_size,- desc_act=False,- sym=sym,- true_sequential=False,- static_groups=False,- model_file_base_name=model_base_name,- )- quantization_config.model_file_base_name = model_base_name-- config_dict = quantization_config.to_dict()- config_dict["quant_method"] = "gptq"- config_dict["autoround_version"] = __version__- config_dict["iters"] = iters- config_dict["lr"] = lr- config_dict["minmax_lr"] = minmax_lr- config_dict["enable_minmax_tuning"] = enable_minmax_tuning- config_dict["enable_quanted_input"] = enable_quanted_input- config_dict["scale_dtype"] = convert_dtype_torch2str_hf(scale_dtype)+ return model+ quantization_config = kwargs["serialization_dict"]+ quantization_config["quant_method"] = "gptq"+ quantization_config.pop("dataset", None) ## pile-10k is not supported in gptq+ quantization_config["desc_act"] = False ## for autogptq API+ quantization_config["true_sequential"] = False+ quantization_config["damp_percent"] = 0.01if modules_in_block_to_quantize is not None:- config_dict["modules_in_block_to_quantize"] = modules_in_block_to_quantize-- with open(join(save_dir, "quantize_config.json"), "w", encoding="utf-8") as f:- json.dump(config_dict, f, indent=2)-- config_dict["quant_method"] = "gptq" ##hf transformers could only recognize this value- model.config.quantization_config = config_dict- model.config.save_pretrained(save_dir)+ quantization_config["modules_in_block_to_quantize"] = modules_in_block_to_quantize+ if hasattr(model, "config"):+ model.config.quantization_config = quantization_config+ save(model, output_dir)+++##+def save(model: torch.nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True):+ """Save model state dict and configs.++ Args:+ model (`nn.Module`):+ Model to be saved. The model can be wrapped or unwrapped.+ save_dir (`str`):+ Directory to which to save. Will be created if it doesn't exist.+ max_shard_size (`str`, defaults to `"10GB"`):+ The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size+ lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).+ <Tip warning={true}>++ If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard+ which will be bigger than `max_shard_size`.++ </Tip>+ safe_serialization (`bool`, defaults to `True`):+ Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).+ """+ max_shard_size = "10000GB" ## API of auto-gptq with marlin does not support shard size+ os.makedirs(save_dir, exist_ok=True)+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ config_file = "quantize_config.json"+ if hasattr(model, "config") and hasattr(model.config, "quantization_config"):+ with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:+ json.dump(model.config.quantization_config, f, indent=2)diff --git a/auto_round/export/export_to_autogptq/qlinear_triton.py b/auto_round/export/export_to_autogptq/qlinear_triton.pynew file mode 100644index 000000000..ea34c881a--- /dev/null+++ b/auto_round/export/export_to_autogptq/qlinear_triton.py@@ -0,0 +1,142 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import math++import numpy as np+import torch+import torch.nn as nn+import transformers++class TritonModuleMixin:+ @classmethod+ def warmup(cls, model, transpose=False, seqlen=2048):+ pass++++class QuantLinear(nn.Module, TritonModuleMixin):+ QUANT_TYPE = "triton"++ def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):+ super().__init__()+ if bits not in [2, 4, 8]:+ raise NotImplementedError("Only 2,4,8 bits are supported.")+ if infeatures % 32 != 0 or outfeatures % 32 != 0:+ raise NotImplementedError("in_feature and out_feature must be divisible by 32.")+ self.infeatures = infeatures+ self.outfeatures = outfeatures+ self.bits = bits+ self.group_size = group_size if group_size != -1 else infeatures+ self.maxq = 2**self.bits - 1++ self.register_buffer(+ "qweight",+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),+ )+ self.register_buffer(+ "qzeros",+ torch.zeros(+ (+ math.ceil(infeatures / self.group_size),+ outfeatures // 32 * self.bits,+ ),+ dtype=torch.int32,+ ),+ )+ self.register_buffer(+ "scales",+ torch.zeros(+ (math.ceil(infeatures / self.group_size), outfeatures),+ dtype=torch.float16,+ ),+ )+ self.register_buffer(+ "g_idx",+ torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),+ )+ if bias:+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))+ else:+ self.bias = None++ self.trainable = trainable++ def post_init(self):+ pass++ def pack(self, linear, scales, zeros, g_idx=None):+ W = linear.weight.data.clone()+ if isinstance(linear, nn.Conv2d):+ W = W.flatten(1)+ if isinstance(linear, transformers.pytorch_utils.Conv1D):+ W = W.t()++ self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx++ scales = scales.t().contiguous()+ zeros = zeros.t().contiguous()+ scale_zeros = zeros * scales+ self.scales = scales.clone().half()+ if linear.bias is not None:+ self.bias = linear.bias.clone().half()++ intweight = []+ for idx in range(self.infeatures):+ intweight.append(+ torch.round((W[:, idx] + scale_zeros[self.g_idx[idx]]) / self.scales[self.g_idx[idx]]).to(torch.int)[+ :, None+ ]+ )+ intweight = torch.cat(intweight, dim=1)+ intweight = intweight.t().contiguous()+ intweight = intweight.numpy().astype(np.uint32)++ i = 0+ row = 0+ qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)+ while row < qweight.shape[0]:+ if self.bits in [2, 4, 8]:+ for j in range(i, i + (32 // self.bits)):+ qweight[row] |= intweight[j] << (self.bits * (j - i))+ i += 32 // self.bits+ row += 1+ else:+ raise NotImplementedError("Only 2,4,8 bits are supported.")++ qweight = qweight.astype(np.int32)+ self.qweight = torch.from_numpy(qweight)++ zeros -= 1+ zeros = zeros.numpy().astype(np.uint32)+ qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)+ i = 0+ col = 0+ while col < qzeros.shape[1]:+ if self.bits in [2, 4, 8]:+ for j in range(i, i + (32 // self.bits)):+ qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))+ i += 32 // self.bits+ col += 1+ else:+ raise NotImplementedError("Only 2,4,8 bits are supported.")++ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)++++++__all__ = ["QuantLinear"]\ No newline at end of filediff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex cbb21e6da..c6c33e117 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -44,7 +44,6 @@ def check_neq_config(config, data_type, bits, group_size, sym):res = []if data_type != config["data_type"]:res.append("data_type")- return resif bits != config["bits"]:res.append("bits")if group_size != config["group_size"]:@@ -172,7 +171,31 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exlayer_names_in_block = get_layer_names_in_block(model)layer_config = kwargs["layer_config"]-+ quantization_config = kwargs["serialization_dict"]+ quantization_config["quant_method"] = "intel/auto-round"+ quantization_config["backend"] = backend+ extra_config = {}+ for layer_name in layer_config:+ if layer_name not in layer_names_in_block and layer_config[layer_name]["bits"] <= 8: ##lm head+ extra_config[layer_name] = {}+ extra_config[layer_name]["bits"] = layer_config[layer_name]["bits"]+ extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]+ extra_config[layer_name]["group_size"] = layer_config[layer_name]["group_size"]+ extra_config[layer_name]["sym"] = layer_config[layer_name]["sym"]+ elif layer_name in layer_names_in_block:+ neq_keys = check_neq_config(+ layer_config[layer_name],+ data_type=quantization_config["data_type"],+ bits=quantization_config["bits"],+ group_size=quantization_config["group_size"],+ sym=quantization_config["sym"],+ )+ if len(neq_keys) > 0:+ extra_config[layer_name] = {}+ for key in neq_keys:+ extra_config[layer_name][key] = layer_config[layer_name][key]+ if len(extra_config) > 0:+ quantization_config["extra_config"] = extra_configwith tctl.threadpool_limits(limits=1):for name in layer_config.keys():@@ -221,33 +244,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exelse:qlayer.pack(layer, scale, zero, None)qlayer.to(device)- quantization_config = kwargs["serialization_dict"]- quantization_config["quant_method"] = "intel/auto-round"- quantization_config["backend"] = backend- extra_config = {}- for layer_name in layer_config:- if layer_config[layer_name]["bits"] > 8:- continue- if layer_name not in layer_names_in_block:- extra_config[layer_name] = {}- extra_config[layer_name]["bits"] = layer_config[layer_name]["bits"]- extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]- extra_config[layer_name]["group_size"] = layer_config[layer_name]["group_size"]- extra_config[layer_name]["sym"] = layer_config[layer_name]["sym"]- else:- neq_keys = check_neq_config(- layer_config[layer_name],- data_type=quantization_config["data_type"],- bits=quantization_config["bits"],- group_size=quantization_config["group_size"],- sym=quantization_config["sym"],- )- if len(neq_keys) > 0:- extra_config[layer_name] = {}- for key in neq_keys:- extra_config[layer_name][key] = layer_config[layer_name][key]- if len(extra_config) > 0:- quantization_config["extra_config"] = extra_config+if hasattr(model, "config"):model.config.quantization_config = quantization_configtokenizer = kwargs["tokenizer"]diff --git a/requirements.txt b/requirements.txtindex 66cbf2523..83ec1fe4e 100644--- a/requirements.txt+++ b/requirements.txt@@ -6,5 +6,4 @@ torchtransformerstritonnumpy < 2.0-threadpoolctl-auto-gptq\ No newline at end of file+threadpoolctl\ No newline at end of file