PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine/fix gptq format (#1853)
Wenhua Cheng09ca561549eb
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 0b5bc359e..dca82caeb 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -45,7 +45,6 @@ import torch.nn as nnimport transformersfrom tqdm import tqdm-import auto_round.export.export_to_autogptq.qlinear_tritonfrom auto_round.export.utils import (filter_quantization_config,get_autogptq_packing_qlinear,@@ -66,10 +65,8 @@ from auto_round.utils import (SUPPORTED_LAYER_TYPES,check_start_with_block_name,check_to_quantized,- copy_python_files_from_model_cache,get_block_names,get_module,- json_serialize,matches_any_regex,set_module,to_standard_regex,@@ -160,9 +157,7 @@ def pack_layer(name, model, backend, device=None):bias = layer.bias is not None##bias = True ## if using the above, llama3 lambada RTN will be NAN , TODO why?- qlayer = QuantLinear( ##pylint: disable=E1123- bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype- )+ qlayer = QuantLinear(bits, group_size, in_features, out_features, bias, g_idx=True) ##pylint: disable=E1123qlayer.device = orig_devicescale = layer.scale@@ -172,8 +167,7 @@ def pack_layer(name, model, backend, device=None):##force to float32 to be compatible with torch 2.0if sym and isinstance(zero, torch.Tensor):layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu")- if isinstance(qlayer, auto_round.export.export_to_autogptq.qlinear_triton.QuantLinear):- zero = int(zero.flatten()[0])+ zero = int(zero.flatten()[0])else:layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zeroif isinstance(zero, torch.Tensor) and zero.dtype == torch.bfloat16:diff --git a/auto_round/export/export_to_autogptq/qlinear_triton.py b/auto_round/export/export_to_autogptq/qlinear_triton.pydeleted file mode 100644index bf11a9bf5..000000000--- a/auto_round/export/export_to_autogptq/qlinear_triton.py+++ /dev/null@@ -1,138 +0,0 @@-# Copyright (c) 2024 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import math--import numpy as np-import torch-import torch.nn as nn-import transformers---class TritonModuleMixin:- @classmethod- def warmup(cls, model, transpose=False, seqlen=2048):- pass---class QuantLinear(nn.Module, TritonModuleMixin):- QUANT_TYPE = "triton"-- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):- super().__init__()- if bits not in [2, 4, 8]:- raise NotImplementedError("Only 2,4,8 bits are supported.")- if infeatures % 32 != 0 or outfeatures % 32 != 0:- raise NotImplementedError("in_feature and out_feature must be divisible by 32.")- self.infeatures = infeatures- self.outfeatures = outfeatures- self.bits = bits- self.group_size = group_size if group_size != -1 else infeatures- self.maxq = 2**self.bits - 1-- self.register_buffer(- "qweight",- torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),- )- self.register_buffer(- "qzeros",- torch.zeros(- (- math.ceil(infeatures / self.group_size),- outfeatures // 32 * self.bits,- ),- dtype=torch.int32,- ),- )- self.register_buffer(- "scales",- torch.zeros(- (math.ceil(infeatures / self.group_size), outfeatures),- dtype=torch.float16,- ),- )- self.register_buffer(- "g_idx",- torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),- )-- if bias:- self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))- else:- self.bias = None-- self.trainable = trainable-- def post_init(self):- pass-- def pack(self, linear, scales, zeros, g_idx=None, device=None):- scales_t = scales.t().contiguous()- self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx- if linear.bias is not None:- self.bias = linear.bias.clone().half()- self.scales = scales_t.clone().half()-- W = linear.weight.data.to(device).clone()- if type(linear) == nn.Conv2d:- W = W.flatten(1)- if type(linear) == transformers.pytorch_utils.Conv1D:- W = W.t()-- repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)- if isinstance(zeros, torch.Tensor):- repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(- torch.int32- )- else:- repeat_zeros = zeros- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)-- del repeat_scales- intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)- order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits- intweight = intweight << order_map- intweight = torch.sum(intweight, dim=-1)-- intweight = intweight.t().contiguous().to(torch.int32)- self.qweight = intweight.to("cpu")-- if isinstance(zeros, torch.Tensor):- zeros = zeros.t().contiguous()- zeros -= 1- zeros = zeros.numpy().astype(np.uint32)- qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)- i = 0- col = 0- while col < qzeros.shape[1]:- for j in range(i, i + (32 // self.bits)):- qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))- i += 32 // self.bits- col += 1-- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)- else:- zeros -= 1- shape = scales_t.shape- value = 0- for j in range(0, (32 // self.bits)):- value |= zeros << (self.bits * j)- qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)---__all__ = ["QuantLinear"]diff --git a/auto_round/export/utils.py b/auto_round/export/utils.pyindex 395cd7fa4..eaa773fd0 100644--- a/auto_round/export/utils.py+++ b/auto_round/export/utils.py@@ -294,68 +294,9 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):Returns:class: The dynamically imported QuantLinear class configured according to the specified parameters."""- use_triton = True- if bits not in [2, 4, 8]:- use_triton = False- disable_exllamav2 = True- disable_exllamav1 = False- disable_marlin = True- use_qigen = False- if "qigen" in backend:- use_triton = False- use_qigen = True- elif "triton" in backend:- use_triton = True- elif "marlin" in backend and sym:- use_triton = False- disable_marlin = False- elif "exllama" in backend: ##need v1 code to export- use_triton = True ##same with triton- disable_marlin = True- elif "cuda" in backend:- use_triton = False- disable_marlin = True- disable_exllamav2 = True- disable_exllamav1 = True- if use_triton:- from auto_round.export.export_to_autogptq.qlinear_triton import QuantLinear-- return QuantLinear- try:- import auto_gptq # pylint: disable=E0401- except:- logger.error(f"please install auto_gptq via 'pip install auto-gptq' to support exporting to {backend}")- exit()-- from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401-- from auto_round.utils.common import get_library_version-- version = get_library_version("auto_gptq")- from packaging.version import Version-- if Version(version) < Version("0.7.2"):- QuantLinear = dynamically_import_QuantLinear(- use_triton=use_triton,- desc_act=False,- group_size=group_size,- bits=bits,- disable_exllama=disable_exllamav1,- disable_exllamav2=disable_exllamav2,- use_qigen=use_qigen,- disable_marlin=disable_marlin,- )- else:- QuantLinear = dynamically_import_QuantLinear( # pylint: disable=E1123- use_triton=use_triton,- desc_act=False,- group_size=group_size,- bits=bits,- disable_exllama=disable_exllamav1,- disable_exllamav2=disable_exllamav2,- use_qigen=use_qigen,- use_marlin=not disable_marlin,- )++ from auto_round_extension.torch.qlinear_torch_zp import QuantLinear+return QuantLineardiff --git a/auto_round/formats.py b/auto_round/formats.pyindex f255baac4..a6947ae23 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -128,6 +128,10 @@ def get_formats(formats = remove_duplicates(formats)+ for fmt in formats:+ if fmt not in SUPPORTED_FORMATS:+ raise ValueError(f"{fmt} is not supported, we only support {SUPPORTED_FORMATS}")+for i in range(len(formats)):if formats[i].startswith("gguf:"):formats[i] = GGUFFormat(formats[i], ar)