Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine/fix gptq format (#1853)

Wenhua Cheng09ca561549eb
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index 0b5bc359e..dca82caeb 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -45,7 +45,6 @@ import torch.nn as nn
import transformers
from tqdm import tqdm
-import auto_round.export.export_to_autogptq.qlinear_triton
from auto_round.export.utils import (
filter_quantization_config,
get_autogptq_packing_qlinear,
@@ -66,10 +65,8 @@ from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
check_start_with_block_name,
check_to_quantized,
- copy_python_files_from_model_cache,
get_block_names,
get_module,
- json_serialize,
matches_any_regex,
set_module,
to_standard_regex,
@@ -160,9 +157,7 @@ def pack_layer(name, model, backend, device=None):
bias = layer.bias is not None
##bias = True ## if using the above, llama3 lambada RTN will be NAN , TODO why?
- qlayer = QuantLinear( ##pylint: disable=E1123
- bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype
- )
+ qlayer = QuantLinear(bits, group_size, in_features, out_features, bias, g_idx=True) ##pylint: disable=E1123
qlayer.device = orig_device
scale = layer.scale
@@ -172,8 +167,7 @@ def pack_layer(name, model, backend, device=None):
##force to float32 to be compatible with torch 2.0
if sym and isinstance(zero, torch.Tensor):
layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu")
- if isinstance(qlayer, auto_round.export.export_to_autogptq.qlinear_triton.QuantLinear):
- zero = int(zero.flatten()[0])
+ zero = int(zero.flatten()[0])
else:
layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero
if isinstance(zero, torch.Tensor) and zero.dtype == torch.bfloat16:
diff --git a/auto_round/export/export_to_autogptq/qlinear_triton.py b/auto_round/export/export_to_autogptq/qlinear_triton.py
deleted file mode 100644
index bf11a9bf5..000000000
--- a/auto_round/export/export_to_autogptq/qlinear_triton.py
+++ /dev/null
@@ -1,138 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import math
-
-import numpy as np
-import torch
-import torch.nn as nn
-import transformers
-
-
-class TritonModuleMixin:
- @classmethod
- def warmup(cls, model, transpose=False, seqlen=2048):
- pass
-
-
-class QuantLinear(nn.Module, TritonModuleMixin):
- QUANT_TYPE = "triton"
-
- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):
- super().__init__()
- if bits not in [2, 4, 8]:
- raise NotImplementedError("Only 2,4,8 bits are supported.")
- if infeatures % 32 != 0 or outfeatures % 32 != 0:
- raise NotImplementedError("in_feature and out_feature must be divisible by 32.")
- self.infeatures = infeatures
- self.outfeatures = outfeatures
- self.bits = bits
- self.group_size = group_size if group_size != -1 else infeatures
- self.maxq = 2**self.bits - 1
-
- self.register_buffer(
- "qweight",
- torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
- )
- self.register_buffer(
- "qzeros",
- torch.zeros(
- (
- math.ceil(infeatures / self.group_size),
- outfeatures // 32 * self.bits,
- ),
- dtype=torch.int32,
- ),
- )
- self.register_buffer(
- "scales",
- torch.zeros(
- (math.ceil(infeatures / self.group_size), outfeatures),
- dtype=torch.float16,
- ),
- )
- self.register_buffer(
- "g_idx",
- torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),
- )
-
- if bias:
- self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))
- else:
- self.bias = None
-
- self.trainable = trainable
-
- def post_init(self):
- pass
-
- def pack(self, linear, scales, zeros, g_idx=None, device=None):
- scales_t = scales.t().contiguous()
- self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
- if linear.bias is not None:
- self.bias = linear.bias.clone().half()
- self.scales = scales_t.clone().half()
-
- W = linear.weight.data.to(device).clone()
- if type(linear) == nn.Conv2d:
- W = W.flatten(1)
- if type(linear) == transformers.pytorch_utils.Conv1D:
- W = W.t()
-
- repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)
- if isinstance(zeros, torch.Tensor):
- repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)
- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(
- torch.int32
- )
- else:
- repeat_zeros = zeros
- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)
-
- del repeat_scales
- intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)
- order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits
- intweight = intweight << order_map
- intweight = torch.sum(intweight, dim=-1)
-
- intweight = intweight.t().contiguous().to(torch.int32)
- self.qweight = intweight.to("cpu")
-
- if isinstance(zeros, torch.Tensor):
- zeros = zeros.t().contiguous()
- zeros -= 1
- zeros = zeros.numpy().astype(np.uint32)
- qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)
- i = 0
- col = 0
- while col < qzeros.shape[1]:
- for j in range(i, i + (32 // self.bits)):
- qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
- i += 32 // self.bits
- col += 1
-
- qzeros = qzeros.astype(np.int32)
- self.qzeros = torch.from_numpy(qzeros)
- else:
- zeros -= 1
- shape = scales_t.shape
- value = 0
- for j in range(0, (32 // self.bits)):
- value |= zeros << (self.bits * j)
- qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value
- qzeros = qzeros.astype(np.int32)
- self.qzeros = torch.from_numpy(qzeros)
-
-
-__all__ = ["QuantLinear"]
diff --git a/auto_round/export/utils.py b/auto_round/export/utils.py
index 395cd7fa4..eaa773fd0 100644
--- a/auto_round/export/utils.py
+++ b/auto_round/export/utils.py
@@ -294,68 +294,9 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):
Returns:
class: The dynamically imported QuantLinear class configured according to the specified parameters.
"""
- use_triton = True
- if bits not in [2, 4, 8]:
- use_triton = False
- disable_exllamav2 = True
- disable_exllamav1 = False
- disable_marlin = True
- use_qigen = False
- if "qigen" in backend:
- use_triton = False
- use_qigen = True
- elif "triton" in backend:
- use_triton = True
- elif "marlin" in backend and sym:
- use_triton = False
- disable_marlin = False
- elif "exllama" in backend: ##need v1 code to export
- use_triton = True ##same with triton
- disable_marlin = True
- elif "cuda" in backend:
- use_triton = False
- disable_marlin = True
- disable_exllamav2 = True
- disable_exllamav1 = True
- if use_triton:
- from auto_round.export.export_to_autogptq.qlinear_triton import QuantLinear
-
- return QuantLinear
- try:
- import auto_gptq # pylint: disable=E0401
- except:
- logger.error(f"please install auto_gptq via 'pip install auto-gptq' to support exporting to {backend}")
- exit()
-
- from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401
-
- from auto_round.utils.common import get_library_version
-
- version = get_library_version("auto_gptq")
- from packaging.version import Version
-
- if Version(version) < Version("0.7.2"):
- QuantLinear = dynamically_import_QuantLinear(
- use_triton=use_triton,
- desc_act=False,
- group_size=group_size,
- bits=bits,
- disable_exllama=disable_exllamav1,
- disable_exllamav2=disable_exllamav2,
- use_qigen=use_qigen,
- disable_marlin=disable_marlin,
- )
- else:
- QuantLinear = dynamically_import_QuantLinear( # pylint: disable=E1123
- use_triton=use_triton,
- desc_act=False,
- group_size=group_size,
- bits=bits,
- disable_exllama=disable_exllamav1,
- disable_exllamav2=disable_exllamav2,
- use_qigen=use_qigen,
- use_marlin=not disable_marlin,
- )
+
+ from auto_round_extension.torch.qlinear_torch_zp import QuantLinear
+
return QuantLinear
diff --git a/auto_round/formats.py b/auto_round/formats.py
index f255baac4..a6947ae23 100644
--- a/auto_round/formats.py
+++ b/auto_round/formats.py
@@ -128,6 +128,10 @@ def get_formats(
formats = remove_duplicates(formats)
+ for fmt in formats:
+ if fmt not in SUPPORTED_FORMATS:
+ raise ValueError(f"{fmt} is not supported, we only support {SUPPORTED_FORMATS}")
+
for i in range(len(formats)):
if formats[i].startswith("gguf:"):
formats[i] = GGUFFormat(formats[i], ar)