Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

avoid underflow and overflow for exllamav2 (#218)

wenhuach2104678e061aa5
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.py
index 69bfe56f1..a1f18cd4b 100644
--- a/auto_round/auto_quantizer.py
+++ b/auto_round/auto_quantizer.py
@@ -264,7 +264,6 @@ class AutoRoundConfig(QuantizationConfigMixin):
self.quant_method = AutoRoundQuantizationMethod.AutoRound
self.post_init()
-
def post_init(self):
r"""Safety checker that arguments are correct."""
if self.bits not in [2, 4, 8]:
@@ -331,11 +330,8 @@ class AutoRoundQuantizer(HfQuantizer):
if hasattr(quantization_config, "backend"): # pragma: no cover
backend = quantization_config.backend
if "hpu" in backend and model.dtype != torch.bfloat16:
- logger.info("We suggest you to set `torch_dtype=torch.bfloat16` for better efficiency with AutoRound.")
+ logger.info("change the dtype to `bfloat16` as HPU does not support float16")
model = model.to(torch.bfloat16)
- elif model.dtype != torch.float16:
- logger.info("We suggest you to set `torch_dtype=torch.float16` for better efficiency with AutoRound.")
- model = model.to(torch.float16)
bits = quantization_config.bits
group_size = quantization_config.group_size
data_type = quantization_config.data_type if hasattr(quantization_config, "data_type") \
@@ -354,11 +350,13 @@ class AutoRoundQuantizer(HfQuantizer):
layer_configs[layer_name]["group_size"] = group_size
layer_configs[layer_name]["data_type"] = data_type
layer_configs[layer_name]["sym"] = sym
+ layer_configs[layer_name]["clip"] = False
else:
layer_configs[layer_name]["bits"] = extra_config[layer_name].get("bits", bits)
layer_configs[layer_name]["group_size"] = extra_config[layer_name].get("group_size", group_size)
layer_configs[layer_name]["data_type"] = extra_config[layer_name].get("data_type", data_type)
layer_configs[layer_name]["sym"] = extra_config[layer_name].get("sym", sym)
+ layer_configs[layer_name]["clip"] = extra_config[layer_name].get("clip", False)
if hasattr(quantization_config, "backend"): # pragma: no cover
backend = quantization_config.backend
elif 'gptq' in quantization_config.quant_method: # pragma: no cover
@@ -386,6 +384,7 @@ class AutoRoundQuantizer(HfQuantizer):
group_size = config["group_size"]
data_type = config["data_type"]
sym = config["sym"]
+ clip = config["clip"]
if not (bits <= 8):
continue
@@ -402,14 +401,25 @@ class AutoRoundQuantizer(HfQuantizer):
in_features = layer.weight.shape[0]
out_features = layer.weight.shape[1]
bias = layer.bias is not None
- new_layer = QuantLinear( # pylint: disable=E1123
- bits,
- group_size,
- in_features,
- out_features,
- bias,
- weight_dtype=layer.weight.dtype,
- )
+ try:
+ new_layer = QuantLinear( # pylint: disable=E1123
+ bits,
+ group_size,
+ in_features,
+ out_features,
+ bias,
+ weight_dtype=layer.weight.dtype,
+ clip = clip
+ )
+ except:
+ new_layer = QuantLinear( # pylint: disable=E1123
+ bits,
+ group_size,
+ in_features,
+ out_features,
+ bias,
+ weight_dtype=layer.weight.dtype,
+ )
new_layer.device = device
set_module(module, layer_name, new_layer)
diff --git a/auto_round_extension/cuda/qliner_exllamav2.py b/auto_round_extension/cuda/qliner_exllamav2.py
index 3f98700d7..3bc415971 100644
--- a/auto_round_extension/cuda/qliner_exllamav2.py
+++ b/auto_round_extension/cuda/qliner_exllamav2.py
@@ -40,7 +40,6 @@ from logging import getLogger
import torch
import torch.nn as nn
-
logger = getLogger(__name__)
try:
@@ -48,12 +47,14 @@ try:
except ImportError as e:
exllama_v2_import_exception = e
+
def error_raiser_exllama(*args, **kwargs):
raise ValueError(
f"Trying to use the exllama v2 backend, but could not import the C++/CUDA dependencies with the following "
f"error: {exllama_v2_import_exception}"
)
+
make_q_matrix = error_raiser_exllama
gemm_half_q_half = error_raiser_exllama
@@ -167,7 +168,7 @@ class QuantLinear(nn.Module):
self.bits = bits
self.group_size = group_size if group_size != -1 else infeatures
self.trainable = trainable
- self.maxq = 2**self.bits - 1
+ self.maxq = 2 ** self.bits - 1
assert infeatures % 32 == 0
assert infeatures % self.group_size == 0
@@ -204,6 +205,7 @@ class QuantLinear(nn.Module):
self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))
else:
self.bias = None
+ self.clip = kwargs.get("clip", False)
def post_init(self, temp_dq):
assert self.qweight.device.type == "cuda"
@@ -218,6 +220,7 @@ class QuantLinear(nn.Module):
self.q_handle = ext_make_q_matrix(self.q_tensors, temp_dq)
def forward(self, x, force_cuda=False):
+ orig_dtype = x.dtype
if x.dtype != torch.float16:
logger.warning_once(
f"The exllama v2 kernel for GPTQ requires a float16 input activation, while {x.dtype} was passed. "
@@ -229,9 +232,12 @@ class QuantLinear(nn.Module):
x = x.half()
output = ext_gemm_half_q_half(x, self.q_handle, self.outfeatures, force_cuda)
-
+ if orig_dtype != torch.float16:
+ output = output.to(orig_dtype)
if self.bias is not None:
output.add_(self.bias)
+ if self.clip:
+ output = torch.clip(output, -65504, 65504)
return output
def temp_dq_size(self):