PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
avoid underflow and overflow for exllamav2 (#218)
wenhuach2104678e061aa5
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex 69bfe56f1..a1f18cd4b 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -264,7 +264,6 @@ class AutoRoundConfig(QuantizationConfigMixin):self.quant_method = AutoRoundQuantizationMethod.AutoRoundself.post_init()-def post_init(self):r"""Safety checker that arguments are correct."""if self.bits not in [2, 4, 8]:@@ -331,11 +330,8 @@ class AutoRoundQuantizer(HfQuantizer):if hasattr(quantization_config, "backend"): # pragma: no coverbackend = quantization_config.backendif "hpu" in backend and model.dtype != torch.bfloat16:- logger.info("We suggest you to set `torch_dtype=torch.bfloat16` for better efficiency with AutoRound.")+ logger.info("change the dtype to `bfloat16` as HPU does not support float16")model = model.to(torch.bfloat16)- elif model.dtype != torch.float16:- logger.info("We suggest you to set `torch_dtype=torch.float16` for better efficiency with AutoRound.")- model = model.to(torch.float16)bits = quantization_config.bitsgroup_size = quantization_config.group_sizedata_type = quantization_config.data_type if hasattr(quantization_config, "data_type") \@@ -354,11 +350,13 @@ class AutoRoundQuantizer(HfQuantizer):layer_configs[layer_name]["group_size"] = group_sizelayer_configs[layer_name]["data_type"] = data_typelayer_configs[layer_name]["sym"] = sym+ layer_configs[layer_name]["clip"] = Falseelse:layer_configs[layer_name]["bits"] = extra_config[layer_name].get("bits", bits)layer_configs[layer_name]["group_size"] = extra_config[layer_name].get("group_size", group_size)layer_configs[layer_name]["data_type"] = extra_config[layer_name].get("data_type", data_type)layer_configs[layer_name]["sym"] = extra_config[layer_name].get("sym", sym)+ layer_configs[layer_name]["clip"] = extra_config[layer_name].get("clip", False)if hasattr(quantization_config, "backend"): # pragma: no coverbackend = quantization_config.backendelif 'gptq' in quantization_config.quant_method: # pragma: no cover@@ -386,6 +384,7 @@ class AutoRoundQuantizer(HfQuantizer):group_size = config["group_size"]data_type = config["data_type"]sym = config["sym"]+ clip = config["clip"]if not (bits <= 8):continue@@ -402,14 +401,25 @@ class AutoRoundQuantizer(HfQuantizer):in_features = layer.weight.shape[0]out_features = layer.weight.shape[1]bias = layer.bias is not None- new_layer = QuantLinear( # pylint: disable=E1123- bits,- group_size,- in_features,- out_features,- bias,- weight_dtype=layer.weight.dtype,- )+ try:+ new_layer = QuantLinear( # pylint: disable=E1123+ bits,+ group_size,+ in_features,+ out_features,+ bias,+ weight_dtype=layer.weight.dtype,+ clip = clip+ )+ except:+ new_layer = QuantLinear( # pylint: disable=E1123+ bits,+ group_size,+ in_features,+ out_features,+ bias,+ weight_dtype=layer.weight.dtype,+ )new_layer.device = deviceset_module(module, layer_name, new_layer)diff --git a/auto_round_extension/cuda/qliner_exllamav2.py b/auto_round_extension/cuda/qliner_exllamav2.pyindex 3f98700d7..3bc415971 100644--- a/auto_round_extension/cuda/qliner_exllamav2.py+++ b/auto_round_extension/cuda/qliner_exllamav2.py@@ -40,7 +40,6 @@ from logging import getLoggerimport torchimport torch.nn as nn-logger = getLogger(__name__)try:@@ -48,12 +47,14 @@ try:except ImportError as e:exllama_v2_import_exception = e+def error_raiser_exllama(*args, **kwargs):raise ValueError(f"Trying to use the exllama v2 backend, but could not import the C++/CUDA dependencies with the following "f"error: {exllama_v2_import_exception}")+make_q_matrix = error_raiser_exllamagemm_half_q_half = error_raiser_exllama@@ -167,7 +168,7 @@ class QuantLinear(nn.Module):self.bits = bitsself.group_size = group_size if group_size != -1 else infeaturesself.trainable = trainable- self.maxq = 2**self.bits - 1+ self.maxq = 2 ** self.bits - 1assert infeatures % 32 == 0assert infeatures % self.group_size == 0@@ -204,6 +205,7 @@ class QuantLinear(nn.Module):self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))else:self.bias = None+ self.clip = kwargs.get("clip", False)def post_init(self, temp_dq):assert self.qweight.device.type == "cuda"@@ -218,6 +220,7 @@ class QuantLinear(nn.Module):self.q_handle = ext_make_q_matrix(self.q_tensors, temp_dq)def forward(self, x, force_cuda=False):+ orig_dtype = x.dtypeif x.dtype != torch.float16:logger.warning_once(f"The exllama v2 kernel for GPTQ requires a float16 input activation, while {x.dtype} was passed. "@@ -229,9 +232,12 @@ class QuantLinear(nn.Module):x = x.half()output = ext_gemm_half_q_half(x, self.q_handle, self.outfeatures, force_cuda)-+ if orig_dtype != torch.float16:+ output = output.to(orig_dtype)if self.bias is not None:output.add_(self.bias)+ if self.clip:+ output = torch.clip(output, -65504, 65504)return outputdef temp_dq_size(self):