PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix qbits issue (#153)
wenhuach21c313fa33e22d
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex e0a6b03d1..8a4952fc4 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -317,6 +317,13 @@ class AutoRoundQuantizer(HfQuantizer):return modeldef _dynamic_import_inference_linear(self, bits, backend):+ if (not torch.cuda.is_available()) or "qbits" in backend or "cpu" in backend:+ try:+ from intel_extension_for_transformers import qbits # pylint: disable=E0401+ except Exception as e:+ raise ImportError("Please install Intel Extension for Transformers via 'pip install "+ "intel-extension-for-transformers' to inference on X86 CPU")+ return qlinear_qbits.QuantLinearif bits == 4 and self.exllama2_available and "exllamav2" in backend:from auto_round_extension.cuda.qliner_exllamav2 import QuantLinearelse:@@ -341,9 +348,10 @@ class AutoRoundQuantizer(HfQuantizer):data_type = config["data_type"]if not (bits <= 8 and data_type == "int"):continue- QuantLinear = self._dynamic_import_inference_linear(bits, backend)+layer = get_module(module, layer_name)device = get_device(layer)+ QuantLinear = self._dynamic_import_inference_linear(bits, backend)if isinstance(layer, nn.Linear):in_features = layer.in_featuresout_features = layer.out_features@@ -363,24 +371,13 @@ class AutoRoundQuantizer(HfQuantizer):weight_dtype=layer.weight.dtype,)- if new_layer.qweight.device.type == "cpu": # fallback to qbits linear when qweight on cpu device- QuantLinear = qlinear_qbits.QuantLinear- new_layer = QuantLinear( # pylint: disable=E1123- bits,- group_size,- in_features,- out_features,- bias,- weight_dtype=layer.weight.dtype,- )-new_layer.device = deviceset_module(module, layer_name, new_layer)def qbits_post_init(self, model):dep_check = Truefor layer in model.modules():- if isinstance(layer,qlinear_qbits.QuantLinear):+ if isinstance(layer, qlinear_qbits.QuantLinear):if dep_check:layer.req_check()layer.post_init()@@ -408,7 +405,7 @@ class AutoRoundQuantizer(HfQuantizer):model = autoround_post_init(model)# there are no side-effects after call qbits_post_init when model quant-type not equal to qbits.model = self.qbits_post_init(model)-+return modeldef _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):diff --git a/auto_round_extension/qbits/qlinear_qbits.py b/auto_round_extension/qbits/qlinear_qbits.pyindex 758e615a4..1c5679468 100644--- a/auto_round_extension/qbits/qlinear_qbits.py+++ b/auto_round_extension/qbits/qlinear_qbits.py@@ -20,12 +20,6 @@ import torchimport torch.nn as nnfrom auto_round.utils import convert_dtype_torch2str, loggerQBITS_AVAILABLE = True-try:- from intel_extension_for_transformers import qbits # noqa: F401-except Exception as e:- QBITS_AVAILABLE = False- logger.warning(- "qlinear_qbits should be used with Intel Extension for Transformers.")BITS_DTYPE_MAPPING = {2: "int2_clip",@@ -62,6 +56,7 @@ class QuantLinear(nn.Module):self.maxq = 2**self.bits - 1self.weight_dtype = weight_dtypeself.asym = True+ self.qbits = Noneself.register_buffer("qweight",@@ -98,6 +93,7 @@ class QuantLinear(nn.Module):def req_check(self):torch_version = str(torch.__version__)if QBITS_AVAILABLE:+ import intel_extension_for_transformersitrex_version = str(intel_extension_for_transformers.__version__)version_match_map = {"1.4": "2.2.0+cpu","1.4.1": "2.2.0+cpu", "1.4.2": "2.3.0+cpu"}@@ -111,6 +107,8 @@ class QuantLinear(nn.Module):exit(1)def post_init(self):+ import intel_extension_for_transformers+ self.qbits = intel_extension_for_transformers.qbitsassert self.qweight.device.type == "cpu"if self.bias is not None:self.bias = self.bias.to(dtype=torch.float32)@@ -142,7 +140,7 @@ class QuantLinear(nn.Module):logger.info(f"QBits repack quantized weight: K:{intweight.shape[0]}, N:{intweight.shape[1]}, weight_dtype:{BITS_DTYPE_MAPPING[self.bits]}, scale_dtype:fp32, compute_dtype:fp32, group_size:{self.group_size}")- self.qweight = qbits.repack_quantized_weight(intweight.contiguous(), scales.float().contiguous(), zeros.contiguous(), torch.empty(0),+ self.qweight = self.qbits.repack_quantized_weight(intweight.contiguous(), scales.float().contiguous(), zeros.contiguous(), torch.empty(0),# weight_dtypeBITS_DTYPE_MAPPING[self.bits],# scale_dtype@@ -167,7 +165,7 @@ class QuantLinear(nn.Module):bias = self.bias if self.bias is not None else torch.empty(0, dtype=torch.float)- qbits.woq_linear(x, self.qweight, bias, outputs,+ self.qbits.woq_linear(x, self.qweight, bias, outputs,convert_dtype_torch2str(torch.float), # compute_dtypeBITS_DTYPE_MAPPING[self.bits], # weight_dtype"fp32", # scale_dtype