PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix exlllamav2 backend issue (#144)
wenhuach21e614d138723d
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex 84aaa5e73..cb7ebee01 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -255,7 +255,7 @@ class AutoRoundQuantizer(HfQuantizer):def __init__(self, quantization_config: QuantizationConfigMixin, **kwargs):super().__init__(quantization_config, **kwargs)- self.exllama2_available = is_autoround_exllamav2_available+ self.exllama2_available = is_autoround_exllamav2_available()def validate_environment(self, *args, **kwargs):if not is_auto_round_available():@@ -311,7 +311,7 @@ class AutoRoundQuantizer(HfQuantizer):return modeldef _dynamic_import_inference_linear(self, bits, backend):- if bits == 4 and self.exllama2_available and "exllama2" in backend:+ if bits == 4 and self.exllama2_available and "exllamav2" in backend:from auto_round_extension.cuda.qliner_exllamav2 import QuantLinearelse:from auto_round_extension.cuda.qliner_triton import QuantLineardiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 375da000c..127ca52cb 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -70,14 +70,13 @@ class WrapperLinear(torch.nn.Module):self.scale_dtype = self.orig_layer.scale_dtypeself.sym = self.orig_layer.symweight_dtype = self.orig_layer.weight.dtype- weight_dtype = torch.float32 ##TODO revert the change to check the accuracy+ weight_dtype = torch.float32self.value = torch.nn.Parameter(torch.zeros(self.orig_layer.weight.shape, device=self.orig_layer.weight.device, dtype=weight_dtype),requires_grad=True,)self.enable_minmax_tuning = enable_minmax_tuningshape = get_scale_shape(self.orig_layer.weight, self.group_size)- weight_dtype = self.orig_layer.weight.dtypeif self.enable_minmax_tuning:self.min_scale = torch.nn.Parameter(torch.zeros(shape, device=self.orig_layer.weight.device, dtype=weight_dtype), requires_grad=True@@ -178,8 +177,7 @@ class WrapperTransformerConv1d(torch.nn.Module):self.sym = self.orig_layer.symself.scale_dtype = self.orig_layer.scale_dtypeweight_dtype = self.orig_layer.weight.dtype- weight_dtype = torch.float32 ##TODO revert the change to check the accuracy-+ weight_dtype = torch.float32device = self.orig_layer.weight.deviceself.weight_t = self.orig_layer.weight.t()self.value = torch.nn.Parameter(