PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix ut issue (#668)
Weiwei0a8ebdfbe132
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex fabc1ef2e..7169e5c56 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -472,8 +472,8 @@ class AutoRound(object):)##check group_size 32 for auto_round- if self.data_type == "int" and hasattr(self, "formats") and (- "auto_round" in self.formats or "auto_gptq" in self.formats or "auto_awq" in self.formats):+ if self.data_type == "int" and hasattr(self, "formats") and any(+ key in fmt for fmt in self.formats for key in ("auto_round", "auto_gptq", "auto_awq")):for n, m in self.model.named_modules():if isinstance(m, self.supported_types):if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:@@ -559,7 +559,7 @@ class AutoRound(object):for index in range(len(formats)):format = formats[index]if format == "auto_round":- if self.sym or self.bits == 3 and "int" in self.data_type:+ if (self.sym or self.bits == 3) and "int" in self.data_type:format = format.replace('auto_round', 'auto_round:auto_gptq')formats[index] = formatif self.bits == 4 and not self.sym and "int" in self.data_type:@@ -1210,7 +1210,6 @@ class AutoRound(object):all_to_quantized_module_names.remove(m.tmp_name)mv_module_from_gpu(block, self.low_cpu_mem_usage)- clear_memory()pbar.update(1)pbar.close()@@ -2934,3 +2933,4 @@ class AutoRoundAdam(AutoRoundOPT):super_group_size=super_group_size,**kwargs,)+diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.pyindex 831a23cfe..066ba5beb 100644--- a/auto_round_extension/torch/qlinear_torch.py+++ b/auto_round_extension/torch/qlinear_torch.py@@ -37,10 +37,6 @@ class QuantLinear(nn.Module):super().__init__()if bits not in [2, 3, 4, 8]:raise NotImplementedError("Only 2,3,4,8 bits are supported.")- if infeatures % 32 != 0 or outfeatures % 32 != 0:- raise NotImplementedError(- "in_feature and out_feature must be divisible by 32."- )self.infeatures = infeaturesself.outfeatures = outfeaturesself.bits = bits@@ -282,6 +278,7 @@ class QuantLinear(nn.Module):repeat_zeros = zeros.repeat_interleave(self.group_size, dim=0)weights = repeat_scales * (weight - repeat_zeros)+ weights = weights.to(x_dtype)out = torch.matmul(x, weights)out = out.to(x_dtype)out = out.reshape(out_shape)@@ -290,3 +287,4 @@ class QuantLinear(nn.Module):__all__ = ["QuantLinear"]+diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.pyindex 0d4461d80..a274f57be 100644--- a/auto_round_extension/torch/qlinear_torch_zp.py+++ b/auto_round_extension/torch/qlinear_torch_zp.py@@ -38,10 +38,6 @@ class QuantLinear(nn.Module):super().__init__()if bits not in [2, 3, 4, 8]:raise NotImplementedError("Only 2,3,4,8 bits are supported.")- if infeatures % 32 != 0 or outfeatures % 32 != 0:- raise NotImplementedError(- "in_feature and out_feature must be divisible by 32."- )self.infeatures = infeaturesself.outfeatures = outfeaturesself.bits = bits@@ -275,6 +271,7 @@ class QuantLinear(nn.Module):repeat_zeros = zeros.repeat_interleave(self.group_size, dim=0)weights = repeat_scales * (weight - repeat_zeros)+ weights = weights.to(x_dtype)out = torch.matmul(x, weights)out = out.to(x_dtype)out = out.reshape(out_shape)@@ -283,3 +280,4 @@ class QuantLinear(nn.Module):__all__ = ["QuantLinear"]+