Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix ut issue (#668)

Weiwei0a8ebdfbe132
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index fabc1ef2e..7169e5c56 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -472,8 +472,8 @@ class AutoRound(object):
)
##check group_size 32 for auto_round
- if self.data_type == "int" and hasattr(self, "formats") and (
- "auto_round" in self.formats or "auto_gptq" in self.formats or "auto_awq" in self.formats):
+ if self.data_type == "int" and hasattr(self, "formats") and any(
+ key in fmt for fmt in self.formats for key in ("auto_round", "auto_gptq", "auto_awq")):
for n, m in self.model.named_modules():
if isinstance(m, self.supported_types):
if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
@@ -559,7 +559,7 @@ class AutoRound(object):
for index in range(len(formats)):
format = formats[index]
if format == "auto_round":
- if self.sym or self.bits == 3 and "int" in self.data_type:
+ if (self.sym or self.bits == 3) and "int" in self.data_type:
format = format.replace('auto_round', 'auto_round:auto_gptq')
formats[index] = format
if self.bits == 4 and not self.sym and "int" in self.data_type:
@@ -1210,7 +1210,6 @@ class AutoRound(object):
all_to_quantized_module_names.remove(m.tmp_name)
mv_module_from_gpu(block, self.low_cpu_mem_usage)
- clear_memory()
pbar.update(1)
pbar.close()
@@ -2934,3 +2933,4 @@ class AutoRoundAdam(AutoRoundOPT):
super_group_size=super_group_size,
**kwargs,
)
+
diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.py
index 831a23cfe..066ba5beb 100644
--- a/auto_round_extension/torch/qlinear_torch.py
+++ b/auto_round_extension/torch/qlinear_torch.py
@@ -37,10 +37,6 @@ class QuantLinear(nn.Module):
super().__init__()
if bits not in [2, 3, 4, 8]:
raise NotImplementedError("Only 2,3,4,8 bits are supported.")
- if infeatures % 32 != 0 or outfeatures % 32 != 0:
- raise NotImplementedError(
- "in_feature and out_feature must be divisible by 32."
- )
self.infeatures = infeatures
self.outfeatures = outfeatures
self.bits = bits
@@ -282,6 +278,7 @@ class QuantLinear(nn.Module):
repeat_zeros = zeros.repeat_interleave(self.group_size, dim=0)
weights = repeat_scales * (weight - repeat_zeros)
+ weights = weights.to(x_dtype)
out = torch.matmul(x, weights)
out = out.to(x_dtype)
out = out.reshape(out_shape)
@@ -290,3 +287,4 @@ class QuantLinear(nn.Module):
__all__ = ["QuantLinear"]
+
diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.py
index 0d4461d80..a274f57be 100644
--- a/auto_round_extension/torch/qlinear_torch_zp.py
+++ b/auto_round_extension/torch/qlinear_torch_zp.py
@@ -38,10 +38,6 @@ class QuantLinear(nn.Module):
super().__init__()
if bits not in [2, 3, 4, 8]:
raise NotImplementedError("Only 2,3,4,8 bits are supported.")
- if infeatures % 32 != 0 or outfeatures % 32 != 0:
- raise NotImplementedError(
- "in_feature and out_feature must be divisible by 32."
- )
self.infeatures = infeatures
self.outfeatures = outfeatures
self.bits = bits
@@ -275,6 +271,7 @@ class QuantLinear(nn.Module):
repeat_zeros = zeros.repeat_interleave(self.group_size, dim=0)
weights = repeat_scales * (weight - repeat_zeros)
+ weights = weights.to(x_dtype)
out = torch.matmul(x, weights)
out = out.to(x_dtype)
out = out.reshape(out_shape)
@@ -283,3 +280,4 @@ class QuantLinear(nn.Module):
__all__ = ["QuantLinear"]
+