Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Move the zero point to CPU for asymmetric layers too (#2394)

Shubham Padkonde89c3dfe97f07
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index 127b3f36b..90be0c0d9 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -166,11 +166,13 @@ def pack_layer(name, model, backend, device=None):
# so far can only pack layer on CPU
qlayer.to("cpu")
##force to float32 to be compatible with torch 2.0
- if sym and isinstance(zero, torch.Tensor):
- layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu")
- zero = int(zero.flatten()[0])
- else:
- layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero
+ layer, scale = layer.to("cpu"), scale.to("cpu")
+ if isinstance(zero, torch.Tensor):
+ # Packing runs on CPU, so the zero point has to follow the weight and
+ # the scale there, whether or not the layer is symmetric.
+ zero = zero.to("cpu")
+ if sym:
+ zero = int(zero.flatten()[0])
if isinstance(zero, torch.Tensor) and zero.dtype == torch.bfloat16:
zero = zero.float()
sig = inspect.signature(qlayer.pack)
diff --git a/test/unit/common/export/test_export_autogptq_export.py b/test/unit/common/export/test_export_autogptq_export.py
index 997f11dd6..8de65c0e6 100644
--- a/test/unit/common/export/test_export_autogptq_export.py
+++ b/test/unit/common/export/test_export_autogptq_export.py
@@ -13,6 +13,10 @@
# limitations under the License.
"""Tests for ``auto_round/export/export_to_autogptq/export.py``."""
+import pytest
+import torch
+
+from auto_round.export.export_to_autogptq import export as autogptq_export
from auto_round.export.export_to_autogptq.export import (
BLOCK_PATTERNS,
GPTQ_REQUIRED_CONFIG_KEYS,
@@ -126,3 +130,65 @@ class TestConvertFromAutogptqDynamic:
def test_empty(self):
assert convert_from_autogptq_dynamic({}) == {}
+
+
+# ---------------------------------------------------------------------------
+# pack_layer
+# ---------------------------------------------------------------------------
+class TestPackLayerDevices:
+ """Packing runs on CPU, so every tensor handed to ``pack`` must be there."""
+
+ @staticmethod
+ def _build_model(sym: bool):
+ layer = torch.nn.Linear(8, 8, bias=False)
+ layer.bits = 4
+ layer.group_size = 8
+ layer.sym = sym
+ layer.scale = torch.ones(8, 1)
+ layer.zp = torch.zeros(8, 1)
+
+ model = torch.nn.Module()
+ model.add_module("linear", layer)
+ return model, layer
+
+ @staticmethod
+ def _stub_qlinear(recorded):
+ class _StubQuantLinear(torch.nn.Module):
+ def __init__(self, bits, group_size, in_features, out_features, bias, g_idx=True):
+ super().__init__()
+ self.weight = torch.nn.Parameter(torch.zeros(out_features, in_features), requires_grad=False)
+
+ def pack(self, layer, scale, zero, g_idx=None, device=None):
+ recorded["zero"] = zero
+
+ return _StubQuantLinear
+
+ @pytest.mark.parametrize("sym", [True, False])
+ def test_zero_point_is_moved_to_cpu(self, monkeypatch, sym):
+ model, layer = self._build_model(sym)
+ recorded = {}
+
+ monkeypatch.setattr(
+ autogptq_export,
+ "get_autogptq_packing_qlinear",
+ lambda *args, **kwargs: self._stub_qlinear(recorded),
+ )
+
+ moved_to_cpu = []
+ original_to = torch.Tensor.to
+
+ def spy_to(self, *args, **kwargs):
+ if self is layer.zp and args and args[0] == "cpu":
+ moved_to_cpu.append(True)
+ return original_to(self, *args, **kwargs)
+
+ monkeypatch.setattr(torch.Tensor, "to", spy_to)
+
+ autogptq_export.pack_layer("linear", model, backend="auto_gptq:exllamav2", device="cpu")
+
+ assert moved_to_cpu, "the zero point was never moved to CPU"
+ zero = recorded["zero"]
+ if sym:
+ assert isinstance(zero, int)
+ else:
+ assert zero.device.type == "cpu"