PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Move the zero point to CPU for asymmetric layers too (#2394)
Shubham Padkonde89c3dfe97f07
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 127b3f36b..90be0c0d9 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -166,11 +166,13 @@ def pack_layer(name, model, backend, device=None):# so far can only pack layer on CPUqlayer.to("cpu")##force to float32 to be compatible with torch 2.0- if sym and isinstance(zero, torch.Tensor):- layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu")- zero = int(zero.flatten()[0])- else:- layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero+ layer, scale = layer.to("cpu"), scale.to("cpu")+ if isinstance(zero, torch.Tensor):+ # Packing runs on CPU, so the zero point has to follow the weight and+ # the scale there, whether or not the layer is symmetric.+ zero = zero.to("cpu")+ if sym:+ zero = int(zero.flatten()[0])if isinstance(zero, torch.Tensor) and zero.dtype == torch.bfloat16:zero = zero.float()sig = inspect.signature(qlayer.pack)diff --git a/test/unit/common/export/test_export_autogptq_export.py b/test/unit/common/export/test_export_autogptq_export.pyindex 997f11dd6..8de65c0e6 100644--- a/test/unit/common/export/test_export_autogptq_export.py+++ b/test/unit/common/export/test_export_autogptq_export.py@@ -13,6 +13,10 @@# limitations under the License."""Tests for ``auto_round/export/export_to_autogptq/export.py``."""+import pytest+import torch++from auto_round.export.export_to_autogptq import export as autogptq_exportfrom auto_round.export.export_to_autogptq.export import (BLOCK_PATTERNS,GPTQ_REQUIRED_CONFIG_KEYS,@@ -126,3 +130,65 @@ class TestConvertFromAutogptqDynamic:def test_empty(self):assert convert_from_autogptq_dynamic({}) == {}+++# ---------------------------------------------------------------------------+# pack_layer+# ---------------------------------------------------------------------------+class TestPackLayerDevices:+ """Packing runs on CPU, so every tensor handed to ``pack`` must be there."""++ @staticmethod+ def _build_model(sym: bool):+ layer = torch.nn.Linear(8, 8, bias=False)+ layer.bits = 4+ layer.group_size = 8+ layer.sym = sym+ layer.scale = torch.ones(8, 1)+ layer.zp = torch.zeros(8, 1)++ model = torch.nn.Module()+ model.add_module("linear", layer)+ return model, layer++ @staticmethod+ def _stub_qlinear(recorded):+ class _StubQuantLinear(torch.nn.Module):+ def __init__(self, bits, group_size, in_features, out_features, bias, g_idx=True):+ super().__init__()+ self.weight = torch.nn.Parameter(torch.zeros(out_features, in_features), requires_grad=False)++ def pack(self, layer, scale, zero, g_idx=None, device=None):+ recorded["zero"] = zero++ return _StubQuantLinear++ @pytest.mark.parametrize("sym", [True, False])+ def test_zero_point_is_moved_to_cpu(self, monkeypatch, sym):+ model, layer = self._build_model(sym)+ recorded = {}++ monkeypatch.setattr(+ autogptq_export,+ "get_autogptq_packing_qlinear",+ lambda *args, **kwargs: self._stub_qlinear(recorded),+ )++ moved_to_cpu = []+ original_to = torch.Tensor.to++ def spy_to(self, *args, **kwargs):+ if self is layer.zp and args and args[0] == "cpu":+ moved_to_cpu.append(True)+ return original_to(self, *args, **kwargs)++ monkeypatch.setattr(torch.Tensor, "to", spy_to)++ autogptq_export.pack_layer("linear", model, backend="auto_gptq:exllamav2", device="cpu")++ assert moved_to_cpu, "the zero point was never moved to CPU"+ zero = recorded["zero"]+ if sym:+ assert isinstance(zero, int)+ else:+ assert zero.device.type == "cpu"