Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

11.3 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Packers must clamp quantized levels into [0, maxq] before bit-packing.

round(W/s + zp) can exceed the level range whenever the searched (scale, zp)
grid does not fully cover the weight range (the search itself evaluates the
clamped loss). Packing the unclamped value shifts extra bits into the
NEIGHBORING packed nibble -- silently corrupting it -- so the packed artifact
no longer reproduces the clamped fake-quantization the search optimized.
"""

from types import SimpleNamespace

import pytest
import torch
from torch import nn

from auto_round.export.export_to_awq.utils import WQLinear_GEMM
from auto_round_extension.torch.qlinear_torch import QuantLinear as QuantLinearDirect
from auto_round_extension.torch.qlinear_torch_zp import QuantLinear as QuantLinearMinusOne

OUT_F, IN_F, GROUP = 64, 64, 32


def _make_inputs(bits, seed=0, with_overshoot=True):
    g = torch.Generator().manual_seed(seed)
    w = torch.randn(OUT_F, IN_F, generator=g) * 0.05
    scale = torch.rand(OUT_F, IN_F // GROUP, generator=g) * 0.005 + 0.02
    zp = torch.randint(0, 2**bits, (OUT_F, IN_F // GROUP), generator=g).float()
    if with_overshoot:
        # force q = round(w/s + zp) far outside [0, maxq] on dedicated rows:
        # scales ~0.02..0.025, so +/-3.0 / 0.02 ~ +/-150 levels of overshoot
        w[0, :] = 3.0
        w[1, :] = -3.0
        zp[0, :] = 0  # negative side undershoots as well
    return w, scale, zp


def _unpack_qweight(qlayer):
    bits = qlayer.bits
    npk = 32 // bits
    maxq = 2**bits - 1
    wf = torch.arange(0, 32, bits, dtype=torch.int32)
    w_int = qlayer.qweight.unsqueeze(1).expand(-1, npk, -1) >> wf.unsqueeze(-1)
    w_int = (w_int & maxq).reshape(-1, qlayer.outfeatures)  # [in, out]
    return w_int.t().to(torch.int64)  # [out, in]


def _unpack_qzeros(qlayer):
    bits = qlayer.bits
    npk = 32 // bits
    maxq = 2**bits - 1
    wf = torch.arange(0, 32, bits, dtype=torch.int32)
    z = qlayer.qzeros.unsqueeze(2).expand(-1, -1, npk) >> wf.unsqueeze(0)
    return (z & maxq).to(torch.int64)  # [in/g, out/npk-1 packed] -> per nibble [in/g, out]


@pytest.mark.parametrize("bits", [2, 4, 8])
def test_pack_clamps_levels_direct_zp(bits):
    torch.manual_seed(0)
    w, scale, zp = _make_inputs(bits)
    q = QuantLinearDirect(bits, GROUP, IN_F, OUT_F, bias=False)
    q.device = torch.device("cpu")
    q.pack(SimpleNamespace(weight=w, bias=None), scale, zp, None, device="cpu")

    expected = torch.clamp(
        torch.round(w / scale.repeat_interleave(GROUP, 1) + zp.repeat_interleave(GROUP, 1)), 0, 2**bits - 1
    ).to(torch.int64)
    got = _unpack_qweight(q)
    assert got.min() >= 0 and got.max() <= 2**bits - 1
    assert torch.equal(got, expected), "packed levels must equal the clamped fake-quantization"


@pytest.mark.parametrize("bits", [2, 4, 8])
def test_pack_clamps_levels_minus_one_zp(bits):
    torch.manual_seed(0)
    w, scale, zp = _make_inputs(bits)
    q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)
    q.device = torch.device("cpu")
    q.pack(SimpleNamespace(weight=w, bias=None), scale, zp, None, device="cpu")

    expected = torch.clamp(
        torch.round(w / scale.repeat_interleave(GROUP, 1) + zp.repeat_interleave(GROUP, 1)), 0, 2**bits - 1
    ).to(torch.int64)
    assert torch.equal(_unpack_qweight(q), expected)


def test_minus_one_zp_packing_survives_zero_zero_point():
    """zp=0 wraps to -1 under the classic (zp-1) packing; the raw -1 word
    corrupts every nibble sharing its packed word. The stored value must be
    clamped so only that one zero-point degrades (to a decodable +1)."""
    torch.manual_seed(0)
    bits = 4
    w, scale, zp = _make_inputs(bits, with_overshoot=False)
    zp[:, :] = 0  # every group sits at the unrepresentable zero-point
    q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)
    q.device = torch.device("cpu")
    q.pack(SimpleNamespace(weight=w, bias=None), scale, zp, None, device="cpu")

    z_nibbles = _unpack_qzeros(q)  # [in/g, out] per-nibble values
    # pre-fix: (0-1) << shift makes whole words -1 -> every nibble reads maxq
    assert z_nibbles.max() <= 1, "zp=0 must not smear garbage across the packed word"
    # neighbors of a clamped cell keep their own packed value: all nibbles equal
    # either 0 (the clamped zp-1) -- uniform here by construction
    assert (z_nibbles == 0).all()
    # and the module still dequantizes (forward runs, no all-ones corruption)
    out = q(torch.eye(IN_F))
    assert torch.isfinite(out).all()


def test_awq_pack_clamps_levels():
    """The AWQ packer shares the unclamped-round spill bug: an out-of-range
    level shifts bits into the neighboring packed nibble (AWQ interleave
    order), corrupting it. Packed levels must equal the clamped grid."""
    torch.manual_seed(0)
    bits, maxq = 4, 15
    w, scale, zp = _make_inputs(bits)  # [out, in], [out, in/g], [out, in/g]
    lin = nn.Linear(IN_F, OUT_F, bias=False)
    with torch.no_grad():
        lin.weight.copy_(w)
    q = WQLinear_GEMM.from_linear(lin, bits, GROUP, scales=scale.t(), zeros=zp.t(), device="cpu")

    expected_q = torch.clamp(
        torch.round(w / scale.repeat_interleave(GROUP, 1) + zp.repeat_interleave(GROUP, 1)), 0, maxq
    ).to(torch.int32)

    def _pack_rows(t):  # mirrors the packer: nibbles along dim 1, AWQ interleave
        pack_num = 32 // bits
        order = torch.tensor([0, 4, 1, 5, 2, 6, 3, 7]) * bits
        packed = t.reshape(t.shape[0], t.shape[1] // pack_num, pack_num) << order
        return torch.sum(packed, dim=-1).to(torch.int32)

    # qweight packs q transposed ([in, out/8], nibbles along the output dim);
    # qzeros pack zp along the output dim ([in/g, out/8])
    assert torch.equal(q.qweight, _pack_rows(expected_q.t().contiguous()))
    assert torch.equal(q.qzeros, _pack_rows(zp.to(torch.int32).t().contiguous()))


def test_direct_zp_roundtrip_in_range():
    """Sanity: for in-range data the packed module reproduces clamped
    fake-quantization bit-exactly (clamp must be a no-op there)."""
    torch.manual_seed(0)
    bits, maxq = 4, 15
    w, scale, zp = _make_inputs(bits, with_overshoot=False)
    # keep q in-range: shrink weights relative to scale
    w = w * 0.05
    q = QuantLinearDirect(bits, GROUP, IN_F, OUT_F, bias=False)
    q.device = torch.device("cpu")
    q.pack(SimpleNamespace(weight=w, bias=None), scale, zp, None, device="cpu")
    s16 = scale.half().float().repeat_interleave(GROUP, 1)
    zrep = zp.repeat_interleave(GROUP, 1)
    expected = torch.clamp(torch.round(w / s16 + zrep), 0, maxq).to(torch.int64)
    assert torch.equal(_unpack_qweight(q), expected)
    # dequantized weights match the reference reconstruction
    ref_w = (expected.to(torch.float32) - zrep) * s16
    got_w = q(torch.eye(IN_F)).t()
    torch.testing.assert_close(got_w, ref_w, rtol=0, atol=1e-3)


def test_minus_one_scalar_zp_packing_survives_zero_zero_point():
    """The scalar zero-point path of the classic (zp-1) packer must clamp like
    the tensor path: an unclamped 0-1 = -1 repeated into every packed word
    corrupts all nibbles; the clamped value degrades that one zp to +1."""
    torch.manual_seed(0)
    bits = 4
    w, scale, _ = _make_inputs(bits, with_overshoot=False)
    q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)
    q.device = torch.device("cpu")
    q.pack(SimpleNamespace(weight=w, bias=None), scale, 0.0, None, device="cpu")

    npk = 32 // bits
    maxq = 2**bits - 1
    wf = torch.arange(0, 32, bits, dtype=torch.int32)
    z = q.qzeros.unsqueeze(2).expand(-1, -1, npk) >> wf.unsqueeze(0)
    z_nibbles = (z & maxq).to(torch.int64)
    # pre-fix: (0-1) built a -1 repeated word -> every nibble reads maxq
    assert (z_nibbles == 0).all(), "scalar zp=0 must clamp to a decodable stored 0"
    out = q(torch.eye(IN_F))
    assert torch.isfinite(out).all()


def test_missing_tensors_gptq_pack_clamps_zero_zero_point():
    """quantize_weight_rtn packs zp with the same (zp-1) convention; an
    all-positive weight yields zp=0 everywhere, which must clamp instead of
    smearing -1 across the packed zero-point words."""
    from auto_round.utils.model_free_utils import quantize_weight_rtn

    torch.manual_seed(0)
    bits, group = 4, 32
    w = torch.rand(16, 64) + 0.1  # strictly positive -> asym zp = 0 per group
    qweight, qzeros, scales = quantize_weight_rtn(
        w, bits=bits, group_size=group, sym=False, packing="auto_round:auto_gptq"
    )

    npk = 32 // bits
    maxq = 2**bits - 1
    wf = torch.arange(0, 32, bits, dtype=torch.int32)
    z = qzeros.unsqueeze(2).expand(-1, -1, npk) >> wf.unsqueeze(0)
    z_nibbles = (z & maxq).to(torch.int64)
    # pre-fix: zp-1 = -1 -> whole int32 words negative -> nibbles read maxq
    assert (z_nibbles == 0).all(), "zp=0 must pack as the clamped 0 (decodes +1)"
    # scales/q stay well-formed
    assert torch.isfinite(scales).all() and scales.shape == (w.shape[1] // group, w.shape[0])


def test_minus_one_3bit_scalar_zp_clamped_both_ends():
    """The 3-bit scalar zero-point path must clamp at both ends: zp=0 packs a
    clean all-zero word (no -1 smear), and an oversized scalar (20 > maxq 7)
    packs bit-identically to the in-range zp=8 (stored 7) instead of spilling
    into neighboring packed fields."""
    bits = 3

    def _pack_with_scalar(scalar):
        torch.manual_seed(0)
        q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)
        q.device = torch.device("cpu")
        w, scale, _ = _make_inputs(bits, with_overshoot=False)
        q.pack(SimpleNamespace(weight=w, bias=None), scale, scalar, None, device="cpu")
        return q

    q0 = _pack_with_scalar(0.0)
    assert (q0.qzeros == 0).all(), "scalar zp=0 must clamp to a stored 0 word"
    assert torch.isfinite(q0(torch.eye(IN_F))).all()

    q_over = _pack_with_scalar(20.0)
    q_ref = _pack_with_scalar(8.0)  # in-range zero point storing 7
    assert torch.equal(
        q_over.qzeros, q_ref.qzeros
    ), "an oversized scalar zero point must clamp to the maxq word, not spill"


@pytest.mark.parametrize("bits", [2, 4, 8])
def test_minus_one_scalar_zp_packs_float_scalars(bits):
    """A float scalar zero point (the natural minmax output) must survive the
    2/4/8-bit classic (zp-1) packing: normalized to int, clamped at both
    ends. zp=1.0 stores 0; an oversized scalar stores maxq."""
    maxq = 2**bits - 1

    def _pack(scalar):
        q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)
        q.device = torch.device("cpu")
        w, scale, _ = _make_inputs(bits, with_overshoot=False)
        torch.manual_seed(0)
        q.pack(SimpleNamespace(weight=w, bias=None), scale, scalar, None, device="cpu")
        return q

    q1 = _pack(1.0)
    q_over = _pack(float(maxq + 5))
    q_ref = _pack(float(maxq + 1))  # in-range zp storing maxq
    assert torch.isfinite(q1(torch.eye(IN_F))).all(), "float scalar zp=1.0 must pack, not crash"
    assert torch.equal(q_over.qzeros, q_ref.qzeros), "oversized float scalar must clamp to maxq"