Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

13.0 KB
Raw Blame
from types import SimpleNamespace
from unittest.mock import patch

import pytest
import torch
from transformers.quantizers.auto import AutoHfQuantizer

from auto_round.data_type.nvfp import (
    calculate_gparam,
    calculate_gparam_e5m3,
    nvfp4_v2,
    nvfp4_v2_gs,
    nvfp4_v2_with_static_gs,
)
from auto_round.data_type.utils import get_quant_func
from auto_round.experimental import qmodules as ar_qmodules
from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _FPLinear
from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization
from auto_round.export.formats import BackendDataType
from auto_round.schemes import PRESET_SCHEMES

QMODULE_MAPPING = {
    BackendDataType.NVFP4.value: ar_qmodules.NVFP4QuantLinear,
}


@pytest.fixture(autouse=True)
def fixed_seed():
    """Ensure deterministic RNG for every test."""
    seed = 42
    print("\nSetting fixed random seed for test:", seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False
    yield
    # (Optional) cleanup or reset after test


def test_calculate_gparam_with_float8_input():
    tensor = torch.tensor([[-2.0, 0.0, 1.0]], dtype=torch.float32).to(torch.float8_e4m3fn)

    global_scale = calculate_gparam(tensor)

    assert global_scale.dtype == torch.float32
    assert torch.isfinite(global_scale)


def test_nvfp4_e5m3_compressed_tensors_loading_uses_no_global_scales():
    class TinyModel(torch.nn.Module):
        def __init__(self):
            super().__init__()
            self.config = SimpleNamespace(model_type="tiny")
            self.model = torch.nn.Module()
            self.model.layers = torch.nn.ModuleList([torch.nn.ModuleDict({"fc": torch.nn.Linear(16, 8)})])

    quantizer = AutoHfQuantizer.from_config(initialize_nvfp4_e5m3_quantization([]))
    model = quantizer._process_model_before_weight_loading(TinyModel())
    layer = model.model.layers[0]["fc"]

    assert isinstance(layer, ar_qmodules.NVFP4E5M3QuantLinear)
    assert set(layer.state_dict()) == {"bias", "weight_packed", "weight_scale"}
    assert quantizer._process_model_after_weight_loading(model) is model


def test_nvfp4_e5m3_gs_compressed_tensors_loading_uses_global_scales():
    class TinyModel(torch.nn.Module):
        def __init__(self):
            super().__init__()
            self.config = SimpleNamespace(model_type="tiny")
            self.model = torch.nn.Module()
            self.model.layers = torch.nn.ModuleList([torch.nn.ModuleDict({"fc": torch.nn.Linear(16, 8)})])

    quantizer = AutoHfQuantizer.from_config(initialize_nvfp4_e5m3_quantization([], with_global_scale=True))
    model = quantizer._process_model_before_weight_loading(TinyModel())
    layer = model.model.layers[0]["fc"]

    assert isinstance(layer, ar_qmodules.NVFP4E5M3GSQuantLinear)
    assert set(layer.state_dict()) == {
        "bias",
        "weight_packed",
        "weight_scale",
        "weight_global_scale",
        "input_global_scale",
    }


@torch.inference_mode()
def test_nvfp4_e5m3_gs_pack_and_dequant_round_trip():
    config = PRESET_SCHEMES["NVFP4_E5M3_GS"]
    original_layer = torch.nn.Linear(64, 32, bias=False)
    weight_global_scale = calculate_gparam_e5m3(original_layer.weight)
    weight_qdq, weight_scale, _ = nvfp4_v2_gs(original_layer.weight, group_size=16, global_scale=weight_global_scale)
    activation = torch.randn(4, 64)
    input_global_scale = calculate_gparam_e5m3(activation)

    packed = _FPLinear(4, 16, 64, 32, False, data_type=config.data_type, act_bits=4)
    packed.pack(
        original_layer,
        weight_scale.reshape(32, 4),
        global_scale=weight_global_scale,
        input_global_scale=input_global_scale,
    )
    assert packed.weight_scale.dtype == torch.uint8

    layer = ar_qmodules.NVFP4E5M3GSQuantLinear.from_original(config, original_layer)
    for name in ("weight_packed", "weight_scale", "weight_global_scale", "input_global_scale"):
        getattr(layer, name).copy_(getattr(packed, name))

    torch.testing.assert_close(layer.dequant_weight_online(), weight_qdq, rtol=1e-6, atol=1e-7)
    expected_activation, _, _ = nvfp4_v2_with_static_gs(activation, group_size=16, tensor_max=activation.abs().max())
    torch.testing.assert_close(layer.qdq_input(activation), expected_activation)


def test_nvfp4_e5m3_gs_dequant_zero_weight_with_zero_global_scale():
    config = PRESET_SCHEMES["NVFP4_E5M3_GS"]
    layer = ar_qmodules.NVFP4E5M3GSQuantLinear(16, 8, config, dtype=torch.float32)
    layer.weight_packed.zero_()
    layer.weight_scale.zero_()
    layer.weight_global_scale.zero_()

    dequantized = layer.dequant_weight_online()

    assert torch.isfinite(dequantized).all()
    assert torch.equal(dequantized, torch.zeros_like(dequantized))


def test_nvfp4_e5m3_gs_cute_falls_back_to_reference_on_cpu():
    config = PRESET_SCHEMES["NVFP4_E5M3_GS"]
    layer = ar_qmodules.CuteNVFP4E5M3GSQuantLinear(16, 8, config, dtype=torch.float32)
    layer.input_global_scale.fill_(123.0)
    activation = torch.randn(2, 16)

    expected, _, _ = nvfp4_v2_with_static_gs(activation, group_size=16, global_scale=layer.input_global_scale)

    assert torch.equal(layer.qdq_input(activation), expected)
    assert layer(activation).shape == (2, 8)


def test_nvfp4_e5m3_qdq_input_uses_reference_fallback_on_cpu():
    config = PRESET_SCHEMES["NVFP4_E5M3"]
    layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)
    activation = torch.randn(2, 3, 16)

    expected, _, _ = nvfp4_v2(activation, bits=config.act_bits, group_size=config.act_group_size)

    assert torch.equal(layer.qdq_input(activation), expected)


def test_nvfp4_e5m3_forward_does_not_cache_dequantized_weight_by_default():
    config = PRESET_SCHEMES["NVFP4_E5M3"]
    layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)
    activation = torch.randn(2, 16)
    dequantized_weight = torch.randn(8, 16)

    with patch.object(layer, "dequant_weight_online", return_value=dequantized_weight) as dequant_weight_online:
        layer(activation)
        layer(activation)

    assert layer._cached_weight is None
    assert dequant_weight_online.call_count == 2


def test_nvfp4_e5m3_forward_caches_dequantized_weight_when_enabled():
    config = PRESET_SCHEMES["NVFP4_E5M3"]
    layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32, cache_weight=True)
    activation = torch.randn(2, 16)
    dequantized_weight = torch.randn(8, 16)

    with patch.object(layer, "dequant_weight_online", return_value=dequantized_weight) as dequant_weight_online:
        layer(activation)
        layer(activation)

    dequant_weight_online.assert_called_once_with()
    assert layer.weight_packed is None
    assert layer.weight_scale is None


def test_nvfp4_e5m3_cannot_clear_released_quantized_weight_buffers():
    config = PRESET_SCHEMES["NVFP4_E5M3"]
    layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32, cache_weight=True)

    layer(torch.randn(2, 16))

    with pytest.raises(RuntimeError, match="quantized weight buffers have been released"):
        layer.clear_weight_cache()


def test_cute_nvfp4_e5m3_does_not_cache_dequantized_weight_by_default(monkeypatch):
    monkeypatch.delenv("AR_NVFP4_E5M3_CACHE_HP_WEIGHT", raising=False)
    config = PRESET_SCHEMES["NVFP4_E5M3"]
    layer = ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)
    activation = torch.randn(2, 16)
    dequantized_weight = torch.randn(8, 16)

    with patch.object(layer, "dequant_weight_online", return_value=dequantized_weight) as dequant_weight_online:
        layer(activation)
        layer(activation)

    assert not layer.cache_weight
    assert dequant_weight_online.call_count == 2


def test_nvfp4_e5m3_cache_weight_environment_override(monkeypatch):
    config = PRESET_SCHEMES["NVFP4_E5M3"]
    monkeypatch.setenv("AR_NVFP4_E5M3_CACHE_HP_WEIGHT", "1")
    assert ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32).cache_weight
    assert ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32).cache_weight
    monkeypatch.setenv("AR_NVFP4_E5M3_CACHE_HP_WEIGHT", "0")
    assert not ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32).cache_weight


def test_nvfp4_e5m3_torch_forward_does_not_call_cute():
    config = PRESET_SCHEMES["NVFP4_E5M3"]
    layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)
    activation = torch.randn(2, 16)

    with patch(
        "auto_round.experimental.qmodules.nvfp4_e5m3.try_cute_nvfp4_e5m3_linear",
    ) as cute_linear:
        output = layer(activation)

    assert output.shape == (2, 8)
    cute_linear.assert_not_called()


def test_nvfp4_e5m3_cute_forward_uses_fused_output_when_available():
    config = PRESET_SCHEMES["NVFP4_E5M3"]
    layer = ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)
    activation = torch.randn(2, 16)
    fused_output = torch.randn(2, 8)

    with patch("auto_round.experimental.qmodules.nvfp4_e5m3.try_cute_nvfp4_e5m3_linear", return_value=fused_output):
        assert layer(activation) is fused_output


def test_cute_nvfp4_e5m3_uses_cached_weight_when_enabled():
    config = PRESET_SCHEMES["NVFP4_E5M3"]
    layer = ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32, cache_weight=True)
    activation = torch.randn(2, 16)
    dequantized_weight = torch.randn(8, 16)

    with (
        patch.object(layer, "dequant_weight_online", return_value=dequantized_weight) as dequant_weight_online,
        patch("auto_round.experimental.qmodules.nvfp4_e5m3.try_cute_nvfp4_e5m3_linear") as cute_linear,
    ):
        layer(activation)
        layer(activation)

    dequant_weight_online.assert_called_once_with()
    cute_linear.assert_not_called()


@pytest.mark.parametrize("scheme", [BackendDataType.NVFP4.value])
@torch.inference_mode()
def test_nvfp4_quantlinear_from_original_and_forward(scheme):
    """
    Test NVFP4 quantization schemes by creating quantized layers
    from an original torch.nn.Linear layer and validating their forward pass.
    """

    # Set random seed for reproducibility
    torch.manual_seed(42)

    # Define layer dimensions
    in_features = 64
    out_features = 512

    # Create an original torch.nn.Linear layer
    original_layer = torch.nn.Linear(in_features, out_features, bias=False)

    # Select the quantization scheme
    config = PRESET_SCHEMES[scheme.upper()]

    # Define weight scale shape
    weight_scale_shape = (out_features, in_features // config.group_size)

    # Quantize the weights using the quantization function
    weight_qdq_func, _ = get_quant_func(dtype=config.data_type, bits=config.bits, sym=True)

    weight_global_scale = calculate_gparam(original_layer.weight, config.group_size)
    weight_qdq, weight_scale, _ = weight_qdq_func(
        original_layer.weight, bits=config.bits, group_size=config.group_size, global_scale=weight_global_scale
    )

    # Generate a random input tensor
    input_tensor = torch.randn((4, in_features), dtype=torch.float32)
    input_global_scale = calculate_gparam(input_tensor, config.act_group_size)

    weight_scale = weight_scale.reshape(weight_scale_shape)

    # Pack the weights using the QuantLinear class
    kwargs = {"act_bits": config.act_bits}
    nvfp4_lin = _FPLinear(
        bits=config.bits,
        group_size=config.group_size,
        infeatures=in_features,
        outfeatures=out_features,
        bias=original_layer.bias is not None,
        data_type=config.data_type,
        **kwargs,
    )

    nvfp4_lin.pack(
        linear=original_layer,
        scales=weight_scale,
        global_scale=weight_global_scale,
        input_global_scale=input_global_scale,
    )

    # Create an NVFP4QuantLinear layer from the original layer
    QuantLinearClass = QMODULE_MAPPING[scheme]
    nvfp4_layer = QuantLinearClass.from_original(
        config=config,
        original_layer=original_layer,
    )

    # Copy the packed weights and scales to the quantized layer
    packed_weight = nvfp4_lin.weight_packed if config.bits == 4 else nvfp4_lin.weight
    nvfp4_layer.weight_packed.data.copy_(packed_weight)
    nvfp4_layer.weight_scale.data.copy_(nvfp4_lin.weight_scale)
    nvfp4_layer.weight_global_scale.data.copy_(nvfp4_lin.weight_global_scale)
    nvfp4_layer.input_global_scale.data.copy_(nvfp4_lin.input_global_scale)

    # Validate layer attributes
    assert nvfp4_layer.in_features == original_layer.in_features
    assert nvfp4_layer.out_features == original_layer.out_features

    # Perform a forward pass with both layers
    original_output = original_layer(input_tensor)
    nvfp4_output = nvfp4_layer(input_tensor)

    # Compute the difference between the outputs
    diff = nvfp4_output - original_output
    # Note: Remove NaN values, as we might get NaN when casting scales to FP8

    diff = diff[~torch.isnan(diff)]
    diff_amax = diff.abs().max()

    # Print the maximum difference for debugging
    print(f"Scheme: {scheme}, Max Difference: {diff_amax}")

    # Assert that the outputs are close within a tolerance
    assert diff_amax < 5e-1, f"Outputs differ too much for scheme {scheme}!"