PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
feat: handle per-tensor FP8 dequantization for Devstral models (#1356)
SwekeReddb6de2bfed
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 6caea4c1a..66af26c9e 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -1031,19 +1031,28 @@ class with_thread_limits(ContextDecorator):@with_thread_limits()-def dequant_block_fp8_weight(- weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list, data_type: str = None+def _dequant_fp8_linear_weight(+ weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list = None, data_type: str = None) -> torch.Tensor:"""Core dequantization logic for block-wise FP8 weights."""dtype = torch.bfloat16if weight_scale is None:return weight- assert len(block_size) == 2# If weight is stored as uint8, view it as float8_e4m3fnif weight.element_size() == 1 and weight.dtype != torch.float8_e4m3fn:weight = weight.view(torch.float8_e4m3fn)+ if block_size is None:+ if weight_scale.numel() > 1 and weight_scale.shape != weight.shape:+ if weight_scale.numel() == weight.shape[0]:+ weight_scale = weight_scale.view(-1, 1)+ elif weight_scale.numel() == weight.shape[-1]:+ weight_scale = weight_scale.view(1, -1)+ return weight.to(dtype) * weight_scale.to(dtype)++ assert len(block_size) == 2+weight, orig_M, orig_N = pad_block_fp8_weight_naive(weight, weight_scale, block_size)weight_shape_len = len(weight.shape)@@ -1073,7 +1082,7 @@ def dequant_block_fp8_weight(# Register FP8 layer dequantization handlers-# Note: Handlers are registered here (after dequant_block_fp8_weight is defined)+# Note: Handlers are registered here (after _dequant_fp8_linear_weight is defined)# to ensure all dependencies are available.@register_fp8_layer("CompressedLinear")def dequant_compressed_linear(layer, dtype=torch.bfloat16, device: str = "cpu"):@@ -1087,16 +1096,17 @@ def dequant_fp8_linear(layer, dtype=torch.bfloat16, device: str = "cpu"):"""Dequantize FP8Linear layer using block-based dequantization."""layer = layer.to(device)weight_scale = layer.weight_scale if hasattr(layer, "weight_scale") else layer.weight_scale_inv+ block_size = getattr(layer, "block_size", None)data_type = getattr(layer, "data_type", None)- # Pass data_type if dequant_block_fp8_weight supports it+ # Pass data_type if _dequant_fp8_linear_weight supports it# Check if function accepts data_type parameterimport inspect- sig = inspect.signature(dequant_block_fp8_weight)+ sig = inspect.signature(_dequant_fp8_linear_weight)if "data_type" in sig.parameters:- return dequant_block_fp8_weight(layer.weight, weight_scale, layer.block_size, data_type=data_type)+ return _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size, data_type=data_type)else:- return dequant_block_fp8_weight(layer.weight, weight_scale, layer.block_size)+ return _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size)def check_to_quantized(config):@@ -1190,7 +1200,6 @@ def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16, device: str = "cpu"if is_gaudi2():device = "cpu"-# Use registry-based dequantizationdq_weight = dequant_fp8_layer(layer, dtype=dtype, device=device)new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))diff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.pyindex 6e77a7b2f..42a7dc83f 100644--- a/test/test_cpu/core/test_autoround.py+++ b/test/test_cpu/core/test_autoround.py@@ -634,27 +634,27 @@ class TestAutoRound:autoround.quantize()def test_dequant_fp8_weight(self):- from auto_round.utils import dequant_block_fp8_weight+ from auto_round.utils.model import _dequant_fp8_linear_weight# test high precision weight (should skip LUT and not crash)weight = torch.randn(587, 7168)weight_scale = torch.randn(5, 56)block_size = [128, 128]- dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)+ dequant_weight = _dequant_fp8_linear_weight(weight, weight_scale, block_size)assert dequant_weight.shape.numel() == 4207616# test low precision weight (should use LUT)weight = torch.randint(0, 255, (587, 7168), dtype=torch.uint8)weight_scale = torch.randn(5, 56)block_size = [128, 128]- dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)+ dequant_weight = _dequant_fp8_linear_weight(weight, weight_scale, block_size)assert dequant_weight.shape.numel() == 4207616# test experts are stacked.weight = torch.randint(0, 255, [32, 5760, 1440], dtype=torch.uint8)weight_scale = torch.randn([32, 5760, 90])block_size = [1, 16]- dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)+ dequant_weight = _dequant_fp8_linear_weight(weight, weight_scale, block_size)assert len(dequant_weight.shape) == 3assert dequant_weight.shape[0] == 32assert dequant_weight.shape.numel() == 32 * 5760 * 1440@@ -663,14 +663,14 @@ class TestAutoRound:"""Test Gaudi2-specific dequantization redirection logic in convert_fp8_layer_to_linear."""from unittest.mock import MagicMock, patch- from auto_round.utils.model import convert_fp8_layer_to_linear, dequant_block_fp8_weight+ from auto_round.utils.model import _dequant_fp8_linear_weight, convert_fp8_layer_to_linear# Test 1: Verify Standard e4m3fn native dequantization resultweight_uint8 = torch.tensor([[126, 0]], dtype=torch.uint8)weight_scale = torch.ones(1, 1, dtype=torch.bfloat16)block_size = [1, 2]- dq_weight = dequant_block_fp8_weight(weight_uint8, weight_scale, block_size)+ dq_weight = _dequant_fp8_linear_weight(weight_uint8, weight_scale, block_size)assert dq_weight.dtype == torch.bfloat16assert abs(dq_weight[0, 0].item() - 448.0) < 1e-3diff --git a/test/test_cpu/core/test_fp8_re_quant.py b/test/test_cpu/core/test_fp8_re_quant.pynew file mode 100644index 000000000..59e42a543--- /dev/null+++ b/test/test_cpu/core/test_fp8_re_quant.py@@ -0,0 +1,97 @@+import unittest++import torch+import torch.nn as nn+from transformers import AutoConfig++from auto_round.utils.model import _dequant_fp8_linear_weight, convert_fp8_layer_to_linear+++class MockFP8Layer:+ def __init__(self, in_features, out_features, has_block_size=False):+ self.in_features = in_features+ self.out_features = out_features+ self.weight = torch.randn(out_features, in_features).to(torch.float8_e4m3fn)+ self.weight_scale = torch.tensor(0.5)+ self.bias = nn.Parameter(torch.zeros(out_features))+ if has_block_size:+ self.block_size = [1, 1]+ self.weight_scale = torch.full((out_features, in_features), 0.5)+ self.data_type = "fp8"++ def to(self, device):+ self.weight = self.weight.to(device)+ self.weight_scale = self.weight_scale.to(device)+ self.bias.data = self.bias.data.to(device)+ return self+++class TestFP8ReQuant(unittest.TestCase):+ def test_per_tensor_dequant(self):+ """Test basic per-tensor dequantization logic."""+ in_features, out_features = 128, 64+ layer = MockFP8Layer(in_features, out_features)++ dq_weight = _dequant_fp8_linear_weight(layer.weight, layer.weight_scale, block_size=None)+ self.assertEqual(dq_weight.shape, (out_features, in_features))+ self.assertEqual(dq_weight.dtype, torch.bfloat16)++ expected = layer.weight.to(torch.bfloat16) * layer.weight_scale.to(torch.bfloat16)+ torch.testing.assert_close(dq_weight, expected)++ def test_per_channel_dequant(self):+ """Test per-channel (vector) dequantization logic."""+ in_features, out_features = 128, 64+ weight = torch.randn(out_features, in_features).to(torch.float8_e4m3fn)++ # Per-channel scale (out_features)+ scale = torch.randn(out_features)+ dq_weight = _dequant_fp8_linear_weight(weight, scale, block_size=None)+ expected = weight.to(torch.bfloat16) * scale.view(-1, 1).to(torch.bfloat16)+ torch.testing.assert_close(dq_weight, expected)++ # Per-channel scale (in_features)+ scale = torch.randn(in_features)+ dq_weight = _dequant_fp8_linear_weight(weight, scale, block_size=None)+ expected = weight.to(torch.bfloat16) * scale.view(1, -1).to(torch.bfloat16)+ torch.testing.assert_close(dq_weight, expected)++ def test_devstral_model_structure(self):+ """+ Specific test for Devstral-2-123B architecture.+ Uses actual configuration metadata from HF.+ """+ print("\nVerifying Devstral-2-123B layer structure conversion...")+ model_id = "mistralai/Devstral-2-123B-Instruct-2512"+ try:+ config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)+ in_features = config.hidden_size+ out_features = config.intermediate_size+ except Exception as e:+ print(f"Skipping remote config fetch due to: {e}. Using hardcoded fallback.")+ in_features, out_features = 12288, 28672 # Devstral 2 parameters++ class FP8Linear(nn.Module):+ def __init__(self, in_f, out_f):+ super().__init__()+ self.in_features = in_f+ self.out_features = out_f+ self.weight = nn.Parameter(torch.randn(out_f, in_f).to(torch.float8_e4m3fn))+ # Devstral uses per-tensor FP8 (no block_size)+ self.weight_scale = nn.Parameter(torch.tensor(0.001953125))+ self.bias = None+ self.data_type = "fp8"++ mock_layer = FP8Linear(in_features, out_features)++ # This should now pass without AttributeError for 'block_size'+ new_layer = convert_fp8_layer_to_linear(mock_layer, dtype=torch.bfloat16)++ self.assertIsInstance(new_layer, nn.Linear)+ self.assertEqual(new_layer.in_features, in_features)+ self.assertEqual(new_layer.out_features, out_features)+ self.assertEqual(new_layer.weight.dtype, torch.bfloat16)++ expected = mock_layer.weight.to(torch.bfloat16) * mock_layer.weight_scale.to(torch.bfloat16)+ torch.testing.assert_close(new_layer.weight, expected)+ print("Devstral layer conversion successful!")