Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat: handle per-tensor FP8 dequantization for Devstral models (#1356)

SwekeReddb6de2bfed
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 6caea4c1a..66af26c9e 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -1031,19 +1031,28 @@ class with_thread_limits(ContextDecorator):
@with_thread_limits()
-def dequant_block_fp8_weight(
- weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list, data_type: str = None
+def _dequant_fp8_linear_weight(
+ weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list = None, data_type: str = None
) -> torch.Tensor:
"""Core dequantization logic for block-wise FP8 weights."""
dtype = torch.bfloat16
if weight_scale is None:
return weight
- assert len(block_size) == 2
# If weight is stored as uint8, view it as float8_e4m3fn
if weight.element_size() == 1 and weight.dtype != torch.float8_e4m3fn:
weight = weight.view(torch.float8_e4m3fn)
+ if block_size is None:
+ if weight_scale.numel() > 1 and weight_scale.shape != weight.shape:
+ if weight_scale.numel() == weight.shape[0]:
+ weight_scale = weight_scale.view(-1, 1)
+ elif weight_scale.numel() == weight.shape[-1]:
+ weight_scale = weight_scale.view(1, -1)
+ return weight.to(dtype) * weight_scale.to(dtype)
+
+ assert len(block_size) == 2
+
weight, orig_M, orig_N = pad_block_fp8_weight_naive(weight, weight_scale, block_size)
weight_shape_len = len(weight.shape)
@@ -1073,7 +1082,7 @@ def dequant_block_fp8_weight(
# Register FP8 layer dequantization handlers
-# Note: Handlers are registered here (after dequant_block_fp8_weight is defined)
+# Note: Handlers are registered here (after _dequant_fp8_linear_weight is defined)
# to ensure all dependencies are available.
@register_fp8_layer("CompressedLinear")
def dequant_compressed_linear(layer, dtype=torch.bfloat16, device: str = "cpu"):
@@ -1087,16 +1096,17 @@ def dequant_fp8_linear(layer, dtype=torch.bfloat16, device: str = "cpu"):
"""Dequantize FP8Linear layer using block-based dequantization."""
layer = layer.to(device)
weight_scale = layer.weight_scale if hasattr(layer, "weight_scale") else layer.weight_scale_inv
+ block_size = getattr(layer, "block_size", None)
data_type = getattr(layer, "data_type", None)
- # Pass data_type if dequant_block_fp8_weight supports it
+ # Pass data_type if _dequant_fp8_linear_weight supports it
# Check if function accepts data_type parameter
import inspect
- sig = inspect.signature(dequant_block_fp8_weight)
+ sig = inspect.signature(_dequant_fp8_linear_weight)
if "data_type" in sig.parameters:
- return dequant_block_fp8_weight(layer.weight, weight_scale, layer.block_size, data_type=data_type)
+ return _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size, data_type=data_type)
else:
- return dequant_block_fp8_weight(layer.weight, weight_scale, layer.block_size)
+ return _dequant_fp8_linear_weight(layer.weight, weight_scale, block_size)
def check_to_quantized(config):
@@ -1190,7 +1200,6 @@ def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16, device: str = "cpu"
if is_gaudi2():
device = "cpu"
-
# Use registry-based dequantization
dq_weight = dequant_fp8_layer(layer, dtype=dtype, device=device)
new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
diff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.py
index 6e77a7b2f..42a7dc83f 100644
--- a/test/test_cpu/core/test_autoround.py
+++ b/test/test_cpu/core/test_autoround.py
@@ -634,27 +634,27 @@ class TestAutoRound:
autoround.quantize()
def test_dequant_fp8_weight(self):
- from auto_round.utils import dequant_block_fp8_weight
+ from auto_round.utils.model import _dequant_fp8_linear_weight
# test high precision weight (should skip LUT and not crash)
weight = torch.randn(587, 7168)
weight_scale = torch.randn(5, 56)
block_size = [128, 128]
- dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)
+ dequant_weight = _dequant_fp8_linear_weight(weight, weight_scale, block_size)
assert dequant_weight.shape.numel() == 4207616
# test low precision weight (should use LUT)
weight = torch.randint(0, 255, (587, 7168), dtype=torch.uint8)
weight_scale = torch.randn(5, 56)
block_size = [128, 128]
- dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)
+ dequant_weight = _dequant_fp8_linear_weight(weight, weight_scale, block_size)
assert dequant_weight.shape.numel() == 4207616
# test experts are stacked.
weight = torch.randint(0, 255, [32, 5760, 1440], dtype=torch.uint8)
weight_scale = torch.randn([32, 5760, 90])
block_size = [1, 16]
- dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)
+ dequant_weight = _dequant_fp8_linear_weight(weight, weight_scale, block_size)
assert len(dequant_weight.shape) == 3
assert dequant_weight.shape[0] == 32
assert dequant_weight.shape.numel() == 32 * 5760 * 1440
@@ -663,14 +663,14 @@ class TestAutoRound:
"""Test Gaudi2-specific dequantization redirection logic in convert_fp8_layer_to_linear."""
from unittest.mock import MagicMock, patch
- from auto_round.utils.model import convert_fp8_layer_to_linear, dequant_block_fp8_weight
+ from auto_round.utils.model import _dequant_fp8_linear_weight, convert_fp8_layer_to_linear
# Test 1: Verify Standard e4m3fn native dequantization result
weight_uint8 = torch.tensor([[126, 0]], dtype=torch.uint8)
weight_scale = torch.ones(1, 1, dtype=torch.bfloat16)
block_size = [1, 2]
- dq_weight = dequant_block_fp8_weight(weight_uint8, weight_scale, block_size)
+ dq_weight = _dequant_fp8_linear_weight(weight_uint8, weight_scale, block_size)
assert dq_weight.dtype == torch.bfloat16
assert abs(dq_weight[0, 0].item() - 448.0) < 1e-3
diff --git a/test/test_cpu/core/test_fp8_re_quant.py b/test/test_cpu/core/test_fp8_re_quant.py
new file mode 100644
index 000000000..59e42a543
--- /dev/null
+++ b/test/test_cpu/core/test_fp8_re_quant.py
@@ -0,0 +1,97 @@
+import unittest
+
+import torch
+import torch.nn as nn
+from transformers import AutoConfig
+
+from auto_round.utils.model import _dequant_fp8_linear_weight, convert_fp8_layer_to_linear
+
+
+class MockFP8Layer:
+ def __init__(self, in_features, out_features, has_block_size=False):
+ self.in_features = in_features
+ self.out_features = out_features
+ self.weight = torch.randn(out_features, in_features).to(torch.float8_e4m3fn)
+ self.weight_scale = torch.tensor(0.5)
+ self.bias = nn.Parameter(torch.zeros(out_features))
+ if has_block_size:
+ self.block_size = [1, 1]
+ self.weight_scale = torch.full((out_features, in_features), 0.5)
+ self.data_type = "fp8"
+
+ def to(self, device):
+ self.weight = self.weight.to(device)
+ self.weight_scale = self.weight_scale.to(device)
+ self.bias.data = self.bias.data.to(device)
+ return self
+
+
+class TestFP8ReQuant(unittest.TestCase):
+ def test_per_tensor_dequant(self):
+ """Test basic per-tensor dequantization logic."""
+ in_features, out_features = 128, 64
+ layer = MockFP8Layer(in_features, out_features)
+
+ dq_weight = _dequant_fp8_linear_weight(layer.weight, layer.weight_scale, block_size=None)
+ self.assertEqual(dq_weight.shape, (out_features, in_features))
+ self.assertEqual(dq_weight.dtype, torch.bfloat16)
+
+ expected = layer.weight.to(torch.bfloat16) * layer.weight_scale.to(torch.bfloat16)
+ torch.testing.assert_close(dq_weight, expected)
+
+ def test_per_channel_dequant(self):
+ """Test per-channel (vector) dequantization logic."""
+ in_features, out_features = 128, 64
+ weight = torch.randn(out_features, in_features).to(torch.float8_e4m3fn)
+
+ # Per-channel scale (out_features)
+ scale = torch.randn(out_features)
+ dq_weight = _dequant_fp8_linear_weight(weight, scale, block_size=None)
+ expected = weight.to(torch.bfloat16) * scale.view(-1, 1).to(torch.bfloat16)
+ torch.testing.assert_close(dq_weight, expected)
+
+ # Per-channel scale (in_features)
+ scale = torch.randn(in_features)
+ dq_weight = _dequant_fp8_linear_weight(weight, scale, block_size=None)
+ expected = weight.to(torch.bfloat16) * scale.view(1, -1).to(torch.bfloat16)
+ torch.testing.assert_close(dq_weight, expected)
+
+ def test_devstral_model_structure(self):
+ """
+ Specific test for Devstral-2-123B architecture.
+ Uses actual configuration metadata from HF.
+ """
+ print("\nVerifying Devstral-2-123B layer structure conversion...")
+ model_id = "mistralai/Devstral-2-123B-Instruct-2512"
+ try:
+ config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
+ in_features = config.hidden_size
+ out_features = config.intermediate_size
+ except Exception as e:
+ print(f"Skipping remote config fetch due to: {e}. Using hardcoded fallback.")
+ in_features, out_features = 12288, 28672 # Devstral 2 parameters
+
+ class FP8Linear(nn.Module):
+ def __init__(self, in_f, out_f):
+ super().__init__()
+ self.in_features = in_f
+ self.out_features = out_f
+ self.weight = nn.Parameter(torch.randn(out_f, in_f).to(torch.float8_e4m3fn))
+ # Devstral uses per-tensor FP8 (no block_size)
+ self.weight_scale = nn.Parameter(torch.tensor(0.001953125))
+ self.bias = None
+ self.data_type = "fp8"
+
+ mock_layer = FP8Linear(in_features, out_features)
+
+ # This should now pass without AttributeError for 'block_size'
+ new_layer = convert_fp8_layer_to_linear(mock_layer, dtype=torch.bfloat16)
+
+ self.assertIsInstance(new_layer, nn.Linear)
+ self.assertEqual(new_layer.in_features, in_features)
+ self.assertEqual(new_layer.out_features, out_features)
+ self.assertEqual(new_layer.weight.dtype, torch.bfloat16)
+
+ expected = mock_layer.weight.to(torch.bfloat16) * mock_layer.weight_scale.to(torch.bfloat16)
+ torch.testing.assert_close(new_layer.weight, expected)
+ print("Devstral layer conversion successful!")