Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

(feat): add support for g2 fp8 on cpu with LUT (#1254)

SwekeR6190869b7f0f
diff --git a/auto_round/envs.py b/auto_round/envs.py
index 4e2ad39bb..448644500 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -29,6 +29,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
"AR_WORK_SPACE": lambda: os.getenv("AR_WORK_SPACE", "ar_work_space").lower(),
"AR_ENABLE_UNIFY_MOE_INPUT_SCALE": lambda: os.getenv("AR_ENABLE_UNIFY_MOE_INPUT_SCALE", "False").lower()
in ["1", "true"],
+ "AR_OMP_NUM_THREADS": lambda: os.getenv("AR_OMP_NUM_THREADS", None),
}
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index f14ad9539..b7b98cf1e 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -16,10 +16,13 @@ import json
import os
import re
from collections import UserDict
+from contextlib import ContextDecorator, contextmanager
from dataclasses import asdict, fields
+from functools import wraps
from pathlib import Path
from typing import Union
+import psutil
import torch
import transformers
@@ -906,16 +909,65 @@ def pad_block_fp8_weight_naive(
return weight, orig_M, orig_N
-def dequant_block_fp8_weight(weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list) -> torch.Tensor:
+class with_thread_limits(ContextDecorator):
+ """
+ Context manager and decorator to temporarily set AR_OMP_NUM_THREADS and PyTorch threads.
+ Inspired by vLLM's thread limit decorator.
+ https://github.com/HabanaAI/vllm-fork/blob/f943a89a20e0e57bca64e1cca05469bfcaaec6f8/vllm/worker/hpu_model_runner.py#L1063-L1115
+ """
+
+ def __init__(self, div=1):
+ self.div = div
+ self.old_omp = None
+ self.old_torch = None
+
+ def __enter__(self):
+ # Save original settings
+ self.old_omp = envs.AR_OMP_NUM_THREADS
+ self.old_torch = torch.get_num_threads()
+
+ try:
+ if psutil is not None:
+ num_cores = len(psutil.Process().cpu_affinity() or [])
+ if num_cores == 0:
+ num_cores = os.cpu_count() or 1
+ else:
+ num_cores = os.cpu_count() or 1
+
+ # Set new limits
+ new_threads = max(1, num_cores // self.div)
+ os.environ["AR_OMP_NUM_THREADS"] = str(new_threads)
+ torch.set_num_threads(new_threads)
+ except Exception:
+ pass
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ # Restore original settings
+ if self.old_omp is None:
+ os.environ.pop("AR_OMP_NUM_THREADS", None)
+ else:
+ os.environ["AR_OMP_NUM_THREADS"] = self.old_omp
+ torch.set_num_threads(self.old_torch)
+
+
+@with_thread_limits()
+def dequant_block_fp8_weight(
+ weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list, data_type: str = None
+) -> torch.Tensor:
+ """Core dequantization logic for block-wise FP8 weights."""
dtype = torch.bfloat16
if weight_scale is None:
return weight
assert len(block_size) == 2
+ # If weight is stored as uint8, view it as float8_e4m3fn
+ if weight.element_size() == 1 and weight.dtype != torch.float8_e4m3fn:
+ weight = weight.view(torch.float8_e4m3fn)
+
weight, orig_M, orig_N = pad_block_fp8_weight_naive(weight, weight_scale, block_size)
weight_shape_len = len(weight.shape)
-
block_size_m, block_size_n = block_size
# mul scale
@@ -1010,12 +1062,17 @@ def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16, device: str = "cpu"
for key in keys:
setattr(new_layer, key, getattr(layer, key, None))
+ from auto_round.utils.device import is_gaudi2
+
+ if is_gaudi2():
+ device = "cpu"
layer = layer.to(device)
if layer.__class__.__name__ == "CompressedLinear":
dq_weight = layer.compressor.decompress_module(layer)
else:
weight_scale = layer.weight_scale if hasattr(layer, "weight_scale") else layer.weight_scale_inv
- dq_weight = dequant_block_fp8_weight(layer.weight, weight_scale, layer.block_size)
+ data_type = getattr(layer, "data_type", None)
+ dq_weight = dequant_block_fp8_weight(layer.weight, weight_scale, layer.block_size, data_type=data_type)
new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
return new_layer
diff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.py
index 3842cd14a..79458e237 100644
--- a/test/test_cpu/core/test_autoround.py
+++ b/test/test_cpu/core/test_autoround.py
@@ -627,15 +627,22 @@ class TestAutoRound:
def test_dequant_fp8_weight(self):
from auto_round.utils import dequant_block_fp8_weight
- # test pad and unpad
+ # test high precision weight (should skip LUT and not crash)
weight = torch.randn(587, 7168)
weight_scale = torch.randn(5, 56)
block_size = [128, 128]
dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)
assert dequant_weight.shape.numel() == 4207616
+ # test low precision weight (should use LUT)
+ weight = torch.randint(0, 255, (587, 7168), dtype=torch.uint8)
+ weight_scale = torch.randn(5, 56)
+ block_size = [128, 128]
+ dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)
+ assert dequant_weight.shape.numel() == 4207616
+
# test experts are stacked.
- weight = torch.randn([32, 5760, 1440])
+ weight = torch.randint(0, 255, [32, 5760, 1440], dtype=torch.uint8)
weight_scale = torch.randn([32, 5760, 90])
block_size = [1, 16]
dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)
@@ -643,6 +650,47 @@ class TestAutoRound:
assert dequant_weight.shape[0] == 32
assert dequant_weight.shape.numel() == 32 * 5760 * 1440
+ def test_dequant_g2_fp8_weight(self):
+ """Test Gaudi2-specific dequantization redirection logic in convert_fp8_layer_to_linear."""
+ from unittest.mock import MagicMock, patch
+
+ from auto_round.utils.model import convert_fp8_layer_to_linear, dequant_block_fp8_weight
+
+ # Test 1: Verify Standard e4m3fn native dequantization result
+ weight_uint8 = torch.tensor([[126, 0]], dtype=torch.uint8)
+ weight_scale = torch.ones(1, 1, dtype=torch.bfloat16)
+ block_size = [1, 2]
+
+ dq_weight = dequant_block_fp8_weight(weight_uint8, weight_scale, block_size)
+ assert dq_weight.dtype == torch.bfloat16
+ assert abs(dq_weight[0, 0].item() - 448.0) < 1e-3
+
+ # Test 2: Verify Gaudi2 redirection in convert_fp8_layer_to_linear
+ mock_layer = MagicMock()
+ mock_layer.in_features = 2
+ mock_layer.out_features = 1
+ mock_layer.bias = None
+ mock_layer.weight = weight_uint8
+ mock_layer.weight_scale = weight_scale
+ mock_layer.block_size = block_size
+ mock_layer.data_type = "fp8_e4m3fn"
+ mock_layer.__class__.__name__ = "FP8Linear"
+
+ # mock_layer.to should return a mock layer that we can check
+ mock_layer.to.return_value = mock_layer
+
+ with patch("auto_round.utils.device.is_gaudi2", return_value=True):
+ convert_fp8_layer_to_linear(mock_layer, device="hpu")
+ # Verify it was moved to CPU
+ mock_layer.to.assert_called_with("cpu")
+
+ with patch("auto_round.utils.device.is_gaudi2", return_value=False):
+ # Reset mock
+ mock_layer.to.reset_mock()
+ convert_fp8_layer_to_linear(mock_layer, device="hpu")
+ # Verify it was moved to HPU (as requested in device arg)
+ mock_layer.to.assert_called_with("hpu")
+
def test_mixed_bit_setting(self, tiny_opt_model_path):
model_name = tiny_opt_model_path
layer_config = {"model.decoder.layers.1.fc1": {"bits": 8, "act_bits": 8}}