PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
(feat): add support for g2 fp8 on cpu with LUT (#1254)
SwekeR6190869b7f0f
diff --git a/auto_round/envs.py b/auto_round/envs.pyindex 4e2ad39bb..448644500 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -29,6 +29,7 @@ environment_variables: dict[str, Callable[[], Any]] = {"AR_WORK_SPACE": lambda: os.getenv("AR_WORK_SPACE", "ar_work_space").lower(),"AR_ENABLE_UNIFY_MOE_INPUT_SCALE": lambda: os.getenv("AR_ENABLE_UNIFY_MOE_INPUT_SCALE", "False").lower()in ["1", "true"],+ "AR_OMP_NUM_THREADS": lambda: os.getenv("AR_OMP_NUM_THREADS", None),}diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex f14ad9539..b7b98cf1e 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -16,10 +16,13 @@ import jsonimport osimport refrom collections import UserDict+from contextlib import ContextDecorator, contextmanagerfrom dataclasses import asdict, fields+from functools import wrapsfrom pathlib import Pathfrom typing import Union+import psutilimport torchimport transformers@@ -906,16 +909,65 @@ def pad_block_fp8_weight_naive(return weight, orig_M, orig_N-def dequant_block_fp8_weight(weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list) -> torch.Tensor:+class with_thread_limits(ContextDecorator):+ """+ Context manager and decorator to temporarily set AR_OMP_NUM_THREADS and PyTorch threads.+ Inspired by vLLM's thread limit decorator.+ https://github.com/HabanaAI/vllm-fork/blob/f943a89a20e0e57bca64e1cca05469bfcaaec6f8/vllm/worker/hpu_model_runner.py#L1063-L1115+ """++ def __init__(self, div=1):+ self.div = div+ self.old_omp = None+ self.old_torch = None++ def __enter__(self):+ # Save original settings+ self.old_omp = envs.AR_OMP_NUM_THREADS+ self.old_torch = torch.get_num_threads()++ try:+ if psutil is not None:+ num_cores = len(psutil.Process().cpu_affinity() or [])+ if num_cores == 0:+ num_cores = os.cpu_count() or 1+ else:+ num_cores = os.cpu_count() or 1++ # Set new limits+ new_threads = max(1, num_cores // self.div)+ os.environ["AR_OMP_NUM_THREADS"] = str(new_threads)+ torch.set_num_threads(new_threads)+ except Exception:+ pass+ return self++ def __exit__(self, exc_type, exc_val, exc_tb):+ # Restore original settings+ if self.old_omp is None:+ os.environ.pop("AR_OMP_NUM_THREADS", None)+ else:+ os.environ["AR_OMP_NUM_THREADS"] = self.old_omp+ torch.set_num_threads(self.old_torch)+++@with_thread_limits()+def dequant_block_fp8_weight(+ weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list, data_type: str = None+) -> torch.Tensor:+ """Core dequantization logic for block-wise FP8 weights."""dtype = torch.bfloat16if weight_scale is None:return weightassert len(block_size) == 2+ # If weight is stored as uint8, view it as float8_e4m3fn+ if weight.element_size() == 1 and weight.dtype != torch.float8_e4m3fn:+ weight = weight.view(torch.float8_e4m3fn)+weight, orig_M, orig_N = pad_block_fp8_weight_naive(weight, weight_scale, block_size)weight_shape_len = len(weight.shape)-block_size_m, block_size_n = block_size# mul scale@@ -1010,12 +1062,17 @@ def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16, device: str = "cpu"for key in keys:setattr(new_layer, key, getattr(layer, key, None))+ from auto_round.utils.device import is_gaudi2++ if is_gaudi2():+ device = "cpu"layer = layer.to(device)if layer.__class__.__name__ == "CompressedLinear":dq_weight = layer.compressor.decompress_module(layer)else:weight_scale = layer.weight_scale if hasattr(layer, "weight_scale") else layer.weight_scale_inv- dq_weight = dequant_block_fp8_weight(layer.weight, weight_scale, layer.block_size)+ data_type = getattr(layer, "data_type", None)+ dq_weight = dequant_block_fp8_weight(layer.weight, weight_scale, layer.block_size, data_type=data_type)new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))return new_layerdiff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.pyindex 3842cd14a..79458e237 100644--- a/test/test_cpu/core/test_autoround.py+++ b/test/test_cpu/core/test_autoround.py@@ -627,15 +627,22 @@ class TestAutoRound:def test_dequant_fp8_weight(self):from auto_round.utils import dequant_block_fp8_weight- # test pad and unpad+ # test high precision weight (should skip LUT and not crash)weight = torch.randn(587, 7168)weight_scale = torch.randn(5, 56)block_size = [128, 128]dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)assert dequant_weight.shape.numel() == 4207616+ # test low precision weight (should use LUT)+ weight = torch.randint(0, 255, (587, 7168), dtype=torch.uint8)+ weight_scale = torch.randn(5, 56)+ block_size = [128, 128]+ dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)+ assert dequant_weight.shape.numel() == 4207616+# test experts are stacked.- weight = torch.randn([32, 5760, 1440])+ weight = torch.randint(0, 255, [32, 5760, 1440], dtype=torch.uint8)weight_scale = torch.randn([32, 5760, 90])block_size = [1, 16]dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)@@ -643,6 +650,47 @@ class TestAutoRound:assert dequant_weight.shape[0] == 32assert dequant_weight.shape.numel() == 32 * 5760 * 1440+ def test_dequant_g2_fp8_weight(self):+ """Test Gaudi2-specific dequantization redirection logic in convert_fp8_layer_to_linear."""+ from unittest.mock import MagicMock, patch++ from auto_round.utils.model import convert_fp8_layer_to_linear, dequant_block_fp8_weight++ # Test 1: Verify Standard e4m3fn native dequantization result+ weight_uint8 = torch.tensor([[126, 0]], dtype=torch.uint8)+ weight_scale = torch.ones(1, 1, dtype=torch.bfloat16)+ block_size = [1, 2]++ dq_weight = dequant_block_fp8_weight(weight_uint8, weight_scale, block_size)+ assert dq_weight.dtype == torch.bfloat16+ assert abs(dq_weight[0, 0].item() - 448.0) < 1e-3++ # Test 2: Verify Gaudi2 redirection in convert_fp8_layer_to_linear+ mock_layer = MagicMock()+ mock_layer.in_features = 2+ mock_layer.out_features = 1+ mock_layer.bias = None+ mock_layer.weight = weight_uint8+ mock_layer.weight_scale = weight_scale+ mock_layer.block_size = block_size+ mock_layer.data_type = "fp8_e4m3fn"+ mock_layer.__class__.__name__ = "FP8Linear"++ # mock_layer.to should return a mock layer that we can check+ mock_layer.to.return_value = mock_layer++ with patch("auto_round.utils.device.is_gaudi2", return_value=True):+ convert_fp8_layer_to_linear(mock_layer, device="hpu")+ # Verify it was moved to CPU+ mock_layer.to.assert_called_with("cpu")++ with patch("auto_round.utils.device.is_gaudi2", return_value=False):+ # Reset mock+ mock_layer.to.reset_mock()+ convert_fp8_layer_to_linear(mock_layer, device="hpu")+ # Verify it was moved to HPU (as requested in device arg)+ mock_layer.to.assert_called_with("hpu")+def test_mixed_bit_setting(self, tiny_opt_model_path):model_name = tiny_opt_model_pathlayer_config = {"model.decoder.layers.1.fc1": {"bits": 8, "act_bits": 8}}