PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Support load FP8 model on HPU (#1449)
Yi Liuaef6d3c41238
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 5877bd389..48aaf1bd6 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -486,8 +486,6 @@ class BaseCompressor(object):logger.info(f"using {self.model.dtype} for quantization tuning")# Some helpers- if "hpu" in str(self.device):- self.inner_supported_types = tuple(x for x in INNER_SUPPORTED_LAYER_TYPES if x != "FP8Linear")self.batch_dim = Noneself.infer_bs_coeff = 1diff --git a/auto_round/modeling/__init__.py b/auto_round/modeling/__init__.pyindex d1bc25269..144e78ca3 100644--- a/auto_round/modeling/__init__.py+++ b/auto_round/modeling/__init__.py@@ -13,3 +13,4 @@# limitations under the License.from .fp8_quant import *+from .hpu_patch import *diff --git a/auto_round/modeling/finegrained_fp8_patch.py b/auto_round/modeling/finegrained_fp8_patch.pynew file mode 100644index 000000000..06d85835b--- /dev/null+++ b/auto_round/modeling/finegrained_fp8_patch.py@@ -0,0 +1,241 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# Copied from https://github.com/huggingface/transformers/blob/main/src/transformers/integrations/finegrained_fp8.py+from transformers.core_model_loading import ConversionOps+from transformers.quantizers.quantizers_utils import should_convert_module+from transformers.utils import is_kernels_available, is_torch_accelerator_available, is_torch_available, logging++if is_torch_available():+ import torch+ import torch.nn as nn++ # import triton+ # import triton.language as tl+ from torch.nn import functional as F+++logger = logging.get_logger(__name__)+++_FP8_DTYPE = torch.float8_e4m3fn+_FP8_MIN = torch.finfo(_FP8_DTYPE).min+_FP8_MAX = torch.finfo(_FP8_DTYPE).max+++class FP8Linear(nn.Linear):+ def __init__(+ self,+ in_features: int,+ out_features: int,+ bias: bool = False,+ dtype=torch.float8_e4m3fn,+ block_size: tuple[int, int] | None = None,+ activation_scheme="dynamic",+ ):+ super().__init__(in_features, out_features)++ # If block size is None, it means that we are doing per-tensor quantization+ self.block_size = block_size+ self.activation_scheme = activation_scheme++ self.weight = torch.nn.Parameter(torch.empty(out_features, in_features, dtype=dtype))++ if self.block_size is None:+ self.weight_scale_inv = nn.Parameter(torch.tensor(1.0, dtype=torch.float32))+ else:+ scale_out_features = (out_features + self.block_size[0] - 1) // self.block_size[0]+ scale_in_features = (in_features + self.block_size[1] - 1) // self.block_size[1]+ self.weight_scale_inv = nn.Parameter(+ torch.empty(scale_out_features, scale_in_features, dtype=torch.float32)+ )++ if self.activation_scheme == "static":+ self.activation_scale = nn.Parameter(torch.tensor(1.0, dtype=torch.float32))++ if bias:+ self.bias = nn.Parameter(torch.empty(self.out_features))+ else:+ self.register_parameter("bias", None)+++def _ceil_div(a, b):+ return (a + b - 1) // b+++def replace_with_fp8_linear(+ model, modules_to_not_convert: list[str] | None = None, quantization_config=None, pre_quantized=False+):+ """+ A helper function to replace all `torch.nn.Linear` modules by `FP8Linear` modules.++ Parameters:+ model (`torch.nn.Module`):+ Input model or `torch.nn.Module` as the function is run recursively.+ modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`):+ Names of the modules to not convert. In practice we keep the `lm_head`+ in full precision for numerical stability reasons.+ quantization_config (`FbgemmFp8Config`):+ The quantization config object that contains the quantization parameters.+ pre_quantized (`book`, defaults to `False`):+ Whether the model is pre-quantized or not+ """++ if quantization_config.dequantize:+ return model++ has_been_replaced = False+ for module_name, module in model.named_modules():+ if not should_convert_module(module_name, modules_to_not_convert):+ continue+ # we need this to correctly materialize the weights during quantization+ module_kwargs = {} if pre_quantized else {"dtype": None}+ new_module = None+ with torch.device("meta"):+ if isinstance(module, nn.Linear):+ new_module = FP8Linear(+ in_features=module.in_features,+ out_features=module.out_features,+ bias=module.bias is not None,+ activation_scheme=quantization_config.activation_scheme,+ block_size=quantization_config.weight_block_size,+ **module_kwargs,+ )+ if new_module is not None:+ model.set_submodule(module_name, new_module)+ has_been_replaced = True++ if not has_been_replaced:+ logger.warning(+ "You are loading your model using fp8 but no linear modules were found in your model."+ " Please double check your model architecture."+ )+ return model+++class Fp8Quantize(ConversionOps):+ """+ A quantization operation that creates two tensors, weight and scale out of a weight.+ """++ def __init__(self, hf_quantizer):+ self.hf_quantizer = hf_quantizer++ def convert(self, input_dict: torch.Tensor, **kwargs) -> dict[str, torch.Tensor]:+ # Unpack single key/value (value may be wrapped in a list)+ target_keys, value = tuple(input_dict.items())[0]+ value = value[0]++ # Resolve block size (support dict-like or attr-like quant_config)+ block_size = None+ if self.hf_quantizer.quantization_config is not None:+ if isinstance(self.hf_quantizer.quantization_config, dict):+ block_size = self.hf_quantizer.quantization_config.get("weight_block_size")+ else:+ block_size = getattr(self.hf_quantizer.quantization_config, "weight_block_size", None)+ if block_size is None:+ block_size = (value.shape[-2], value.shape[-1])++ block_m, block_n = block_size+ rows, cols = value.shape[-2], value.shape[-1]++ # Enforce exact tiling like your original+ if rows % block_m != 0 or cols % block_n != 0:+ raise ValueError(+ (+ f"Matrix dimensions ({rows}, {cols}) must be divisible by block sizes"+ f" ({block_m}, {block_n}). for {target_keys}"+ )+ )++ # Leading dims can be empty (2D) or include num_experts/... (3D+)+ leading_shape = value.shape[:-2]+ rows_tiles = rows // block_m+ cols_tiles = cols // block_n++ original_shape = value.shape+ value_fp32 = value.to(torch.float32)++ # Reshape to (..., rows_tiles, block_m, cols_tiles, block_n)+ reshaped = value_fp32.reshape(*leading_shape, rows_tiles, block_m, cols_tiles, block_n)++ # Per-tile max-abs over the block dims+ # dims: block_m is at -3, block_n is at -1 after the reshape+ max_abs = reshaped.abs().amax(dim=(-3, -1))+ safe_max_abs = torch.where(max_abs > 0, max_abs, torch.ones_like(max_abs))++ # Tile scale (we store inverse scale like your Linear: weight_scale_inv)+ scales = _FP8_MAX / safe_max_abs+ scales = torch.where(max_abs > 0, scales, torch.ones_like(scales)) # keep zeros stable++ # Broadcast scales back over the block dims and quantize+ # max_abs/scales shape: (..., rows_tiles, cols_tiles)+ scales_broadcast = scales.unsqueeze(-1).unsqueeze(-3) # -> (..., rows_tiles, 1, cols_tiles, 1)+ scaled = reshaped * scales_broadcast++ quantized = torch.clamp(scaled, min=_FP8_MIN, max=_FP8_MAX).to(_FP8_DTYPE)++ quantized = quantized.reshape(original_shape)++ inv_scales = (1.0 / scales).to(torch.float32) # shape: (*leading, rows_tiles, cols_tiles)+ if target_keys.endswith("weight"):+ scale_key = target_keys.rsplit(".", 1)[0] + ".weight_scale_inv"+ else:+ scale_key = target_keys + "_scale_inv"++ # Return both quantized weights and per-tile inverse scales (keeps leading dims, e.g., num_experts)+ return {+ target_keys: quantized,+ scale_key: inv_scales,+ }+++class Fp8Dequantize(ConversionOps):+ """Inverse operation of :class:`Fp8Quantize`. Takes a pair (weight, scale) and reconstructs the fp32 tensor."""++ def __init__(self, hf_quantizer):+ self.hf_quantizer = hf_quantizer++ def convert(+ self,+ input_dict: dict[str, torch.Tensor],+ full_layer_name: str | None = None,+ **kwargs,+ ) -> dict[str, torch.Tensor]:+ if len(input_dict) < 2:+ # case where we only got weights, need to check for "weight$"+ return {full_layer_name: input_dict["weight$"]}++ quantized = input_dict["weight$"][0]+ scales = input_dict["weight_scale_inv"][0]++ rows, cols = quantized.shape[-2:]+ block_size = self.hf_quantizer.quantization_config.weight_block_size+ if block_size is None:+ block_size = (quantized.shape[-2], quantized.shape[-1])++ block_m, block_n = block_size++ if rows % block_m != 0 or cols % block_n != 0:+ raise ValueError(+ f"Matrix dimensions ({rows}, {cols}) must be divisible by block sizes ({block_m}, {block_n})."+ )+ quantized = quantized.to(scales.dtype)+ reshaped = quantized.reshape(-1, rows // block_m, block_m, cols // block_n, block_n)+ expanded_scales = scales.reshape(-1, rows // block_m, cols // block_n)+ expanded_scales = expanded_scales.unsqueeze(-1).unsqueeze(2)+ dequantized = reshaped * expanded_scales++ return {+ full_layer_name: dequantized.reshape(quantized.shape),+ }diff --git a/auto_round/modeling/hpu_patch.py b/auto_round/modeling/hpu_patch.pynew file mode 100644index 000000000..521caec4b--- /dev/null+++ b/auto_round/modeling/hpu_patch.py@@ -0,0 +1,35 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++from auto_round.logger import logger+++def patch_finegrained_fp8():+ """Use importlib to replace transformers.integrations.finegrained_fp8 with auto-round's HPU-compatible version."""+ try:+ from auto_round.utils import is_hpex_available++ if not is_hpex_available():+ return # No patching needed on non-HPU devices++ import importlib+ import sys++ # Import auto-round's HPU-compatible finegrained_fp8_patch module+ finegrained_fp8_patch = importlib.import_module("auto_round.modeling.finegrained_fp8_patch")++ # Replace transformers.integrations.finegrained_fp8 in sys.modules+ sys.modules["transformers.integrations.finegrained_fp8"] = finegrained_fp8_patch++ logger.info(+ "✓ Replaced transformers.integrations.finegrained_fp8 with auto_round.modeling.finegrained_fp8_patch"+ )++ except Exception as e:+ import warnings++ logger.warning(f"Failed to patch finegrained_fp8: {e}")+++# Apply patch on import if HPU is available+patch_finegrained_fp8()diff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex a16f441bf..e9a11923d 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -15,6 +15,7 @@ import functoolsimport gcimport osimport re+import sysfrom contextlib import ContextDecorator, contextmanagerfrom functools import lru_cachefrom itertools import combinations@@ -339,6 +340,71 @@ class override_cuda_device_capability(ContextDecorator):return False+class fake_cuda_for_hpu(ContextDecorator):+ """Context manager/decorator to fake CUDA availability for HPU devices."""++ def __init__(self):+ self._orig_is_available = None++ def __enter__(self):+ if is_hpex_available():+ self._orig_is_available = torch.cuda.is_available+ torch.cuda.is_available = lambda: True+ return self++ def __exit__(self, exc_type, exc, exc_tb):+ if is_hpex_available() and hasattr(self, "_orig_is_available"):+ torch.cuda.is_available = self._orig_is_available+ del self._orig_is_available+ return False+++class fake_triton_for_hpu(ContextDecorator):+ """Context manager/decorator to fake triton availability for HPU devices."""++ def __init__(self):+ self._orig_triton = None+ self._orig_triton_language = None+ self._had_triton = False+ self._had_triton_language = False++ def __enter__(self):+ if is_hpex_available():+ # Save original state+ self._had_triton = "triton" in sys.modules+ self._had_triton_language = "triton.language" in sys.modules++ if self._had_triton:+ self._orig_triton = sys.modules["triton"]+ if self._had_triton_language:+ self._orig_triton_language = sys.modules["triton.language"]++ # Create and inject fake triton module+ class FakeTriton:+ def __getattr__(self, name):+ return None++ fake_triton = FakeTriton()+ fake_triton.jit = lambda func: func # Make triton.jit a no-op decorator+ sys.modules["triton"] = fake_triton+ sys.modules["triton.language"] = FakeTriton()+ return self++ def __exit__(self, exc_type, exc, exc_tb):+ if is_hpex_available():+ # Restore original state+ if self._had_triton and self._orig_triton is not None:+ sys.modules["triton"] = self._orig_triton+ elif not self._had_triton and "triton" in sys.modules:+ del sys.modules["triton"]++ if self._had_triton_language and self._orig_triton_language is not None:+ sys.modules["triton.language"] = self._orig_triton_language+ elif not self._had_triton_language and "triton.language" in sys.modules:+ del sys.modules["triton.language"]+ return False++def get_packing_device(device: str | torch.device | None = "auto") -> torch.device:"""Selects the packing device.diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 2536c0a93..1442512c9 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -272,7 +272,10 @@ def llm_load_model(from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizerfrom auto_round.utils.device import (_use_hpu_compile_mode,+ fake_cuda_for_hpu,+ fake_triton_for_hpu,get_device_and_parallelism,+ is_hpex_available,override_cuda_device_capability,)@@ -289,13 +292,15 @@ def llm_load_model(if "deepseek" in pretrained_model_name_or_path.lower() and trust_remote_code:logger.warning("trust_remote_code is enabled by default, please ensure its correctness.")- if _use_hpu_compile_mode():- model = model_cls.from_pretrained(- pretrained_model_name_or_path,- torch_dtype=torch_dtype,- trust_remote_code=trust_remote_code,- device_map="auto" if use_auto_mapping else None,- )+ if is_hpex_available():+ # For loading FP8 model on HPU+ with fake_cuda_for_hpu(), fake_triton_for_hpu(), override_cuda_device_capability():+ model = model_cls.from_pretrained(+ pretrained_model_name_or_path,+ torch_dtype=torch_dtype,+ trust_remote_code=trust_remote_code,+ device_map="auto" if use_auto_mapping else None,+ )else:try:model = model_cls.from_pretrained(diff --git a/test/test_hpu/test_quant_fp8.py b/test/test_hpu/test_quant_fp8.pynew file mode 100644index 000000000..eaaa61741--- /dev/null+++ b/test/test_hpu/test_quant_fp8.py@@ -0,0 +1,35 @@+import os+import shutil++import pytest+import torch++from auto_round import AutoRound++MODEL_LIST = (+ "Qwen/Qwen3-0.6B-FP8",+ "Qwen/Qwen3-0.6B",+)+++class TestAutoRound:+ save_dir = "./saved"++ def check_nan_inf_in_tensor(self, tensor, name=""):+ return torch.isnan(tensor).any() or torch.isinf(tensor).any()++ @pytest.mark.parametrize("model_name", MODEL_LIST)+ def test_small_model_rtn_generation(self, model_name):+ ar = AutoRound(model_name, iters=0, scheme="FP8_STATIC", nsamples=16)+ model, folder = ar.quantize_and_save(output_dir=self.save_dir, format="llm_compressor")+ # all linears except lm_head should be quantized to FP8+ fp8_linear_count = 0+ for name, module in model.named_modules():+ if "FP8QLinear" in type(module).__name__:+ assert module.weight.dtype == torch.float8_e4m3fn, f"{name} is not in FP8"+ assert not self.check_nan_inf_in_tensor(+ module.weight.to(torch.float32)+ ), f"{name} has NaN or Inf in weights"+ fp8_linear_count += 1+ assert fp8_linear_count > 0, "No FP8 linear layer found in the quantized model"+ shutil.rmtree(self.save_dir, ignore_errors=True)