PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import torch
from transformers.quantizers.quantizer_finegrained_fp8 import (
FineGrainedFP8HfQuantizer as OriginalFineGrainedFP8HfQuantizer,
)
from auto_round.utils import is_transformers_version_greater_or_equal_5, is_transformers_version_greater_or_equal_5_4_0
from auto_round.utils import logger as auto_round_logger
from auto_round.utils.device import override_cuda_device_capability
# Patching replace_with_fp8_linear to disable expert replacement
# https://github.com/huggingface/transformers/blob/78bb85146c59258a0710c8d08311d98d52303c38/src/transformers/integrations/finegrained_fp8.py#L720
def oot_replace_with_fp8_linear(
model, modules_to_not_convert: list[str] | None = None, quantization_config=None, pre_quantized=False
):
"""
A helper function to replace all `torch.nn.Linear` modules by `FP8Linear` modules.
Parameters:
model (`torch.nn.Module`):
Input model or `torch.nn.Module` as the function is run recursively.
modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`):
Names of the modules to not convert. In practice we keep the `lm_head`
in full precision for numerical stability reasons.
quantization_config (`FbgemmFp8Config`):
The quantization config object that contains the quantization parameters.
pre_quantized (`bool`, defaults to `False`):
Whether the model is pre-quantized or not
"""
from transformers.integrations.finegrained_fp8 import (
FP8Linear,
logger,
should_convert_module,
)
if quantization_config.dequantize:
return model
has_been_replaced = False
for module_name, module in model.named_modules():
if not should_convert_module(module_name, modules_to_not_convert):
continue
# we need this to correctly materialize the weights during quantization
module_kwargs = {} if pre_quantized else {"dtype": None}
new_module = None
with torch.device("meta"):
# Note: Disable replacing experts, as we do not want concatenated experts
# if module_name.endswith(".experts"):
# new_module = FP8Expert(
# config=model.config, block_size=quantization_config.weight_block_size, **module_kwargs
# )
# elif isinstance(module, nn.Linear):
if isinstance(module, torch.nn.Linear):
# handle bias
if is_transformers_version_greater_or_equal_5_4_0():
module_kwargs["has_bias"] = module.bias is not None
else:
module_kwargs["bias"] = module.bias is not None
new_module = FP8Linear(
in_features=module.in_features,
out_features=module.out_features,
activation_scheme=quantization_config.activation_scheme,
block_size=quantization_config.weight_block_size,
**module_kwargs,
)
if new_module is not None:
model.set_submodule(module_name, new_module)
has_been_replaced = True
if not has_been_replaced:
logger.warning(
"You are loading your model using fp8 but no linear modules were found in your model."
" Please double check your model architecture."
)
return model
_orig_validate_environment = OriginalFineGrainedFP8HfQuantizer.validate_environment
def _patch_fp8_dequantize_for_ragged_tiles():
"""Patch transformers FP8 dequantization to handle over-provisioned scale grids.
Some pre-quantized checkpoints keep FP8 weights in non-padded shape while the
corresponding ``weight_scale_inv`` grid is computed on padded block tiles.
Example: weight ``(576, 7168)`` with scale grid ``(5, 56)`` for block size
``(128, 128)`` (5 rows of tiles implies padded rows to 640).
Upstream strict divisibility checks fail on those tensors. We keep upstream
behavior by default and only fall back on this specific shape mismatch.
"""
try:
import transformers.integrations.finegrained_fp8 as transformers_fp8
from auto_round.utils.weight_handler import _dequant_fp8_linear_weight
except ImportError:
return
if not hasattr(transformers_fp8, "Fp8Dequantize"):
return
cls = transformers_fp8.Fp8Dequantize
if hasattr(cls, "_auto_round_orig_dequantize_one"):
return
if not hasattr(cls, "_dequantize_one"):
return
cls._auto_round_orig_dequantize_one = cls._dequantize_one
def _dequantize_one_with_padding_fallback(self, quantized, scales, output_dtype=None):
try:
return cls._auto_round_orig_dequantize_one(self, quantized, scales, output_dtype=output_dtype)
except ValueError as e:
if "not divisible by scale grid" not in str(e):
raise
# Match upstream MXFP8 handling for uint8 E8M0 scales.
if scales.dtype == torch.uint8:
scales = (scales.to(torch.float32) - 127.0).exp2()
block_size = None
qcfg = getattr(self.hf_quantizer, "quantization_config", None)
if qcfg is not None:
if isinstance(qcfg, dict):
block_size = qcfg.get("weight_block_size")
else:
block_size = getattr(qcfg, "weight_block_size", None)
# Falls back to AutoRound's robust dequantizer that supports
# over-provisioned scale grids by padding then slicing back.
dequantized = _dequant_fp8_linear_weight(quantized, scales, block_size=block_size)
if output_dtype is not None:
dequantized = dequantized.to(output_dtype)
return dequantized
cls._dequantize_one = _dequantize_one_with_padding_fallback
auto_round_logger.debug("Applied FP8 ragged-tile dequantization fallback patch.")
@override_cuda_device_capability()
def oot_validate_environment(self, *args, **kwargs):
return _orig_validate_environment(self, *args, **kwargs)
def apply_fp8_expert_replacement_patch():
# Apply the ragged-tile dequantization fallback unconditionally so that
# dequantize-on-load (FineGrainedFP8Config(dequantize=True)) works on
# CPU and HPU as well as CUDA.
_patch_fp8_dequantize_for_ragged_tiles()
if is_transformers_version_greater_or_equal_5() and torch.cuda.is_available():
try:
import transformers.integrations.finegrained_fp8 as transformers_fp8
transformers_fp8.replace_with_fp8_linear = oot_replace_with_fp8_linear
auto_round_logger.debug("Applied FP8 expert replacement patch to transformers.")
OriginalFineGrainedFP8HfQuantizer.validate_environment = oot_validate_environment
auto_round_logger.debug(
"Patched FineGrainedFP8HfQuantizer.validate_environment to bypass device "
"capability check for loading FP8 models on unsupported GPUs."
)
except ImportError as e:
auto_round_logger.warning(f"Could not apply FP8 expert replacement patch as {e}.")
apply_fp8_expert_replacement_patch()