Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

7.7 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch
from transformers.quantizers.quantizer_finegrained_fp8 import (
    FineGrainedFP8HfQuantizer as OriginalFineGrainedFP8HfQuantizer,
)

from auto_round.utils import is_transformers_version_greater_or_equal_5, is_transformers_version_greater_or_equal_5_4_0
from auto_round.utils import logger as auto_round_logger
from auto_round.utils.device import override_cuda_device_capability


# Patching replace_with_fp8_linear to disable expert replacement
# https://github.com/huggingface/transformers/blob/78bb85146c59258a0710c8d08311d98d52303c38/src/transformers/integrations/finegrained_fp8.py#L720
def oot_replace_with_fp8_linear(
    model, modules_to_not_convert: list[str] | None = None, quantization_config=None, pre_quantized=False
):
    """
    A helper function to replace all `torch.nn.Linear` modules by `FP8Linear` modules.

    Parameters:
        model (`torch.nn.Module`):
            Input model or `torch.nn.Module` as the function is run recursively.
        modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`):
            Names of the modules to not convert. In practice we keep the `lm_head`
            in full precision for numerical stability reasons.
        quantization_config (`FbgemmFp8Config`):
            The quantization config object that contains the quantization parameters.
        pre_quantized (`bool`, defaults to `False`):
            Whether the model is pre-quantized or not
    """
    from transformers.integrations.finegrained_fp8 import (
        FP8Linear,
        logger,
        should_convert_module,
    )

    if quantization_config.dequantize:
        return model

    has_been_replaced = False
    for module_name, module in model.named_modules():
        if not should_convert_module(module_name, modules_to_not_convert):
            continue
        # we need this to correctly materialize the weights during quantization
        module_kwargs = {} if pre_quantized else {"dtype": None}
        new_module = None

        with torch.device("meta"):
            # Note: Disable replacing experts, as we do not want concatenated experts
            # if module_name.endswith(".experts"):
            #     new_module = FP8Expert(
            #         config=model.config, block_size=quantization_config.weight_block_size, **module_kwargs
            #     )
            # elif isinstance(module, nn.Linear):
            if isinstance(module, torch.nn.Linear):
                # handle bias
                if is_transformers_version_greater_or_equal_5_4_0():
                    module_kwargs["has_bias"] = module.bias is not None
                else:
                    module_kwargs["bias"] = module.bias is not None
                new_module = FP8Linear(
                    in_features=module.in_features,
                    out_features=module.out_features,
                    activation_scheme=quantization_config.activation_scheme,
                    block_size=quantization_config.weight_block_size,
                    **module_kwargs,
                )
            if new_module is not None:
                model.set_submodule(module_name, new_module)
                has_been_replaced = True

    if not has_been_replaced:
        logger.warning(
            "You are loading your model using fp8 but no linear modules were found in your model."
            " Please double check your model architecture."
        )
    return model


_orig_validate_environment = OriginalFineGrainedFP8HfQuantizer.validate_environment


def _patch_fp8_dequantize_for_ragged_tiles():
    """Patch transformers FP8 dequantization to handle over-provisioned scale grids.

    Some pre-quantized checkpoints keep FP8 weights in non-padded shape while the
    corresponding ``weight_scale_inv`` grid is computed on padded block tiles.
    Example: weight ``(576, 7168)`` with scale grid ``(5, 56)`` for block size
    ``(128, 128)`` (5 rows of tiles implies padded rows to 640).

    Upstream strict divisibility checks fail on those tensors. We keep upstream
    behavior by default and only fall back on this specific shape mismatch.
    """

    try:
        import transformers.integrations.finegrained_fp8 as transformers_fp8

        from auto_round.utils.weight_handler import _dequant_fp8_linear_weight
    except ImportError:
        return

    if not hasattr(transformers_fp8, "Fp8Dequantize"):
        return

    cls = transformers_fp8.Fp8Dequantize
    if hasattr(cls, "_auto_round_orig_dequantize_one"):
        return

    if not hasattr(cls, "_dequantize_one"):
        return

    cls._auto_round_orig_dequantize_one = cls._dequantize_one

    def _dequantize_one_with_padding_fallback(self, quantized, scales, output_dtype=None):
        try:
            return cls._auto_round_orig_dequantize_one(self, quantized, scales, output_dtype=output_dtype)
        except ValueError as e:
            if "not divisible by scale grid" not in str(e):
                raise

            # Match upstream MXFP8 handling for uint8 E8M0 scales.
            if scales.dtype == torch.uint8:
                scales = (scales.to(torch.float32) - 127.0).exp2()

            block_size = None
            qcfg = getattr(self.hf_quantizer, "quantization_config", None)
            if qcfg is not None:
                if isinstance(qcfg, dict):
                    block_size = qcfg.get("weight_block_size")
                else:
                    block_size = getattr(qcfg, "weight_block_size", None)

            # Falls back to AutoRound's robust dequantizer that supports
            # over-provisioned scale grids by padding then slicing back.
            dequantized = _dequant_fp8_linear_weight(quantized, scales, block_size=block_size)
            if output_dtype is not None:
                dequantized = dequantized.to(output_dtype)
            return dequantized

    cls._dequantize_one = _dequantize_one_with_padding_fallback
    auto_round_logger.debug("Applied FP8 ragged-tile dequantization fallback patch.")


@override_cuda_device_capability()
def oot_validate_environment(self, *args, **kwargs):
    return _orig_validate_environment(self, *args, **kwargs)


def apply_fp8_expert_replacement_patch():
    # Apply the ragged-tile dequantization fallback unconditionally so that
    # dequantize-on-load (FineGrainedFP8Config(dequantize=True)) works on
    # CPU and HPU as well as CUDA.
    _patch_fp8_dequantize_for_ragged_tiles()

    if is_transformers_version_greater_or_equal_5() and torch.cuda.is_available():
        try:
            import transformers.integrations.finegrained_fp8 as transformers_fp8

            transformers_fp8.replace_with_fp8_linear = oot_replace_with_fp8_linear
            auto_round_logger.debug("Applied FP8 expert replacement patch to transformers.")
            OriginalFineGrainedFP8HfQuantizer.validate_environment = oot_validate_environment
            auto_round_logger.debug(
                "Patched FineGrainedFP8HfQuantizer.validate_environment to bypass device "
                "capability check for loading FP8 models on unsupported GPUs."
            )
        except ImportError as e:
            auto_round_logger.warning(f"Could not apply FP8 expert replacement patch as {e}.")


apply_fp8_expert_replacement_patch()