Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

8.5 KB
Raw Blame
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import Any

import torch

from auto_round.algorithms.transforms.hadamard import HADAMARDS, RotationConfig
from auto_round.compressors.utils import is_mx_fp, is_nv_fp
from auto_round.utils import logger

SUPPORTED_QUANTIZATION_SCHEMES = ["MXFP8", "MXFP4", "NVFP4"]


FP8_GRANULARITY_TENSOR = "tensor"
FP8_GRANULARITY_HEAD = "head"


def normalize_fp8_granularity(granularity: str | None) -> str:
    if granularity is None:
        return FP8_GRANULARITY_TENSOR
    granularity = granularity.lower()
    if granularity in (FP8_GRANULARITY_TENSOR, FP8_GRANULARITY_HEAD):
        return granularity
    raise ValueError(
        f"Invalid FP8 calibration granularity: {granularity}. "
        f"Supported granularities are: {FP8_GRANULARITY_TENSOR}, {FP8_GRANULARITY_HEAD}."
    )


def per_tensor_fp8_qdq(
    tensor: torch.Tensor, tensor_max: None | torch.Tensor = None
) -> tuple[torch.Tensor, torch.Tensor]:
    from auto_round.data_type.fp8 import quant_fp8_sym

    qdq_tensor, scale, _ = quant_fp8_sym(tensor, max_scale=1.0, tensor_max=tensor_max, group_size=0, v=0)
    return qdq_tensor, scale


def per_head_fp8_qdq(tensor: torch.Tensor, tensor_max: None | torch.Tensor = None) -> tuple[torch.Tensor, torch.Tensor]:
    """Quantize/dequantize an attention tensor with one FP8 scale per attention head."""
    if tensor.dim() < 4:
        raise ValueError(f"Per-head FP8 calibration expects [batch, heads, seq, head_dim], got {tuple(tensor.shape)}")
    if tensor_max is not None and tensor_max.shape != (tensor.shape[1],):
        raise ValueError(
            f"Per-head FP8 calibration expects tensor_max shape ({tensor.shape[1]},), " f"got {tuple(tensor_max.shape)}"
        )

    info = torch.finfo(torch.float8_e4m3fn)
    orig_dtype = tensor.dtype
    if tensor.dtype == torch.float16:
        tensor = tensor.to(torch.bfloat16)

    if tensor_max is None:
        max_tensor = tensor.abs().amax(dim=(0, 2, 3)).to(torch.float32)
    else:
        max_tensor = tensor_max.to(tensor.device, dtype=torch.float32)

    scale = torch.clip(max_tensor / info.max, min=float(1.0 / (info.max * 512.0)))
    qdq_scale = scale.view(1, -1, 1, 1)
    fp8_res = torch.clip(tensor / qdq_scale, info.min, info.max)
    fp8_res = fp8_res.to(torch.float8_e4m3fn)
    return (fp8_res.to(tensor.dtype) * qdq_scale).to(orig_dtype), scale


def fp8_qdq(
    tensor: torch.Tensor, tensor_max: None | torch.Tensor = None, granularity: str | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
    granularity = normalize_fp8_granularity(granularity)
    if granularity == FP8_GRANULARITY_HEAD:
        return per_head_fp8_qdq(tensor, tensor_max=tensor_max)
    return per_tensor_fp8_qdq(tensor, tensor_max=tensor_max)


@torch.compiler.disable
def update_parameter_data(module: torch.nn.Module, new_val: torch.Tensor, name: str):
    """
    Update the data of a parameter in a module.
    If the parameter does not exist, it will be created.
    """
    if hasattr(module, name):
        param = getattr(module, name)
        if isinstance(param, torch.nn.Parameter):
            if param.shape == new_val.shape:
                param.data.copy_(new_val)
            else:
                logger.warning(
                    "Replacing parameter %s in module %s with a new shape: %s -> %s",
                    name,
                    module.__class__.__name__,
                    tuple(param.shape),
                    tuple(new_val.shape),
                )
                setattr(module, name, torch.nn.Parameter(new_val))
        else:
            module.register_parameter(name, torch.nn.Parameter(new_val))
    else:
        logger.warning_once(
            f"Parameter {name} not found in module {module.__class__.__name__}, creating new parameter."
        )
        module.register_parameter(name, torch.nn.Parameter(new_val))


NVFP4_KV_DTYPE = "nvfp4"
NVFP4_KV_BLOCK_SIZE = 16


def normalize_static_kv_dtype(static_kv_dtype: str | torch.dtype) -> torch.dtype | str:
    """Normalize the static KV-cache quantization dtype.

    Returns a ``torch.dtype`` for float dtypes, or the ``"nvfp4"`` sentinel
    (see :data:`NVFP4_KV_DTYPE`) for NVFP4 KV cache quantization.
    """
    if isinstance(static_kv_dtype, torch.dtype):
        return static_kv_dtype
    valid_dtype_name_lst = ["float16", "bfloat16", "fp8", "float32", "float", NVFP4_KV_DTYPE, "fp4", "float4_e2m1"]
    valid_torch_dtype = {
        "float16": torch.float16,
        "bfloat16": torch.bfloat16,
        "fp8": torch.float8_e4m3fn,
        "float8_e4m3fn": torch.float8_e4m3fn,
        "float32": torch.float32,
        "float": torch.float32,  # Alias for float32
    }
    if static_kv_dtype in (NVFP4_KV_DTYPE, "fp4", "float4_e2m1"):
        return NVFP4_KV_DTYPE
    if static_kv_dtype in valid_dtype_name_lst:
        new_dtype = valid_torch_dtype[static_kv_dtype]
    elif static_kv_dtype in valid_torch_dtype.values():
        new_dtype = static_kv_dtype
    else:
        raise ValueError(
            f"Invalid static kv dtype: {static_kv_dtype}. "
            # f"Valid options are: {', '.join(valid_dtype_name_lst  + list(valid_torch_dtype.values()))}."
        )
    return new_dtype


def is_attention_module(module: torch.nn.Module):
    # FIXME: Handle this better.
    return (
        "attention" in module.__class__.__name__.lower()
        and module.__class__.__name__ != "Llama4VisionAttention"  # llama4 vision attention doesn't have cache
        and (
            hasattr(module, "k_proj")
            or hasattr(module, "v_proj")
            or hasattr(module, "qkv_proj")
            or hasattr(module, "kv_b_proj")  # for DeepSpeed
        )
    )


def _clean_param_or_buff_if_exists(module: torch.nn.Module, name_tuple: tuple[str, ...]):
    """
    Deletes parameters or buffers from a module if they exist.

    :param module: module to delete parameters/buffers from
    :param name_tuple: tuple of parameter/buffer names to delete
    """
    for name in name_tuple:
        if hasattr(module, name):
            try:
                delattr(module, name)
            except Exception as e:
                logger.warning(f"Could not delete {name} from module {module}: {e}")


def clean_model_parameters_and_buffers_(model: torch.nn.Module, name_tuple: tuple[str, ...]):
    """
    Cleans parameters and buffers from all modules in the model.

    :param model: model to clean parameters/buffers from
    :param name_tuple: tuple of parameter/buffer names to delete
    """
    for module in model.modules():
        _clean_param_or_buff_if_exists(module, name_tuple)


def is_triton_kernel_available(data_type: str) -> bool:
    """
    Best-effort check for whether Triton kernel path can be used.
    """
    if is_nv_fp(data_type):
        return False
    try:
        import triton  # pylint: disable=E0401
    except Exception:
        return False

    if not torch.cuda.is_available():
        return False

    try:
        from auto_round.algorithms.transforms.hadamard.utils.triton.mxfp4 import (  # pylint: disable=E0401
            mxfp4_forward_kernel_wrapper,
        )
    except Exception:
        return False

    return True


def dump_group_size_to_rotation_config(rotation_config: str | dict | RotationConfig, group_size: int):
    from auto_round.algorithms.transforms.hadamard.config import dump_group_size_to_rotation_config as _impl

    return _impl(rotation_config, group_size)


def to_dict_rotation_config(rotation_config: str | dict | RotationConfig):
    from auto_round.algorithms.transforms.hadamard.config import to_dict_rotation_config as _impl

    return _impl(rotation_config)


def normalize_rotation_config(rotation_config: str | dict | RotationConfig | None, data_type: str) -> dict[str, Any]:
    from auto_round.algorithms.transforms.hadamard.config import normalize_rotation_config as _impl

    return _impl(rotation_config, data_type)


def check_supported_schemes(scheme: str):
    if scheme not in SUPPORTED_QUANTIZATION_SCHEMES:
        raise ValueError(
            f"Unsupported quantization scheme: {scheme}. Currently {SUPPORTED_QUANTIZATION_SCHEMES} are supported."
        )