PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Any
import torch
from auto_round.algorithms.transforms.hadamard import HADAMARDS, RotationConfig
from auto_round.compressors.utils import is_mx_fp, is_nv_fp
from auto_round.utils import logger
SUPPORTED_QUANTIZATION_SCHEMES = ["MXFP8", "MXFP4", "NVFP4"]
FP8_GRANULARITY_TENSOR = "tensor"
FP8_GRANULARITY_HEAD = "head"
def normalize_fp8_granularity(granularity: str | None) -> str:
if granularity is None:
return FP8_GRANULARITY_TENSOR
granularity = granularity.lower()
if granularity in (FP8_GRANULARITY_TENSOR, FP8_GRANULARITY_HEAD):
return granularity
raise ValueError(
f"Invalid FP8 calibration granularity: {granularity}. "
f"Supported granularities are: {FP8_GRANULARITY_TENSOR}, {FP8_GRANULARITY_HEAD}."
)
def per_tensor_fp8_qdq(
tensor: torch.Tensor, tensor_max: None | torch.Tensor = None
) -> tuple[torch.Tensor, torch.Tensor]:
from auto_round.data_type.fp8 import quant_fp8_sym
qdq_tensor, scale, _ = quant_fp8_sym(tensor, max_scale=1.0, tensor_max=tensor_max, group_size=0, v=0)
return qdq_tensor, scale
def per_head_fp8_qdq(tensor: torch.Tensor, tensor_max: None | torch.Tensor = None) -> tuple[torch.Tensor, torch.Tensor]:
"""Quantize/dequantize an attention tensor with one FP8 scale per attention head."""
if tensor.dim() < 4:
raise ValueError(f"Per-head FP8 calibration expects [batch, heads, seq, head_dim], got {tuple(tensor.shape)}")
if tensor_max is not None and tensor_max.shape != (tensor.shape[1],):
raise ValueError(
f"Per-head FP8 calibration expects tensor_max shape ({tensor.shape[1]},), " f"got {tuple(tensor_max.shape)}"
)
info = torch.finfo(torch.float8_e4m3fn)
orig_dtype = tensor.dtype
if tensor.dtype == torch.float16:
tensor = tensor.to(torch.bfloat16)
if tensor_max is None:
max_tensor = tensor.abs().amax(dim=(0, 2, 3)).to(torch.float32)
else:
max_tensor = tensor_max.to(tensor.device, dtype=torch.float32)
scale = torch.clip(max_tensor / info.max, min=float(1.0 / (info.max * 512.0)))
qdq_scale = scale.view(1, -1, 1, 1)
fp8_res = torch.clip(tensor / qdq_scale, info.min, info.max)
fp8_res = fp8_res.to(torch.float8_e4m3fn)
return (fp8_res.to(tensor.dtype) * qdq_scale).to(orig_dtype), scale
def fp8_qdq(
tensor: torch.Tensor, tensor_max: None | torch.Tensor = None, granularity: str | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
granularity = normalize_fp8_granularity(granularity)
if granularity == FP8_GRANULARITY_HEAD:
return per_head_fp8_qdq(tensor, tensor_max=tensor_max)
return per_tensor_fp8_qdq(tensor, tensor_max=tensor_max)
@torch.compiler.disable
def update_parameter_data(module: torch.nn.Module, new_val: torch.Tensor, name: str):
"""
Update the data of a parameter in a module.
If the parameter does not exist, it will be created.
"""
if hasattr(module, name):
param = getattr(module, name)
if isinstance(param, torch.nn.Parameter):
if param.shape == new_val.shape:
param.data.copy_(new_val)
else:
logger.warning(
"Replacing parameter %s in module %s with a new shape: %s -> %s",
name,
module.__class__.__name__,
tuple(param.shape),
tuple(new_val.shape),
)
setattr(module, name, torch.nn.Parameter(new_val))
else:
module.register_parameter(name, torch.nn.Parameter(new_val))
else:
logger.warning_once(
f"Parameter {name} not found in module {module.__class__.__name__}, creating new parameter."
)
module.register_parameter(name, torch.nn.Parameter(new_val))
NVFP4_KV_DTYPE = "nvfp4"
NVFP4_KV_BLOCK_SIZE = 16
def normalize_static_kv_dtype(static_kv_dtype: str | torch.dtype) -> torch.dtype | str:
"""Normalize the static KV-cache quantization dtype.
Returns a ``torch.dtype`` for float dtypes, or the ``"nvfp4"`` sentinel
(see :data:`NVFP4_KV_DTYPE`) for NVFP4 KV cache quantization.
"""
if isinstance(static_kv_dtype, torch.dtype):
return static_kv_dtype
valid_dtype_name_lst = ["float16", "bfloat16", "fp8", "float32", "float", NVFP4_KV_DTYPE, "fp4", "float4_e2m1"]
valid_torch_dtype = {
"float16": torch.float16,
"bfloat16": torch.bfloat16,
"fp8": torch.float8_e4m3fn,
"float8_e4m3fn": torch.float8_e4m3fn,
"float32": torch.float32,
"float": torch.float32, # Alias for float32
}
if static_kv_dtype in (NVFP4_KV_DTYPE, "fp4", "float4_e2m1"):
return NVFP4_KV_DTYPE
if static_kv_dtype in valid_dtype_name_lst:
new_dtype = valid_torch_dtype[static_kv_dtype]
elif static_kv_dtype in valid_torch_dtype.values():
new_dtype = static_kv_dtype
else:
raise ValueError(
f"Invalid static kv dtype: {static_kv_dtype}. "
# f"Valid options are: {', '.join(valid_dtype_name_lst + list(valid_torch_dtype.values()))}."
)
return new_dtype
def is_attention_module(module: torch.nn.Module):
# FIXME: Handle this better.
return (
"attention" in module.__class__.__name__.lower()
and module.__class__.__name__ != "Llama4VisionAttention" # llama4 vision attention doesn't have cache
and (
hasattr(module, "k_proj")
or hasattr(module, "v_proj")
or hasattr(module, "qkv_proj")
or hasattr(module, "kv_b_proj") # for DeepSpeed
)
)
def _clean_param_or_buff_if_exists(module: torch.nn.Module, name_tuple: tuple[str, ...]):
"""
Deletes parameters or buffers from a module if they exist.
:param module: module to delete parameters/buffers from
:param name_tuple: tuple of parameter/buffer names to delete
"""
for name in name_tuple:
if hasattr(module, name):
try:
delattr(module, name)
except Exception as e:
logger.warning(f"Could not delete {name} from module {module}: {e}")
def clean_model_parameters_and_buffers_(model: torch.nn.Module, name_tuple: tuple[str, ...]):
"""
Cleans parameters and buffers from all modules in the model.
:param model: model to clean parameters/buffers from
:param name_tuple: tuple of parameter/buffer names to delete
"""
for module in model.modules():
_clean_param_or_buff_if_exists(module, name_tuple)
def is_triton_kernel_available(data_type: str) -> bool:
"""
Best-effort check for whether Triton kernel path can be used.
"""
if is_nv_fp(data_type):
return False
try:
import triton # pylint: disable=E0401
except Exception:
return False
if not torch.cuda.is_available():
return False
try:
from auto_round.algorithms.transforms.hadamard.utils.triton.mxfp4 import ( # pylint: disable=E0401
mxfp4_forward_kernel_wrapper,
)
except Exception:
return False
return True
def dump_group_size_to_rotation_config(rotation_config: str | dict | RotationConfig, group_size: int):
from auto_round.algorithms.transforms.hadamard.config import dump_group_size_to_rotation_config as _impl
return _impl(rotation_config, group_size)
def to_dict_rotation_config(rotation_config: str | dict | RotationConfig):
from auto_round.algorithms.transforms.hadamard.config import to_dict_rotation_config as _impl
return _impl(rotation_config)
def normalize_rotation_config(rotation_config: str | dict | RotationConfig | None, data_type: str) -> dict[str, Any]:
from auto_round.algorithms.transforms.hadamard.config import normalize_rotation_config as _impl
return _impl(rotation_config, data_type)
def check_supported_schemes(scheme: str):
if scheme not in SUPPORTED_QUANTIZATION_SCHEMES:
raise ValueError(
f"Unsupported quantization scheme: {scheme}. Currently {SUPPORTED_QUANTIZATION_SCHEMES} are supported."
)