PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright (c) 2023 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
def check_compressed_tensors_supported(raise_error: bool = False):
try:
import compressed_tensors # noqa: F401
return True
except ImportError:
msg = (
"Please `pip install compressed-tensors` since the 'llm_compressor' output format requires "
"the optional dependency 'compressed-tensors'."
)
if raise_error:
raise ImportError(msg) from None
return False
if check_compressed_tensors_supported():
from compressed_tensors.quantization import ( # pylint: disable=E0401
QuantizationConfig,
QuantizationScheme,
QuantizationStatus,
is_preset_scheme,
preset_name_to_scheme,
)
# please refer to https://github.com/vllm-project/llm-compressor/blob/
# 29f4d5644b48e9c8ebb7e36d5be9f7c92747ceb7/src/llmcompressor/modifiers/quantization/quantization/mixin.py#L168
def initialize_quantization(scheme, targets=None, config_groups=None, kv_cache_scheme=None, ignore=None):
"""
Attach quantization schemes to modules in the model and initialize the quantization config
"""
# apply scheme and status to model
if targets is None:
targets = ["Linear"]
if ignore is None:
ignore = ["lm_head"]
using_mxfp4_for_mxfp8 = False
check_compressed_tensors_supported(raise_error=True)
if scheme is not None and config_groups is not None:
raise ValueError("Please specify either `scheme` or `config_groups`")
if scheme is not None:
# takes precedence over config_groups
if not is_preset_scheme(scheme) and scheme.lower() == "mxfp8":
scheme = "mxfp4"
using_mxfp4_for_mxfp8 = True # mxfp8 is not supported.
if isinstance(scheme, str) and is_preset_scheme(scheme):
# attach targets to scheme
scheme = {scheme: targets}
config_groups = {}
for idx, key in enumerate(scheme.keys()):
if is_preset_scheme(key):
scheme = preset_name_to_scheme(key, scheme[key])
else:
scheme = QuantizationScheme.model_validate({"targets": scheme[key], **scheme})
if using_mxfp4_for_mxfp8:
scheme.weights.num_bits = 8
scheme.input_activations.num_bits = 8
scheme.format = "mxfp8-quantized"
group_name = f"group_{idx}"
config_groups[group_name] = scheme
if config_groups is None or len(config_groups) == 0:
default_quant_scheme = QuantizationScheme(targets=targets)
config_groups = {"group_0": default_quant_scheme}
return QuantizationConfig(
config_groups=config_groups,
kv_cache_scheme=kv_cache_scheme,
quantization_status=QuantizationStatus.COMPRESSED,
ignore=ignore,
)
def initialize_nvfp4_e5m3_quantization(ignore=None, with_global_scale=False):
"""Build stable compressed-tensors metadata for NVFP4 E5M3 (optionally with per-tensor global scales)."""
def quant_args(dynamic):
return {
"actorder": None,
"block_structure": None,
"dynamic": dynamic,
"group_size": 16,
"num_bits": 4,
"observer": "minmax",
"observer_kwargs": {},
"strategy": "tensor_group",
"symmetric": True,
"type": "float",
}
return {
"config_groups": {
"group_0": {
"input_activations": quant_args("local"),
"output_activations": None,
"targets": ["Linear"],
"weights": quant_args(False),
}
},
"format": "nvfp4-e5m3-gs-pack-quantized" if with_global_scale else "nvfp4-e5m3-pack-quantized",
"global_compression_ratio": None,
"ignore": list(dict.fromkeys(ignore or ["lm_head"])),
"kv_cache_scheme": None,
"quant_method": "compressed-tensors",
"quantization_status": "compressed",
}