Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

5.1 KB
Raw Blame
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright (c) 2023 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


def check_compressed_tensors_supported(raise_error: bool = False):
    try:
        import compressed_tensors  # noqa: F401

        return True
    except ImportError:
        msg = (
            "Please `pip install compressed-tensors` since the 'llm_compressor' output format requires "
            "the optional dependency 'compressed-tensors'."
        )
        if raise_error:
            raise ImportError(msg) from None
        return False


if check_compressed_tensors_supported():
    from compressed_tensors.quantization import (  # pylint: disable=E0401
        QuantizationConfig,
        QuantizationScheme,
        QuantizationStatus,
        is_preset_scheme,
        preset_name_to_scheme,
    )


# please refer to https://github.com/vllm-project/llm-compressor/blob/
# 29f4d5644b48e9c8ebb7e36d5be9f7c92747ceb7/src/llmcompressor/modifiers/quantization/quantization/mixin.py#L168
def initialize_quantization(scheme, targets=None, config_groups=None, kv_cache_scheme=None, ignore=None):
    """
    Attach quantization schemes to modules in the model and initialize the quantization config
    """

    # apply scheme and status to model
    if targets is None:
        targets = ["Linear"]
    if ignore is None:
        ignore = ["lm_head"]
    using_mxfp4_for_mxfp8 = False
    check_compressed_tensors_supported(raise_error=True)
    if scheme is not None and config_groups is not None:
        raise ValueError("Please specify either `scheme` or `config_groups`")

    if scheme is not None:
        # takes precedence over config_groups
        if not is_preset_scheme(scheme) and scheme.lower() == "mxfp8":
            scheme = "mxfp4"
            using_mxfp4_for_mxfp8 = True  # mxfp8 is not supported.
        if isinstance(scheme, str) and is_preset_scheme(scheme):
            # attach targets to scheme
            scheme = {scheme: targets}

        config_groups = {}
        for idx, key in enumerate(scheme.keys()):
            if is_preset_scheme(key):
                scheme = preset_name_to_scheme(key, scheme[key])
            else:
                scheme = QuantizationScheme.model_validate({"targets": scheme[key], **scheme})
            if using_mxfp4_for_mxfp8:
                scheme.weights.num_bits = 8
                scheme.input_activations.num_bits = 8
                scheme.format = "mxfp8-quantized"

            group_name = f"group_{idx}"
            config_groups[group_name] = scheme

    if config_groups is None or len(config_groups) == 0:
        default_quant_scheme = QuantizationScheme(targets=targets)
        config_groups = {"group_0": default_quant_scheme}

    return QuantizationConfig(
        config_groups=config_groups,
        kv_cache_scheme=kv_cache_scheme,
        quantization_status=QuantizationStatus.COMPRESSED,
        ignore=ignore,
    )


def initialize_nvfp4_e5m3_quantization(ignore=None, with_global_scale=False):
    """Build stable compressed-tensors metadata for NVFP4 E5M3 (optionally with per-tensor global scales)."""

    def quant_args(dynamic):
        return {
            "actorder": None,
            "block_structure": None,
            "dynamic": dynamic,
            "group_size": 16,
            "num_bits": 4,
            "observer": "minmax",
            "observer_kwargs": {},
            "strategy": "tensor_group",
            "symmetric": True,
            "type": "float",
        }

    return {
        "config_groups": {
            "group_0": {
                "input_activations": quant_args("local"),
                "output_activations": None,
                "targets": ["Linear"],
                "weights": quant_args(False),
            }
        },
        "format": "nvfp4-e5m3-gs-pack-quantized" if with_global_scale else "nvfp4-e5m3-pack-quantized",
        "global_compression_ratio": None,
        "ignore": list(dict.fromkeys(ignore or ["lm_head"])),
        "kv_cache_scheme": None,
        "quant_method": "compressed-tensors",
        "quantization_status": "compressed",
    }