Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

3.2 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections.abc import Callable

import torch

from auto_round.context.base import BaseContext
from auto_round.utils.device import (
    clear_memory,
    clear_memory_if_reached_threshold,
    set_auto_device_map_for_block_with_tuning,
    set_non_auto_device_map,
)
from auto_round.utils.device_manager import device_manager

__all__ = ["CompressContext"]


class CompressContext(BaseContext):

    def __init__(
        self,
        low_cpu_mem_usage: bool = True,
        low_gpu_mem_usage: bool = False,
        enable_torch_compile: bool = True,
        calibration_data_device: str = "auto",
        is_immediate_packing: bool = False,
        is_immediate_saving: bool = False,
        formats: list | str | None = None,
        output_dir: str = "./compressed_models",
        static_kv_dtype: torch.dtype | None = None,  # TODO later this should be scheme wenhuach
        static_attention_dtype: torch.dtype | None = None,
        static_kv_granularity: str = "tensor",
        static_attention_granularity: str = "tensor",
        **kwargs,
    ) -> None:
        super().__init__()
        self.low_cpu_mem_usage = low_cpu_mem_usage
        self.low_gpu_mem_usage = low_gpu_mem_usage
        # Device placement for block calibration data: auto | off | cpu | csv.
        # Threaded from the same-name API/CLI parameter (single source of truth).
        self.calibration_data_device = calibration_data_device
        self.formats = formats
        self.output_dir = output_dir
        # All device / device-list state lives on the process-wide DeviceManager
        # singleton, which is configured from ``device_map`` before this context is
        # created.  CompressContext just reads from it -- it never owns a copy.
        self.cache_device = torch.device("cpu") if low_gpu_mem_usage else device_manager.device

        self.enable_torch_compile = enable_torch_compile
        self.immediate_packing = is_immediate_packing
        self.is_immediate_packing = is_immediate_packing
        self.is_immediate_saving = is_immediate_saving
        self.static_kv_dtype = static_kv_dtype
        self.static_attention_dtype = static_attention_dtype
        self.static_kv_granularity = static_kv_granularity
        self.static_attention_granularity = static_attention_granularity
        # Resolved --data_parallel policy (ParallelPolicy or None = off);
        # set once at the CLI entry, read by the data-parallel engine
        self.parallel_policy = kwargs.pop("parallel_policy", None)

    def clear_memory(self, tensor=None):
        """Clear GPU/CPU memory only when ``low_gpu_mem_usage`` is enabled."""
        if self.low_gpu_mem_usage:
            clear_memory(tensor, device_list=device_manager.device_list)