PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections.abc import Callable
import torch
from auto_round.context.base import BaseContext
from auto_round.utils.device import (
clear_memory,
clear_memory_if_reached_threshold,
set_auto_device_map_for_block_with_tuning,
set_non_auto_device_map,
)
from auto_round.utils.device_manager import device_manager
__all__ = ["CompressContext"]
class CompressContext(BaseContext):
def __init__(
self,
low_cpu_mem_usage: bool = True,
low_gpu_mem_usage: bool = False,
enable_torch_compile: bool = True,
calibration_data_device: str = "auto",
is_immediate_packing: bool = False,
is_immediate_saving: bool = False,
formats: list | str | None = None,
output_dir: str = "./compressed_models",
static_kv_dtype: torch.dtype | None = None, # TODO later this should be scheme wenhuach
static_attention_dtype: torch.dtype | None = None,
static_kv_granularity: str = "tensor",
static_attention_granularity: str = "tensor",
**kwargs,
) -> None:
super().__init__()
self.low_cpu_mem_usage = low_cpu_mem_usage
self.low_gpu_mem_usage = low_gpu_mem_usage
# Device placement for block calibration data: auto | off | cpu | csv.
# Threaded from the same-name API/CLI parameter (single source of truth).
self.calibration_data_device = calibration_data_device
self.formats = formats
self.output_dir = output_dir
# All device / device-list state lives on the process-wide DeviceManager
# singleton, which is configured from ``device_map`` before this context is
# created. CompressContext just reads from it -- it never owns a copy.
self.cache_device = torch.device("cpu") if low_gpu_mem_usage else device_manager.device
self.enable_torch_compile = enable_torch_compile
self.immediate_packing = is_immediate_packing
self.is_immediate_packing = is_immediate_packing
self.is_immediate_saving = is_immediate_saving
self.static_kv_dtype = static_kv_dtype
self.static_attention_dtype = static_attention_dtype
self.static_kv_granularity = static_kv_granularity
self.static_attention_granularity = static_attention_granularity
# Resolved --data_parallel policy (ParallelPolicy or None = off);
# set once at the CLI entry, read by the data-parallel engine
self.parallel_policy = kwargs.pop("parallel_policy", None)
def clear_memory(self, tensor=None):
"""Clear GPU/CPU memory only when ``low_gpu_mem_usage`` is enabled."""
if self.low_gpu_mem_usage:
clear_memory(tensor, device_list=device_manager.device_list)