PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from functools import cache
import torch
from auto_round.logger import logger
@cache
def is_distributed():
import torch.distributed as dist
return dist.is_initialized() and dist.get_world_size() > 1
def setup_ddp_if_needed_(ar, block: torch.nn.Module, device_list: list[int]):
"""Prepare ``block`` for distributed training and return a gradient-sync hook.
Returns ``(block, sync_fn)`` where ``sync_fn()`` must be called after gradient
accumulation and before ``optimizer.step()``.
* **Non-distributed**: returns ``(block, noop)``.
* **Single GPU per rank**: wraps ``block`` with DDP, which synchronizes
gradients automatically during the backward pass. ``sync_fn`` is a no-op.
* **Multi-GPU per rank**: the block's submodules are sharded across GPUs
and cannot be DDP-wrapped. ``sync_fn`` performs a manual ``all_reduce``
(AVG) on every parameter that has a gradient, so that each rank sees the
same averaged gradient before taking a step.
.. note::
``ReduceOp.AVG`` is safe for SignSGD because
``sign(avg(g)) == sign(sum(g))`` for any non-zero sum.
Args:
ar: AutoRound instance invoking the helper.
block: Model block that may need DDP wrapping.
device_list: Device identifiers assigned to the current rank.
Returns:
``(block, sync_fn)``
"""
import torch.distributed as dist
from auto_round.algorithms.parallel.spmd import current_rank
rank = current_rank()
if rank is not None:
# in-process thread ranks (--data_parallel): each rank tunes
# its own replica; the hook averages gradients across the ranks
return block, lambda: rank.sync_gradients(block)
# The process group can be created and destroyed by individual tests or
# by successive launcher invocations. Do not reuse a cached result from
# a previous process-group lifetime.
is_distributed.cache_clear()
visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", "unset")
if not is_distributed():
return block, _noop_sync
num_devices = len(device_list)
if num_devices == 1:
from torch.nn.parallel import DistributedDataParallel as DDP
logger.warning_once("AutoRound DDP is an experimental feature, please use with caution.")
logger.trace(
"[Rank: %d] Wrapping block with DDP on device_list=%s, CUDA_VISIBLE_DEVICES=%s",
dist.get_rank(),
device_list,
visible_devices,
)
# Ensure all block parameters are on the DDP device before wrapping.
_move_block_to_device(block, device_list[0])
block = DDP(block, device_ids=device_list, find_unused_parameters=True)
return block, _noop_sync
# Multi-GPU per rank: block is sharded, can't DDP-wrap.
# Use manual all_reduce on gradients before optimizer step.
logger.warning_once("AutoRound multi-GPU DDP is an experimental feature, " "please use with caution.")
logger.trace(
(
"[Rank: %d] Multi-GPU(device_list=%s, CUDA_VISIBLE_DEVICES=%s) per rank (%d GPUs),"
" using manual all_reduce sync"
),
dist.get_rank(),
str(device_list),
visible_devices,
num_devices,
)
def _sync_gradients():
_all_reduce_model_grads(block)
return block, _sync_gradients
def _noop_sync():
pass
def make_index_sampler(nsamples: int, batch_size: int):
"""The tuning loop's sample sampler.
Serial and torchrun ranks (whose calibration data the caller already
partitioned) draw from all ``nsamples``. An in-process thread rank draws
from its own contiguous shard of the calibration pool.
"""
from auto_round.algorithms.parallel.spmd import current_rank
from auto_round.compressors.utils import IndexSampler
rank = current_rank()
if rank is not None:
return rank.index_sampler(nsamples, batch_size)
return IndexSampler(nsamples, batch_size)
def all_reduce_loss(loss: float, weight: float = 1.0) -> float:
"""Global iteration loss across ranks; identity when not data-parallel.
``loss`` is the rank's normalized loss and ``weight`` its normalizer
(element or valid-token count), so the result is the weighted mean
sum(loss * weight) / sum(weight) -- the serial value over the union of
the ranks' samples. Every rank must take the same best-iteration and
early-stop decisions (it holds the same parameters after each synced
step), so the decisions use the global loss.
"""
from auto_round.algorithms.parallel.spmd import current_rank
rank = current_rank()
if rank is not None:
return rank.all_reduce_weighted_mean(loss, weight)
return loss
def _move_block_to_device(module: torch.nn.Module, device):
"""Move *module* to *device*, used before DDP wrapping."""
module.to(device)
def _all_reduce_model_grads(module: torch.nn.Module):
"""All-reduce (AVG) gradients of all parameters in *module* across ranks."""
import torch.distributed as dist
# Lightweight CUDA smoke tests can call this helper without creating a
# process group. Synchronization is only meaningful for initialized
# distributed jobs; otherwise leave gradients unchanged.
if not dist.is_initialized() or dist.get_world_size() <= 1:
return
comm_device = torch.cuda.current_device() if torch.cuda.is_available() else None
if comm_device is not None:
comm_device = torch.device("cuda", comm_device)
for param in module.parameters():
if param.grad is not None:
grad = param.grad
if comm_device is not None and grad.is_cuda and grad.device != comm_device:
synced = grad.to(comm_device)
dist.all_reduce(synced, op=dist.ReduceOp.AVG)
grad.copy_(synced.to(grad.device))
else:
dist.all_reduce(grad, op=dist.ReduceOp.AVG)