PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import random
from typing import Any, Union
import torch
from torch.amp import autocast
from auto_round.compressors.config_resolution import LayerConfigResolutionError
# Explicit compatibility exports for callers that historically imported GGUF and
# ignore-layer helpers from compressors.utils.
from auto_round.compressors.layer_config_resolver import get_fp_layer_names
from auto_round.export.formats.backends.gguf import (
_apply_gguf_shape_fallback,
_infer_gguf_n_layers_from_model,
_resolve_gguf_n_layers,
get_layer_config_by_gguf_format,
gguf_type_fallback,
)
from auto_round.logger import logger
from auto_round.schemes import BackendDataType # re-exported: qlinear_fp/qlinear_int import it from here
from auto_round.schemes import (
QuantizationScheme,
is_mx_fp,
is_mx_int,
is_nv_fp,
is_standard_fp,
)
from auto_round.utils import (
check_to_quantized,
get_layer_names_in_block,
get_module,
)
from auto_round.utils.device_manager import device_manager
def _as_scheme(ar_or_scheme) -> "QuantizationScheme":
"""Resolve a compressor-like object or QuantizationScheme to a QuantizationScheme.
`ar` (the compressor) exposes the same flat attribute names as QuantizationScheme
(bits, data_type, act_bits, ...), so QuantizationScheme.from_dict can read them
directly without needing ar.scheme to already be resolved.
"""
if isinstance(ar_or_scheme, QuantizationScheme):
return ar_or_scheme
return QuantizationScheme(
bits=ar_or_scheme.bits,
group_size=ar_or_scheme.group_size,
sym=getattr(ar_or_scheme, "sym", True),
data_type=ar_or_scheme.data_type,
act_bits=ar_or_scheme.act_bits,
act_group_size=getattr(ar_or_scheme, "act_group_size", None),
act_sym=getattr(ar_or_scheme, "act_sym", None),
act_data_type=ar_or_scheme.act_data_type,
act_dynamic=getattr(ar_or_scheme, "act_dynamic", None),
super_bits=getattr(ar_or_scheme, "super_bits", None),
super_group_size=getattr(ar_or_scheme, "super_group_size", None),
)
# ``is_standard_fp`` / ``is_mx_fp`` / ``is_nv_fp`` / ``is_mx_int`` (data_type-string
# classifiers) now live in ``auto_round.schemes`` as the single authority and are
# re-exported above so existing ``from auto_round.compressors.utils import is_mx_fp``
# call sites keep working.
def is_wint_woq(ar):
"""Returns True for integer weight-only quantization with non-quantized activations (`act_bits >= 16`)."""
return _as_scheme(ar).is_wint_woq()
def is_wfp8afp8(ar):
return _as_scheme(ar).is_wfp8afp8()
def is_wint8aint8(ar):
return _as_scheme(ar).is_wint8aint8()
def is_act_static(ar_or_format):
if isinstance(ar_or_format, str):
return "fp8_static" in ar_or_format.lower()
return _as_scheme(ar_or_format).is_act_static()
def is_dynamic_wint8aint8(ar_or_format):
if isinstance(ar_or_format, str):
return "int8_w8a8" in ar_or_format.lower()
return _as_scheme(ar_or_format).is_dynamic_wint8aint8()
def is_wint4aint4(ar_or_scheme):
if isinstance(ar_or_scheme, str):
return "int4" in ar_or_scheme.lower()
return _as_scheme(ar_or_scheme).is_wint4aint4()
def is_dynamic_afp8(ar_or_format):
return _as_scheme(ar_or_format).is_dynamic_afp8()
def is_block_wfp8(ar_or_format):
return _as_scheme(ar_or_format).is_block_wfp8()
def block_forward(
block: torch.nn.Module,
input_ids: torch.Tensor,
input_others: dict,
amp: bool = False,
amp_dtype: torch.dtype = torch.float16,
device: torch.device = torch.device("cpu"),
output_return_id: int = 0,
) -> torch.Tensor | dict:
"""Performs a forward pass through a block with the given inputs.
Args:
block: The block to perform the forward pass on.
input_ids: The input IDs.
input_others: A dictionary containing other input data.
amp: A boolean indicating whether to use automatic mixed precision.
amp_dtype: The data type for automatic mixed precision.
device: The target device.
output_return_id: if the output has more than one tenor, return the specified idx tensor.
Returns:
output: The output of the forward pass.
"""
from auto_round.utils.model import to_device
if input_ids.device != device:
input_ids = to_device(input_ids, device)
input_others = to_device(input_others, device)
input_tuple = input_others.pop("positional_inputs", None)
if "alibi" in input_others.keys() and input_others["alibi"] is not None:
alibi = input_others["alibi"]
input_others["alibi"] = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])
from auto_round.special_model_handler import prepare_special_model_block_inputs
input_others, input_tuple = prepare_special_model_block_inputs(block, input_ids, input_others, input_tuple)
# Use the block's actual parameter name for the first positional argument.
# For DDP-wrapped blocks, inspect the underlying module's signature.
_actual_block = getattr(block, "module", block)
# ``inspect.signature`` is relatively expensive and the forward signature is
# fixed for the lifetime of a block, yet this runs on *every* batch of every
# reference/quantized-input/tuning pass (hundreds of thousands of calls for
# large nsamples). Resolve the parameter names once and cache them on the
# module so subsequent calls are a single attribute read.
param_names = getattr(_actual_block, "_ar_forward_param_names", None)
if param_names is None:
import inspect as _inspect
param_names = [p for p in _inspect.signature(_actual_block.forward).parameters.keys() if p != "self"]
try:
_actual_block._ar_forward_param_names = param_names
except (AttributeError, RuntimeError): # pragma: no cover - exotic module types
pass
block_input_kwarg = param_names[0] if param_names else "hidden_states"
if block_input_kwarg not in input_others:
input_others[block_input_kwarg] = input_ids
# Convert positional inputs to keyword args for any remaining positional parameters.
positional_inputs = input_tuple or ()
if positional_inputs:
for i, val in enumerate(positional_inputs):
param_idx = i + 1 # hidden_states is params[0]
if param_idx < len(param_names):
param_name = param_names[param_idx]
if param_name not in input_others:
input_others[param_name] = val
positional_inputs = ()
# DDP-mirror device audit (last line of defense): every tensor kwarg must
# sit on the same accelerator device as the block's parameters. A stale
# kwarg (e.g. an entry recorded on the primary GPU while the replica
# lives elsewhere) silently overrides the staged first-arg inside this
# function and crashes cross-device inside the block forward -- serial
# runs with a single shared device stay unaffected. Move the stragglers
# onto the block's device AND name them (fail-visible).
try:
_block_dev = next(block.parameters()).device
except StopIteration: # pragma: no cover - param-less blocks
_block_dev = device
def _audit(value, prefix, bad):
if torch.is_tensor(value):
if value.device.type in ("cuda", "xpu", "hpu") and value.device != _block_dev:
bad.append(f"{prefix}:{value.device}")
elif isinstance(value, (list, tuple)):
for _i, _v in enumerate(value):
_audit(_v, f"{prefix}[{_i}]", bad)
elif isinstance(value, dict):
for _k, _v in value.items():
_audit(_v, f"{prefix}.{_k}", bad)
def _relocate(value):
# Mirror _audit's condition exactly: only accelerator tensors that
# mismatch the block device move. A wholesale to_device() here would
# also migrate host tensors nested in lists/dicts onto the accelerator
# (semantic change + VRAM), which the audit never flagged.
if torch.is_tensor(value):
if value.device.type in ("cuda", "xpu", "hpu") and value.device != _block_dev:
return value.to(_block_dev)
return value
if isinstance(value, list):
return [_relocate(_v) for _v in value]
if isinstance(value, tuple):
return tuple(_relocate(_v) for _v in value)
if isinstance(value, dict):
return {k: _relocate(v) for k, v in value.items()}
return value
_bad = []
for _k, _v in input_others.items():
_audit(_v, str(_k), _bad)
if _bad:
logger.warning(
"[block-forward] device sweep: moving %d stale kwarg tensor(s) onto %s: %s%s",
len(_bad),
_block_dev,
", ".join(_bad[:8]),
" ..." if len(_bad) > 8 else "",
)
for _k in list(input_others.keys()):
if torch.is_tensor(input_others[_k]) and input_others[_k].device.type in ("cuda", "xpu", "hpu"):
if input_others[_k].device != _block_dev:
input_others[_k] = input_others[_k].to(_block_dev)
elif isinstance(input_others[_k], (list, tuple, dict)):
input_others[_k] = _relocate(input_others[_k])
if amp:
with autocast(device_type=str(device).split(":")[0], dtype=amp_dtype): # pragma: no cover
output = block(**input_others)
else:
output = block(**input_others)
if isinstance(output_return_id, int) and isinstance(output, (list, tuple)):
output = output[output_return_id]
return output
def check_skippable_keywords(key):
"""
Prints a reminder if a key is not stored during quantization fine-tuning.
"""
skippable_cache_keys = ("past_key_value",)
for cache_key in skippable_cache_keys:
if cache_key not in key:
return True
return False
def check_need_act_calibration(
is_act_dynamic: bool | None,
act_data_type: str | None = None,
act_bits: int | None = 16,
static_kv_dtype: str | None = None,
static_attention_dtype: str | None = None,
) -> bool:
if static_kv_dtype is not None or static_attention_dtype is not None:
return True
if act_bits is None or act_bits > 8:
return False
# None is dynamic
if is_act_dynamic is not None and not is_act_dynamic:
return True
return act_data_type is not None and "static" in act_data_type
def collect_best_params(block, cache_device="cpu"):
"""Collect the best parameters from the block to the specified device."""
params = {}
if hasattr(block, "orig_layer"):
for key in block.params.keys():
params[key] = block.params[key].data.to(cache_device, copy=True)
else:
for n, m in block.named_modules():
if hasattr(m, "orig_layer"):
params[n] = {}
for key in m.params.keys():
params[n][key] = m.params[key].data.to(cache_device, copy=True)
return params
def collect_best_params_local(block: torch.nn.Module) -> dict[str, Any]:
"""Best-params snapshot duplicated on each parameter's own device.
With a CPU cache device this is not used (host parking is the point of
``low_gpu_mem_usage``). With a non-CPU cache device the historical path
copied every parameter to that single device -- concentrating all devices'
snapshot bytes on one GPU and paying cross-device copies on every improving
iteration. Duplicating each parameter on the device that already hosts it
keeps the footprint spread like the weights themselves, removes the
cross-device traffic, and makes the unwrap copy-back local. Values and the
restore path are unchanged. Falls back to a host snapshot (with a warning)
when a local copy fails.
"""
params = {}
try:
if hasattr(block, "orig_layer"):
for key, p_ in block.params.items():
params[key] = p_.data.to(p_.data.device, copy=True)
else:
for n, m in block.named_modules():
if hasattr(m, "orig_layer"):
params[n] = {key: p_.data.to(p_.data.device, copy=True) for key, p_ in m.params.items()}
return params
except RuntimeError as e:
logger.warning("[snapshot] local copy failed (%s); parking the snapshot on host", e)
return collect_best_params(block, "cpu")
def snapshot_best_params(block: torch.nn.Module, cache_device: "str | torch.device" = "cpu") -> dict[str, Any]:
"""Collect the best-params snapshot, keeping multi-device blocks on-device.
A CPU ``cache_device`` (``low_gpu_mem_usage``) keeps the historical host
snapshot. A non-CPU ``cache_device`` means VRAM was budgeted for caching;
each parameter is then duplicated on its own device instead of gathering
every device's copies onto one cache device. Single-device blocks are
unaffected either way (the cache device already equals each parameter's
device).
"""
try:
non_cpu = torch.device(str(cache_device)).type != "cpu"
except (ValueError, RuntimeError):
non_cpu = False # unrecognized label: keep the historical path (it will raise as before)
if non_cpu:
return collect_best_params_local(block)
return collect_best_params(block, cache_device)
def infer_bits_by_data_type(data_type: str):
"""Infer bits by data_type
Args:
data_type (str): data_type
Returns:
int: bits inferred by data_type, None means cannot infer correct bits by data_type
"""
from auto_round.utils import SUPPORTED_DTYPES
if data_type is None:
return 16
for supported_dtype in SUPPORTED_DTYPES:
if data_type.startswith(supported_dtype) and len(data_type) > len(supported_dtype):
##first check the following two bits
suc_2str = data_type[len(supported_dtype) : len(supported_dtype) + 2]
if str.isdigit(suc_2str):
return int(suc_2str)
if str.isdigit(data_type[len(supported_dtype)]):
return int(data_type[len(supported_dtype)])
return None
def set_layer_config(
model: torch.nn.Module,
layer_config: dict[str, Union[str, dict, "QuantizationScheme"]],
default_scheme: Union[str, "QuantizationScheme"],
default_scale_dtype: torch.dtype | str,
supported_types: tuple,
inner_supported_types: tuple,
quant_block_list=None,
ignore_layers: str = "",
quant_lm_head: bool = False,
enable_gguf_official_mixed: bool = True,
is_mllm: bool = False,
fill_default_value=True,
) -> tuple[dict, bool, dict]:
"""Compatibility adapter for the pure layer-config resolver and explicit apply phase."""
from auto_round.compressors.config_resolution import (
ResolvedQuantizationConfig,
ResolvedScheme,
resolve_scheme_value,
)
from auto_round.compressors.layer_config_resolver import (
apply_plan_to_model,
extract_regex_config,
has_quantized_layer_outside_blocks,
resolve_layer_config,
)
from auto_round.schemes import get_gguf_scheme
if isinstance(default_scheme, ResolvedScheme):
resolved_scheme = default_scheme
elif isinstance(default_scheme, QuantizationScheme):
resolved_scheme = ResolvedScheme.from_scheme(
default_scheme,
preset_name=get_gguf_scheme(default_scheme) or None,
)
else:
resolved_scheme = resolve_scheme_value(default_scheme, {})
resolved = resolve_layer_config(
model=model,
scheme=resolved_scheme,
layer_config=layer_config,
scale_dtype=default_scale_dtype,
supported_types=supported_types,
inner_supported_types=inner_supported_types,
quant_block_list=quant_block_list,
ignore_layers=ignore_layers,
quant_lm_head=quant_lm_head,
enable_gguf_official_mixed=enable_gguf_official_mixed,
is_mllm=is_mllm,
fill_default_value=fill_default_value,
)
regex_config = extract_regex_config(
model=model,
scheme=resolved_scheme,
layer_config=layer_config,
scale_dtype=default_scale_dtype,
supported_types=supported_types,
inner_supported_types=inner_supported_types,
ignore_layers=ignore_layers,
fill_default_value=fill_default_value,
)
has_outside = has_quantized_layer_outside_blocks(resolved)
plan = ResolvedQuantizationConfig(
scheme=resolved_scheme,
formats=(),
layer_config=resolved,
regex_config=regex_config,
has_qlayer_outside_block=has_outside,
scale_dtype=default_scale_dtype,
quant_block_list=quant_block_list,
)
apply_plan_to_model(model, plan)
return (
{name: dict(config) for name, config in plan.layer_config.items()},
plan.has_qlayer_outside_block,
{name: dict(config) for name, config in plan.regex_config.items()},
)
def get_shared_keys(model):
"""
Retrieves shared keys from the model's state dictionary.
Args:
model (torch.nn.Module): The model to retrieve shared keys from.
Returns:
tuple: tuple of shared keys.
"""
from auto_round.special_model_handler import SPECIAL_SHARED_CACHE_KEYS
from auto_round.utils import SHARED_CACHE_KEYS
shared_keys = SHARED_CACHE_KEYS
shared_keys += SPECIAL_SHARED_CACHE_KEYS.get(model.__class__.__name__, ())
return shared_keys
def init_cache(positional_inputs, inputs):
"""
Initializes special model inputs by adding positional inputs if missing.
Args:
positional_inputs (list): List of positional inputs to add to inputs.
inputs (dict): Dictionary of model inputs.
Modifies:
inputs (dict): Adds "positional_inputs" key if not present.
"""
from auto_round.utils.model import to_device
if "positional_inputs" not in inputs: # for chatglm Series
inputs["positional_inputs"] = []
for idx, item in enumerate(positional_inputs):
inputs["positional_inputs"] = to_device(positional_inputs)
def reset_params(inputs):
"""
Resets specific input parameters to avoid saving the key-value cache during fine-tuning.
Args:
inputs (dict): Dictionary of model inputs.
Modifies:
inputs (dict): Sets "use_cache" to False if the key is present.
"""
if "use_cache" in inputs.keys(): # Not storing kv cache
inputs["use_cache"] = False
class IndexSampler:
"""A cyclic sampler that returns shuffled index batches.
This sampler maintains internal state so that each call to `next_batch()`
continues from where it left off. When the remaining number of samples is
less than `batch_size`, the sampler reshuffles all indices and starts from
the beginning, discarding the last incomplete batch.
Attributes:
nsamples (int): Total number of samples.
batch_size (int): Number of indices to return in each batch.
index (int): Current position in the index list.
indices (list[int]): Shuffled list of indices.
"""
def __init__(
self,
nsamples: int,
batch_size: int,
indices: "list[int] | None" = None,
rng: "random.Random | None" = None,
) -> None:
"""Initializes the sampler.
Args:
nsamples (int): Total number of samples (must be >= batch_size).
batch_size (int): Number of indices per batch.
indices (list[int], optional): Explicit index pool to shuffle
(defaults to ``range(nsamples)``). Used by shard-constrained
DDP sampling; ``nsamples`` must equal ``len(indices)``.
rng (random.Random, optional): Private shuffle generator (defaults
to the global ``random`` module). Data-parallel rank threads
pass their own so concurrent reshuffles stay reproducible.
Raises:
ValueError: If batch_size is not in the range (0, nsamples].
"""
if indices is None:
indices = list(range(nsamples))
elif len(indices) != nsamples:
raise ValueError(f"indices pool ({len(indices)}) does not match nsamples ({nsamples})")
if batch_size <= 0 or batch_size > nsamples:
raise ValueError("batch_size must be > 0 and <= nsamples")
self.nsamples: int = nsamples
self.batch_size: int = batch_size
self.index: int = 0
self._rng = rng if rng is not None else random
self.indices: list[int] = list(indices)
self._rng.shuffle(self.indices)
def next_batch(self) -> list[int]:
"""Returns the next batch of shuffled indices.
If the remaining indices are fewer than `batch_size`, the sampler
reshuffles the entire list and starts from the beginning.
Returns:
list[int]: A list of size `batch_size` containing sample indices.
"""
if self.index + self.batch_size > self.nsamples:
self._rng.shuffle(self.indices)
self.index = 0
batch = self.indices[self.index : self.index + self.batch_size]
self.index += self.batch_size
return batch
def _get_quantized_layer_names_outside_blocks(model, layer_config, supported_types, quant_block_list) -> list:
"""Gets the names of quantized layers outside blocks in the model.
Returns:
list: List of layer names outside blocks.
"""
if layer_config is None or len(layer_config) == 0:
return []
layer_names = []
all_layers_in_block = get_layer_names_in_block(model, supported_types, quant_block_list)
for key in layer_config.keys():
if key in all_layers_in_block:
continue
layer = get_module(model, key)
if layer is None:
raise LayerConfigResolutionError(f"could not find layer '{key}' in the model")
if type(layer) in supported_types and check_to_quantized(layer_config[key]):
layer_names.append(key)
return layer_names
def _get_diffusion_save_folder_name(format) -> str:
"""Generates the save folder name based on the provided format string.
If there are multiple formats to handle, the function creates a subfolder
named after the format string with special characters replaced. If there's
only one format, it returns the original output directory directly.
Args:
format_str (str): The format identifier (e.g., 'gguf:q2_k_s').
Returns:
str: The path to the folder where results should be saved.
"""
from auto_round.context.compress import CompressContext
from auto_round.context.model import ModelContext
compress_context = CompressContext.get_context()
model_context = ModelContext.get_context()
# Replace special characters to make the folder name filesystem-safe
sanitized_format = format.get_backend_name().replace(":", "-").replace("_", "-")
formats = compress_context.formats
# Use a subfolder only if there are multiple formats
if len(formats) > 1:
return os.path.join(compress_context.output_dir, sanitized_format, "transformer")
# if use is_immediate_saving, we need to save model in self.output_dir/transformer folder
return (
os.path.join(compress_context.output_dir, "transformer")
if compress_context.is_immediate_saving
else compress_context.output_dir
)
def _get_save_folder_name(format, *args, **kwargs) -> str:
"""Generates the save folder name based on the provided format string.
If there are multiple formats to handle, the function creates a subfolder
named after the format string with special characters replaced. If there's
only one format, it returns the original output directory directly.
Args:
format_str (str): The format identifier (e.g., 'gguf:q2_k_s').
Returns:
str: The path to the folder where results should be saved.
"""
from auto_round.context.compress import CompressContext
from auto_round.context.model import ModelContext
compress_context = CompressContext.get_context()
model_context = ModelContext.get_context()
if model_context.is_diffusion:
return _get_diffusion_save_folder_name(format)
# Replace special characters to make the folder name filesystem-safe
sanitized_format = format.get_backend_name().replace(":", "-").replace("_", "-")
# Use a subfolder only if there are multiple formats
if len(compress_context.formats) > 1:
return os.path.join(compress_context.output_dir, sanitized_format)
return compress_context.output_dir
PACK_WRAP = {"pre": 0.0, "fmt": 0.0}
def immediate_pack(name: str, layer_config: dict):
import time as _time
_t0 = _time.perf_counter()
from auto_round.context.compress import CompressContext
from auto_round.context.model import ModelContext
compress_context = CompressContext.get_context()
model_context = ModelContext.get_context()
if not compress_context.is_immediate_packing:
return
PACK_WRAP["pre"] += _time.perf_counter() - _t0
_t1 = _time.perf_counter()
compress_context.formats[0].immediate_pack(
name=name,
model=model_context.model,
device=device_manager.device,
output_dir=_get_save_folder_name(compress_context.formats[0]),
layer_config=layer_config,
tokenizer=model_context.tokenizer,
mllm=model_context.is_mllm,
processor=getattr(model_context, "processor", None),
image_processor=getattr(model_context, "image_processor", None),
quant_nontext_module=getattr(model_context, "quant_nontext_module", False),
)
PACK_WRAP["fmt"] += _time.perf_counter() - _t1