PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import gc
import importlib
import os
from collections.abc import Callable
from typing import Any
import torch
import transformers
from packaging import version
from transformers import AutoConfig
from auto_round import envs
from auto_round.compressors.utils import get_shared_keys
from auto_round.context.base import BaseContext
from auto_round.logger import logger
from auto_round.modeling.nemotron_h_omni import apply_nemotron_h_omni_post_load
from auto_round.modeling.unfused_moe import apply_model_monkey_patches
from auto_round.special_model_handler import _handle_special_model, update_module
from auto_round.utils import (
cast_model_dtype,
check_and_mark_quantized_module,
diffusion_load_model,
install_debug_layer_config_patch,
is_diffusion_model,
is_mllm_model,
is_moe_model,
is_moe_model_via_config,
llm_load_model,
mllm_load_model,
unsupported_meta_device,
)
from auto_round.utils.device import _force_trim_malloc
from auto_round.utils.device_manager import device_manager, get_ar_device
__all__ = ["ModelContext"]
_CUSTOM_MOE_REPLACEMENT_MODULES = {
"gpt_oss": "auto_round.modeling.fused_moe.gpt_oss",
}
class ModelContext(BaseContext):
_is_initialized = False
# model_related
_model_loaded = False
_init_model = False
hook_handles = []
def __init__(
self,
model: torch.nn.Module | str | None = None,
tokenizer: Any = None,
platform: str = "hf",
model_dtype: str | torch.dtype | None = None,
trust_remote_code: bool = True,
config: AutoConfig | None = None,
amp: bool = True,
need_calib: bool = True,
is_act_quantize: bool = False,
quant_nontext_module: bool = False,
**kwargs,
) -> None:
super().__init__()
self.quantized = False
self.is_mllm = False
self.is_diffusion = False
self.is_model_patched = False
self.is_moe_model = False
# Set by CalibCompressor._replace_forward; used by recover_forward to detect
# new-arch diffusion mode where positional wrapper must be stripped after caching.
self._has_true_orig_forward_set = False
assert model is not None, "model must be provided for ModelContext"
self.model = model
self.tokenizer = tokenizer
# MLLM / diffusion artifacts – always present so callers need no getattr guards.
# _load_model() will populate the ones that are relevant to the model type.
self.processor = None
self.image_processor = None
self.pipe = None
self.preloaded_diffusion_pipeline = False
# AWQ weight-clip thresholds kept for downstream block quantizers.
# Populated by AWQTransform when ``apply_clip`` is enabled; keyed by
# layer ``global_name`` -> per-group clip magnitude tensor. SignRound /
# SignRoundV2 use these to initialize their tunable weight range.
self.awq_clip_values: dict = {}
if envs.AR_USE_MODELSCOPE:
platform = "model_scope"
self.platform = platform
self.model_dtype = model_dtype
self.trust_remote_code = trust_remote_code
self.config = config
self.amp = amp
self.need_calib = need_calib
self.quant_nontext_module = quant_nontext_module
# Remember the original string model name
# so OffloadManager can later materialize blocks directly from this
# checkpoint.
self.disk_stream_model_dir = model if isinstance(model, str) else None
self._disk_stream_index = None
# Load model and run basic initialization eagerly so the model is ready
# by the time BaseCompressor.post_init() runs.
self._load_model()
if self._disk_stream_index is None and unsupported_meta_device(self.model):
raise RuntimeError(
"AutoRound does not support parameters on meta device. "
"Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."
)
if self._disk_stream_index is not None:
self._materialize_disk_stream_non_block_params()
check_and_mark_quantized_module(self.model)
self.model = self.model.eval()
self.shared_cache_keys = get_shared_keys(self.model)
self.is_moe_model = is_moe_model(self.model)
self._import_custom_moe_replacements(getattr(self.model, "config", None))
self._set_amp_dtype()
if is_act_quantize and self.amp_dtype == torch.float16:
logger.warning("force to use bf16 for quantization tuning when enabling activation quantization")
self.amp_dtype = torch.bfloat16
if self.model.dtype != torch.bfloat16:
self.model = cast_model_dtype(self.model, torch.bfloat16)
else:
logger.debug(f"using {self.model.dtype} for quantization tuning")
# Reclaim C heap fragmentation left by model/tokenizer loading so
# that the quantize loop starts from a tighter RSS baseline.
gc.collect()
_force_trim_malloc()
@property
def device(self) -> str:
"""The active (major) device, single-sourced from the DeviceManager."""
return device_manager.device
@device.setter
def device(self, value) -> None:
device_manager.device = value
def _load_model(self):
# Debug helper: when AR_DEBUG_LAYER_NUM is set, patch transformers config
# loading so every branch below (llm / mllm / diffusion / meta skeleton)
# loads only the first N decoder layers. No-op otherwise.
install_debug_layer_config_patch()
if is_diffusion_model(self.model):
self.is_diffusion = True
self.preloaded_diffusion_pipeline = not isinstance(self.model, str)
default_torch_dtype = "auto"
if self.amp and get_ar_device(self.device).supports_bf16():
default_torch_dtype = torch.bfloat16
self.pipe, self.model = diffusion_load_model(
self.model,
platform=self.platform,
device="cpu",
model_dtype=self.model_dtype,
default_torch_dtype=default_torch_dtype,
)
elif is_mllm_model(self.model, platform=self.platform):
self.is_mllm = True
if isinstance(self.model, str):
# Multimodal checkpoints used to
# bypass disk streaming entirely -- mllm_load_model fully
# materializes the checkpoint on CPU, infeasible for a 100B+
# VLM (e.g. Ornith). Build the same meta skeleton the text
# path uses (build_meta_model resolves the multimodal class
# from config.architectures) and load the processor stack
# cheaply; non-block params INCLUDING the whole vision tower
# (small, and needed real for calibration forwards and RTN)
# are materialized right after the meta-device guard in
# __init__, exactly like the text path. Falls back to the
# full mllm_load_model on any failure.
loaded_via_meta = False
if self._should_use_meta_skeleton(self._peek_config()):
try:
self.model, self.tokenizer, self._disk_stream_index = self._build_disk_stream_model(
self.disk_stream_model_dir
)
loaded_via_meta = True
from transformers import AutoProcessor
try:
self.processor = AutoProcessor.from_pretrained(
self.disk_stream_model_dir, trust_remote_code=self.trust_remote_code
)
self.image_processor = getattr(self.processor, "image_processor", None)
except Exception:
logger.warning(
"AutoProcessor failed for %s; keeping the meta skeleton (text-only calibration).",
self.disk_stream_model_dir,
exc_info=True,
)
self.processor = None
self.image_processor = None
except Exception:
logger.warning(
"Building a multimodal meta skeleton for %s failed; " "falling back to a full CPU load.",
self.disk_stream_model_dir,
exc_info=True,
)
self._disk_stream_index = None
if not loaded_via_meta:
self.model, self.processor, self.tokenizer, self.image_processor = mllm_load_model(
self.model, platform=self.platform, device="cpu", model_dtype=self.model_dtype
)
elif isinstance(self.model, str):
config = self.config
try:
if config is None:
config = AutoConfig.from_pretrained(self.model, trust_remote_code=self.trust_remote_code)
self._import_custom_moe_replacements(config)
except (OSError, ValueError) as e:
logger.debug(
"Failed to load config via AutoConfig.from_pretrained for %s: %s. "
"Proceeding without config-based checks.",
self.model,
e,
)
self.is_model_patched = apply_model_monkey_patches(
model_name=self.model, trust_remote_code=self.trust_remote_code
)
# Reclaim temporary HTTP/config objects from model type detection
# and AutoConfig loading before the large model allocation. This
# reduces heap fragmentation especially on HPU where habana internal
# allocations amplify fragmentation into persistent RSS growth.
gc.collect()
_force_trim_malloc()
if self._should_use_meta_skeleton(config):
try:
self.model, self.tokenizer, self._disk_stream_index = self._build_disk_stream_model(
self.disk_stream_model_dir
)
except Exception:
logger.warning(
"Building a meta skeleton for %s failed; falling back to a normal full CPU load.",
self.disk_stream_model_dir,
exc_info=True,
)
self._disk_stream_index = None
self.model, self.tokenizer = llm_load_model(
self.model,
platform=self.platform,
device="cpu",
model_dtype=self.model_dtype,
trust_remote_code=self.trust_remote_code,
)
else:
self.model, self.tokenizer = llm_load_model(
self.model,
platform=self.platform,
device="cpu", # always load cpu first
model_dtype=self.model_dtype,
trust_remote_code=self.trust_remote_code,
)
elif self.tokenizer is None and not self.is_diffusion and self.need_calib:
raise ValueError("A tokenizer must be set for non-str model input")
# Nemotron-H Omni wrappers: restore FP32 SSM/router tensors and the gated-norm
# group_size after loading. No-op for every other model.
apply_nemotron_h_omni_post_load(self.model)
self._model_loaded = True
# Clear tuning_device from any previous quantization passes.
# Previous AutoRound runs may have set tuning_device on modules to match
# their device_map (e.g., cpu). When re-quantizing with a different
# device_map, stale tuning_device causes device mismatches (WrapperLinear
# uses orig_layer.tuning_device instead of the current device_manager.device).
for m in self.model.modules():
if hasattr(m, "tuning_device"):
delattr(m, "tuning_device")
def _peek_config(self):
"""Best-effort config lookup before the model object exists."""
if self.config is not None:
return self.config
if not isinstance(self.model, str):
return getattr(self.model, "config", None)
try:
return AutoConfig.from_pretrained(self.model, trust_remote_code=self.trust_remote_code)
except Exception as exc:
logger.debug("could not peek the config of %s: %s", self.model, exc)
return None
def _should_use_meta_skeleton(self, config) -> bool:
"""Whether to build a meta skeleton instead of a full CPU load.
Deliberately narrow: this only targets ``transformers>=5`` models whose experts are
stored as a *fused* 3D ``nn.Parameter``. The fused parameter may be produced by a
Transformers checkpoint converter from per-expert 2D tensors, or already be stored
as 3D ``gate_up_proj``/``down_proj`` tensors in the checkpoint. AutoRound has to
split either layout back into per-expert ``nn.Linear`` modules to quantize it, and
splitting real tensors needs the fused tensor and its per-expert copies alive at
once, i.e. ~2x one experts module on top of an already fully resident model.
Building on meta makes that unfuse allocation-free and lets the tuning loop
materialize one block at a time from the checkpoint. Every other model (dense, or
a MoE that already ships as ``ModuleList`` of ``Linear``) has nothing to gain and
keeps the ordinary load path. ``AR_DISK_STREAM_MODEL=1`` forces this on for any
model; ``AR_DISABLE_META_LOAD=1`` turns the automatic choice off.
"""
if envs.AR_DISK_STREAM_MODEL:
return True
if envs.AR_DISABLE_META_LOAD:
return False
# The debug "load only N layers" path (AR_DEBUG_LAYER_NUM) relies on the
# normal from_pretrained load so the truncated config is honored; the meta
# skeleton streams the full checkpoint block-by-block and would ignore it.
if envs.AR_DEBUG_LAYER_NUM is not None:
return False
if not isinstance(self.model, str):
return False
if version.parse(transformers.__version__) < version.parse("5.0.0"):
return False
if config is None:
return False
from auto_round.modeling.fused_moe.moe_experts_interface import (
_config_model_types,
config_has_fused_moe_experts,
)
from auto_round.modeling.fused_moe.replace_modules import BUILTIN_MODULES
from auto_round.utils.disk_stream_util import checkpoint_has_native_fused_moe_experts
# Families with a dedicated replacement drive their own (already memory-aware)
# materialization; do not reroute them through the meta skeleton.
dedicated_model_type = next(
(model_type for model_type in _config_model_types(config) if model_type in BUILTIN_MODULES), None
)
if dedicated_model_type is not None:
logger.debug("meta-skeleton load skipped: %s has a dedicated MoE replacement", dedicated_model_type)
return False
checkpoint_dir = self._resolve_local_checkpoint_dir()
if checkpoint_dir is None:
logger.debug("meta-skeleton load skipped: %s is not addressable as a local directory", self.model)
return False
if not config_has_fused_moe_experts(config) and not checkpoint_has_native_fused_moe_experts(checkpoint_dir):
return False
self.disk_stream_model_dir = checkpoint_dir
logger.info(
"Fused-MoE checkpoint detected: building a meta skeleton and materializing weights per block "
"(set `AR_DISABLE_META_LOAD=1` to load the whole model on CPU instead)."
)
return True
def _resolve_local_checkpoint_dir(self) -> str | None:
"""Return a local directory holding the checkpoint shards, or ``None``.
``SafetensorsIndex`` reads shards by path, so a hub repo id has to be resolved to
a local snapshot first. Only the metadata + weights are fetched, and nothing is
read into memory here -- ``from_pretrained`` would have downloaded the same files.
"""
if os.path.isdir(self.model):
return self.model
if envs.AR_USE_MODELSCOPE:
return None
try:
from huggingface_hub import snapshot_download
return snapshot_download(
self.model,
allow_patterns=["*.safetensors", "*.safetensors.index.json", "*.json", "*.txt", "*.model"],
)
except Exception as exc:
logger.debug("could not resolve %s to a local snapshot: %s", self.model, exc)
return None
def _build_disk_stream_model(self, model_name: str):
"""Build an all-meta skeleton instead of
fully materializing the checkpoint on CPU RAM. Left fully meta here
(not even embeddings/lm_head materialized yet) so it passes the
existing ``unsupported_meta_device`` guard, which only allows models
that are either fully real or fully meta (with ``model.path`` set).
Non-block params are materialized for real right after that guard
runs, in ``__init__``, via ``_materialize_disk_stream_non_block_params``.
"""
from auto_round.utils.disk_stream_util import build_meta_model
from auto_round.utils.missing_tensors import keep_special_fp32_params
model, tokenizer, index = build_meta_model(model_name, trust_remote_code=self.trust_remote_code)
keep_special_fp32_params(model, model_name)
model.path = model_name
# Stash the index on the model object
# itself so downstream code that only has a reference to the model
# (not this ModelContext) -- e.g. AutoScheme's gen_layer_config, which
# runs after ModelContext has already turned the string into an object
# -- can still find it instead of re-scanning the checkpoint.
model._disk_stream_index = index
return model, tokenizer, index
def _materialize_disk_stream_non_block_params(self) -> None:
"""Materialize embeddings/lm_head/norm (everything outside the
quantizable decoder blocks) for real, leaving the (typically 100+GB
combined) decoder blocks on meta for later per-block materialize/free
by OffloadManager. No-op unless the model was built via
``_build_disk_stream_model``."""
if self._disk_stream_index is None:
return
from auto_round.utils import flatten_list, get_block_names
from auto_round.utils.disk_stream_util import materialize_non_block_params
block_prefixes = flatten_list(get_block_names(self.model, quant_vision=self.quant_nontext_module))
materialize_non_block_params(self.model, block_prefixes, self._disk_stream_index, device="cpu")
# Tied output embeddings (e.g. lm_head.weight
# tied to embed_tokens.weight) have no entry of their own in the checkpoint's
# safetensors index -- the on-disk format only stores the input embedding and
# relies on the model re-establishing the tie at load time. A normal
# from_pretrained() handles this automatically; our meta-skeleton +
# per-tensor materialize path does not, so without this the tied output
# module is silently left on meta (materialize_non_block_params only logs a
# warning and moves on). Re-tying now, after the real input embedding has
# been materialized above, makes the tied module real too by sharing the
# same (now real) Parameter object.
if hasattr(self.model, "tie_weights"):
self.model.tie_weights()
def _import_custom_moe_replacements(self, model_or_config) -> None:
model_type = getattr(model_or_config, "model_type", None)
module_name = _CUSTOM_MOE_REPLACEMENT_MODULES.get(model_type)
if module_name is None:
return
module = importlib.import_module(module_name)
from auto_round.modeling.fused_moe.replace_modules import BUILTIN_MODULES
BUILTIN_MODULES.setdefault(model_type, module)
logger.debug(f"Loaded custom MoE replacement module for {model_type}")
def _patch_custom_moe_modules(self) -> None:
model_type = getattr(getattr(self.model, "config", None), "model_type", None)
if model_type != "qwen3_vl_moe":
return
for module in self.model.modules():
if module.__class__.__name__ != "Qwen3VLMoeTextSparseMoeBlock":
continue
if hasattr(module, "top_k"):
continue
gate = getattr(module, "gate", None)
top_k = getattr(gate, "top_k", None)
if top_k is not None:
module.top_k = top_k
def _set_amp_dtype(self) -> None:
"""Sets the automatic mixed precision (AMP) data type for the model based on the device and configuration.
The device only exposes capability/preference primitives
(``supports_bf16`` / ``prefers_bf16``); this method composes them into
the final ``amp`` / ``amp_dtype`` decision.
"""
device = get_ar_device(self.device)
if not self.amp:
self.amp_dtype = torch.float32
else:
amp_dtype = torch.bfloat16
if self.model.dtype != torch.float32:
amp_dtype = self.model.dtype
# bf16-preferring backends (CPU/HPU/...) override the model dtype.
if device.prefers_bf16():
amp_dtype = torch.bfloat16
# Fall back to fp32 (and disable amp) when bf16 is unsupported.
if amp_dtype == torch.bfloat16 and not device.supports_bf16():
self.amp = False
amp_dtype = torch.float32
logger.warning(
f"amp is set to FALSE as the current {self.device} device does not support the 'bf16' data type."
)
self.amp_dtype = amp_dtype
if self.model.dtype != self.amp_dtype:
self.model = cast_model_dtype(self.model, self.amp_dtype)
def apply_patches(self, formats):
"""Apply format-specific model structure patches.
Must be called after formats are resolved (list[OutputFormat]) and before
BaseQuantizer.post_init() so that configure_layer_config() operates on the
final model structure (post update_module). Eliminates the need for a
subsequent refresh_quantizer_for_initialized_model() call.
"""
# It is best to modify the model structure in the quantize function and check the format,
# because it may cause the gguf format to not be exported normally.
self._patch_custom_moe_modules()
self.model = update_module(
self.model, formats=formats, trust_remote_code=self.trust_remote_code, cleanup_original=False
)
self.model = _handle_special_model(self.model)
# Temporary names must be assigned after handle_moe_model;
# placing them earlier would cause them to be removed when the module is replaced.
for n, m in self.model.named_modules():
m.global_name = n
if self.amp and self.model.dtype != self.amp_dtype:
self.model = cast_model_dtype(self.model, self.amp_dtype)
self._init_model = True
self._is_initialized = True
def replace_forward(self, register_hook):
"""Replaces the forward function.
register_hook(layer_name, module, hook_handles)
"""
assert self._init_model, "should load and initialize model first"
hook_handles = []
for n, m in self.model.named_modules():
register_hook(n, m, hook_handles)
self.hook_handles = hook_handles
def recover_forward(self, restore_positional_wrapper=None):
"""Recovers the forward function.
Args:
restore_positional_wrapper: If True, restores forward to the wrapped version
(needed for LLM calibration where positional wrapper is used during quantization).
If False, restores to the true original forward (needed for diffusion).
If None (default), auto-detects: uses False for diffusion models.
"""
assert self._init_model, "should load and initialize model first"
# Auto-detect for diffusion: when _true_orig_forward is present (set by
# CalibCompressor._replace_forward), we are in new-arch diffusion mode where
# the positional wrapper must be fully removed after caching.
if restore_positional_wrapper is None:
restore_positional_wrapper = not getattr(self, "_has_true_orig_forward_set", False)
if not restore_positional_wrapper:
logger.debug("recover_forward: auto-detected diffusion mode, stripping positional wrapper")
for n, m in self.model.named_modules():
if hasattr(m, "orig_forward"):
true_orig = getattr(m, "_true_orig_forward", m.orig_forward)
if restore_positional_wrapper:
# Restore orig_forward so that any wrapper (e.g. from
# wrap_block_forward_positional_to_kwargs) can still access it.
# The wrapper holds a closure reference to orig_forward.
m.forward = getattr(m, "_wrapped_forward_before_replace", m.orig_forward)
m.orig_forward = true_orig
else:
# Full recovery: restore the true original forward. Used for diffusion
# where the positional wrapper must be fully removed after caching.
m.forward = true_orig
# Keep _true_orig_forward so the wrapped forward's base_hook can
# still call it during quantization tuning.
m._true_orig_forward = true_orig
delattr(m, "orig_forward")
if hasattr(m, "_wrapped_forward_before_replace"):
delattr(m, "_wrapped_forward_before_replace")
for hook_handle in self.hook_handles:
hook_handle.remove()
self.hook_handles = []