PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from abc import ABC, abstractmethod
from contextlib import contextmanager
from dataclasses import dataclass
import torch
from tqdm import tqdm
from auto_round.modeling.fused_moe.utils import is_linearized_layout
from auto_round.utils import (
LazyImport,
dump_mem_usage,
dump_memory_usage_ctx,
global_state,
is_transformers_version_greater_or_equal_5,
logger,
)
BUILTIN_MODULES = {
# Llama4 has no use_experts_implementation, needs custom replacement to handle fused MoE blocks.
"llama4": LazyImport("auto_round.modeling.fused_moe.llama4"),
# DeepseekV2Attention enables q_scale calibration for deepseek v2 on Gaudi (#1299)
"deepseek_v2": LazyImport("auto_round.modeling.fused_moe.deepseek_v2"),
# Qwen3.5 MoE uses block-local materialization to avoid eagerly unfusing the full model.
"qwen3_5_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),
"qwen3_5_moe_text": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),
# Step 3.5 MoE: splits fused MoELinear into per-expert nn.Linear
"step3p5": LazyImport("auto_round.modeling.fused_moe.step3_5_moe"),
# Qwen3-Omni MoE: thinker (no shared expert)
"qwen3_omni_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_omni"),
# Kimi remote MoE: routed experts run under @torch.no_grad() in eval mode
"kimi_vl": LazyImport("auto_round.modeling.fused_moe.kimi"),
"kimi_linear": LazyImport("auto_round.modeling.fused_moe.kimi"),
"kimi_k3": LazyImport("auto_round.modeling.fused_moe.kimi"),
}
_FUSED_MOE_PROJECTION_NAMES = frozenset({"gate_up_proj", "gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"})
def _is_fused_moe_parameter(name: str, parameter: torch.Tensor) -> bool:
return parameter.ndim == 3 and any(part in _FUSED_MOE_PROJECTION_NAMES for part in name.split("."))
# Module name prefixes to exclude from MoE unfusing, keyed by model_type.
# Modules under these prefixes stay in their original fused 3D format during
# quantization. ShardWriter expands them to per-expert 2D tensors at save time.
MOE_SKIP_PREFIXES: dict[str, list[str]] = {
"qwen3_omni_moe": ["talker."],
}
# transformers >= 5.0.0 supports the general linear_loop experts interface.
if not is_transformers_version_greater_or_equal_5():
# transformers == 5.2.0: IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)
BUILTIN_MODULES["qwen3_vl_moe"] = LazyImport("auto_round.modeling.fused_moe.qwen3_vl_moe")
# transformers == 5.2.0: IndexError: index 4 is out of bounds for dimension 0 with size 4
BUILTIN_MODULES["gpt_oss"] = LazyImport("auto_round.modeling.fused_moe.gpt_oss")
def _handle_moe_modules(model: torch.nn.Module) -> list[str]:
"""Handle fused MOE modules using transformers' linear_loop backend.
Modules under prefixes listed in MOE_SKIP_PREFIXES for the model's
model_type are left in their original fused format.
Args:
model: The model to process
Returns:
List of module names that were processed
"""
from auto_round.modeling.fused_moe.moe_experts_interface import (
is_linear_loop_available,
prepare_model_for_moe_quantization,
)
if not is_linear_loop_available():
logger.warning(
"transformers' linear_loop experts interface not available (requires transformers 5.0+). "
"MOE modules with @use_experts_implementation decorator will fall back to custom replacements "
"if registered."
)
return []
model_type = getattr(getattr(model, "config", None), "model_type", None)
skip_prefixes = MOE_SKIP_PREFIXES.get(model_type, [])
# Use transformers' experts interface
unfused = prepare_model_for_moe_quantization(model, skip_prefixes=skip_prefixes)
if unfused:
logger.info(f"Prepared {len(unfused)} MOE modules for quantization")
return unfused
def _import_required_replacements(model: torch.nn.Module) -> None:
"""Scan model and trigger lazy imports for registered replacement modules."""
if not is_custom_model(model):
return
model_type = model.config.model_type
_ = BUILTIN_MODULES[model_type].__name__ # Trigger lazy import
logger.debug(f"Loaded replacement module for {model_type}")
def is_custom_model(model: torch.nn.Module) -> bool:
"""Check if the model has a custom replacement registered via BUILTIN_MODULES.
Returns True if the model's model_type matches a key in BUILTIN_MODULES.
"""
if hasattr(model, "config") and hasattr(model.config, "model_type"):
return model.config.model_type in BUILTIN_MODULES
return False
def _find_first_moe_block(model: torch.nn.Module) -> tuple[str, torch.nn.Module] | tuple[None, None]:
"""Return ``(name, module)`` of the first experts-like module, or ``(None, None)``."""
for name, module in model.named_modules():
if name.endswith((".experts", ".moe")):
return name, module
return None, None
def _describe_moe_block(model: torch.nn.Module, name: str) -> str | None:
"""Render the experts module at ``name`` for the before/after comparison."""
try:
module = model.get_submodule(name)
except AttributeError:
return None
return f"({module.__class__.__name__}):\n{module}"
@contextmanager
def log_moe_block_transition(model: torch.nn.Module, label: str = "replacement"):
"""Log the first experts module before and after a structural change.
The structural unfuse happens in one of two places depending on how the model was
built: on the meta skeleton (``build_meta_model``) or during ``apply_replacements``.
Both wrap themselves in this so the transition is reported wherever it actually
occurs, and only when something really changed -- printing an identical before/after
pair (which is what the other path sees, since it then has nothing left to do) is
just noise.
"""
name, _ = _find_first_moe_block(model)
before = _describe_moe_block(model, name) if name is not None else None
try:
yield
finally:
if name is not None:
after = _describe_moe_block(model, name)
if after is not None and after != before:
logger.info(f"Experts (before {label}) [{name}] {before}")
logger.info(f"Experts (after {label}) [{name}] {after}")
else:
logger.debug(f"Experts [{name}] left unchanged by {label} {before}")
@dump_mem_usage("Materializing model", log_level="debug")
def materialize_model_(model: torch.nn.Module) -> None:
def _materialize_module(module: torch.nn.Module) -> None:
if isinstance(module, ReplacementModuleBase):
module.materialize_weights()
model.apply(_materialize_module)
# Any tensor still on meta after per-block materialization means the meta-skeleton
# loader could not resolve it from the checkpoint (e.g. a checkpoint-conversion
# layout AutoRound does not reverse yet). Leaving it on meta only defers the failure
# to a cryptic "Cannot copy out of meta tensor" crash on the next `.to(device)`, so
# fail fast with an actionable message.
still_meta = [name for name, param in model.named_parameters() if param.device.type == "meta"]
still_meta += [name for name, buffer in model.named_buffers() if buffer.device.type == "meta"]
if still_meta:
from auto_round import envs
preview = ", ".join(still_meta[:8]) + (f" (+{len(still_meta) - 8} more)" if len(still_meta) > 8 else "")
# The auto meta-skeleton is only enabled for fused-MoE checkpoints; an explicit
# AR_DISK_STREAM_MODEL=1 forces it for anything else. Point at whichever toggle
# actually turned it on so the hint stays correct (AR_DISABLE_META_LOAD is
# the MoE-only auto path).
hint = "unset AR_DISK_STREAM_MODEL" if envs.AR_DISK_STREAM_MODEL else "set AR_DISABLE_META_LOAD=1"
raise RuntimeError(
f"Failed to materialize {len(still_meta)} checkpoint tensor(s), still on meta: {preview}. "
f"To load the whole model on CPU instead, {hint} (uses more RAM)."
)
logger.debug("All parameters and buffers have been materialized from meta device.")
release_original_module_(model)
def release_original_module_(model: torch.nn.Module) -> None:
def _clear_source_module(module: torch.nn.Module) -> None:
if isinstance(module, ReplacementModuleBase):
module.release_original_module()
model.apply(_clear_source_module)
def safe_to_cpu_(model: torch.nn.Module) -> None:
"""Move a model to CPU unless it still contains unmaterialized meta tensors."""
has_meta_tensor = any(param.device.type == "meta" for param in model.parameters()) or any(
buffer.device.type == "meta" for buffer in model.buffers()
)
if not has_meta_tensor:
model.to("cpu")
class ReplacementModuleBase(ABC, torch.nn.Module):
"""
Abstract base class for module replacement during calibration phase.
Replacement modules replace original modules to ensure all components
receive data for proper quantization statistics.
Subclasses must:
1. Implement `original_module_class()` to return the target module class name
2. Implement `__init__()` with signature:
(self, original, config)
"""
# Registry: module class name -> replacement module class
_replacement_registry: dict[str, type["ReplacementModuleBase"]] = {}
supports_gguf_fused_moe: bool = False
# Set False for replacements that only patch behavior and also apply to linearized experts.
skip_linearized_experts: bool = True
def __init_subclass__(cls, **kwargs):
"""Automatically register subclasses in the replacement registry."""
super().__init_subclass__(**kwargs)
# Only register if it's a concrete implementation (not ABC)
if not getattr(cls, "__abstractmethods__", None):
if cls.original_module_class() is None:
raise TypeError(
f"{cls.__name__} must implement 'original_module_class()' class method "
"to return the name of the module class it replaces"
)
if cls.original_module_class() in cls._replacement_registry:
existing = cls._replacement_registry[cls.original_module_class()]
raise ValueError(
f"Module '{cls.original_module_class()}' already registered to "
f"{existing.__name__}. Cannot register {cls.__name__}."
)
cls._replacement_registry[cls.original_module_class()] = cls
logger.trace(f"Registered {cls.__name__} for replacing {cls.original_module_class()}")
def __init__(self, original: torch.nn.Module):
super().__init__()
if any(_is_fused_moe_parameter(name, parameter) for name, parameter in original.named_parameters()):
self._auto_round_replaced_fused_moe = True
_global_tracker.register_replacement(
name=str(id(self)),
original=original,
replacement=self,
)
self._materialized = False
@classmethod
def get_replacement_class(cls, module_class_name: str) -> type["ReplacementModuleBase"]:
"""Get replacement class for a given module class name."""
return cls._replacement_registry.get(module_class_name)
@classmethod
def is_registered(cls, module_class_name: str) -> bool:
"""Check if a module class has a replacement implementation."""
return module_class_name in cls._replacement_registry
@classmethod
def is_to_be_replaced(
cls,
original: torch.nn.Module,
) -> bool:
"""Determine if the given module should be replaced.
Users can extend this method to add custom logic for replacement.
"""
return cls.is_registered(original.__class__.__name__)
@classmethod
def get_registered_modules(cls) -> list:
"""Get list of all registered module class names."""
return list(cls._replacement_registry.keys())
@classmethod
@abstractmethod
def original_module_class(cls) -> str:
"""Return the class name of the module this replaces."""
@classmethod
@abstractmethod
def from_original(
cls,
original: torch.nn.Module,
config,
) -> "ReplacementModuleBase":
"""Create replacement module from original module."""
def materialize_weights(self):
"""Materialize weights if needed."""
if not self._materialized:
self._materialize_weights()
self.post_process_materialization()
def _materialize_weights(self) -> None:
"""Materialize weights from the original module.
Subclasses should override this method to implement
weight materialization logic.
"""
def release_original_module(self) -> None:
"""Release reference to the original module to free memory."""
# Release from global tracker
_global_tracker.release_original(self)
def _get_original_module(self) -> torch.nn.Module:
"""Get the original module associated with this replacement."""
return _global_tracker.get_original(self)
def post_process_materialization(self) -> None:
"""Mark the replacement module as materialized."""
self._materialized = True
self.release_original_module()
# Note: adapted from llm-compressor
# https://github.com/vllm-project/llm-compressor/blob/main/src/llmcompressor/modeling/moe_context.py
def apply_replacements(
model: torch.nn.Module,
auto_detect_moe: bool = True,
gguf_export: bool = False,
) -> torch.nn.Module:
"""
Function to apply module replacements to a model.
This scans all modules in the model and replaces any registered modules with their
replacement equivalents. Non-permanent modules are tracked for later restoration.
The model is modified in-place, so the same model object should be used.
Args:
model: The model to apply module replacement to (modified in-place).
auto_detect_moe: If True, automatically detect and handle fused MOE modules
(transformers 5.0+ pattern). Default is True.
gguf_export: If True, skip custom fused-MoE replacements without an explicit
reversible GGUF fusion specification.
Returns:
The model with modules replaced.
"""
_import_required_replacements(model)
with log_moe_block_transition(model, "replacement"):
# Custom replacements first
if is_custom_model(model):
_apply_custom_replacements(model, gguf_export=gguf_export)
if auto_detect_moe and is_transformers_version_greater_or_equal_5():
_handle_moe_modules(model)
return model
@dump_mem_usage("applying custom replacements")
def _apply_custom_replacements(model: torch.nn.Module, gguf_export: bool = False) -> list:
"""Scan model and replace registered modules with custom implementations.
Args:
model: The model to scan and apply replacements to (modified in-place).
Returns:
List of (name, replacement_class) tuples for replaced modules.
"""
replaced = []
# Step 1: Collect all modules that need replacement
logger.debug("Scanning for modules to replace")
modules_to_replace = []
skipped_modules = []
for name, module in model.named_modules():
# skip replaced modules
if isinstance(module, ReplacementModuleBase):
continue
class_name = module.__class__.__name__
if ReplacementModuleBase.is_registered(class_name):
replacement_cls = ReplacementModuleBase.get_replacement_class(class_name)
if (
gguf_export
and any(_is_fused_moe_parameter(name, parameter) for name, parameter in module.named_parameters())
and not replacement_cls.supports_gguf_fused_moe
):
skipped_modules.append((name, class_name, "skipped because GGUF fusion spec is unavailable"))
continue
# If llm-compressor already linearized experts, skip custom replacement.
if (
replacement_cls.skip_linearized_experts
and hasattr(module, "experts")
and is_linearized_layout(module.experts)
):
skipped_modules.append((name, class_name, "skipped because linearize_moe"))
continue
if replacement_cls.is_to_be_replaced(module):
modules_to_replace.append((name, module, class_name))
else:
skipped_modules.append((name, class_name, "skipped because replacement criteria not met"))
if skipped_modules:
for name, class_name, reason in skipped_modules:
logger.info(f"Skipped replacement for {name} ({class_name}): {reason}")
# Step 2: Replace modules
if modules_to_replace:
logger.info(f"Found {len(modules_to_replace)} modules to replace")
for name, module, class_name in tqdm(modules_to_replace, desc="Replacing modules"):
module = model.get_submodule(name)
# The module might have been replaced earlier in the loop (parent-first replacement).
# Skip if the class has changed or it no longer matches replacement criteria.
if module.__class__.__name__ != class_name:
logger.debug(
f"Skipping replacement for {name}: class changed from {class_name} to {module.__class__.__name__}"
)
continue
with dump_memory_usage_ctx(f"Replacing module {name}", log_level="debug"):
replacement_cls = ReplacementModuleBase.get_replacement_class(class_name)
if not replacement_cls.is_to_be_replaced(module):
logger.debug(f"Skipping replacement for {name}: no longer matches replacement criteria")
continue
replacement = replacement_cls.from_original(
module,
model.config,
)
model.set_submodule(name, replacement)
replaced.append((name, replacement_cls))
else:
logger.debug("No modules found for replacement")
# Log what was replaced
if replaced:
global_state.replaced_module_count = len(replaced)
logger.info(f"Replaced {len(replaced)} modules")
return replaced
@dataclass
class ReplacedModuleInfo:
original_module: torch.nn.Module
replacement_module: ReplacementModuleBase
class ModuleReplacementTracker:
"""Tracker to maintain mapping between replacement modules and their original modules.
This is a singleton class - only one instance can exist.
"""
_instance = None
_initialized = False
def __new__(cls):
if cls._instance is None:
cls._instance = super().__new__(cls)
return cls._instance
def __init__(self):
# Only initialize once
if ModuleReplacementTracker._initialized:
return
# Map from replacement module id to original module
self._replacement_to_original: dict[int, torch.nn.Module] = {}
# Map from module name to ReplacedModuleInfo
self._name_to_info: dict[str, ReplacedModuleInfo] = {}
ModuleReplacementTracker._initialized = True
@classmethod
def get_instance(cls) -> "ModuleReplacementTracker":
"""Get the singleton instance of the tracker."""
if cls._instance is None:
cls._instance = cls()
return cls._instance
def register_replacement(self, name: str, original: torch.nn.Module, replacement: ReplacementModuleBase) -> None:
"""Register a module replacement."""
self._replacement_to_original[id(replacement)] = original
self._name_to_info[name] = ReplacedModuleInfo(original_module=original, replacement_module=replacement)
logger.trace(f"Registered replacement for module: {name}")
def get_original(self, replacement: ReplacementModuleBase) -> torch.nn.Module:
"""Get the original module for a given replacement module."""
return self._replacement_to_original.get(id(replacement))
def get_info_by_name(self, name: str) -> ReplacedModuleInfo:
"""Get replacement info by module name."""
return self._name_to_info.get(name)
def release_original(self, replacement: ReplacementModuleBase) -> None:
"""Release the original module associated with a replacement module."""
replacement_id = id(replacement)
if replacement_id in self._replacement_to_original:
original = self._replacement_to_original[replacement_id]
# Delete the original module to free memory
del original
del self._replacement_to_original[replacement_id]
logger.trace(f"Released original module for replacement {replacement_id}")
def release_all_originals(self) -> None:
"""Release all tracked original modules."""
count = len(self._replacement_to_original)
if count > 0:
self._replacement_to_original.clear()
logger.debug(f"Released {count} original modules from tracker")
def clear(self) -> None:
"""Clear all tracked information."""
self._replacement_to_original.clear()
self._name_to_info.clear()
logger.debug("Cleared module replacement tracker")
_global_tracker = ModuleReplacementTracker()