PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
WNA16 does not apply optimized RTN for moe layers by default (#1245)
Wenhua Cheng9588bf902923
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex a2af9e011..aec2bbee1 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -93,6 +93,7 @@ from auto_round.utils import (is_fp8_linear,is_fp8_model,is_hpex_available,+ is_moe_model,llm_load_model,memory_monitor,mv_module_from_gpu,@@ -390,6 +391,11 @@ class BaseCompressor(object):)# Automatically adjust the disable_opt_rtn option if the user does not explicitly set it.+ # To avoid None issue, we keep a copy though it's a little ugly+ self.orig_disable_opt_rtn = disable_opt_rtn+ if self.iters != 0 and self.orig_disable_opt_rtn is not None:+ logger.warning("`disable_opt_rtn` only works when `iters` is set to 0, ignore it now.")+ disable_opt_rtn = Trueif (self.bits >= 8and self.act_bits >= 16@@ -397,12 +403,16 @@ class BaseCompressor(object):and self.data_type == "int"and disable_opt_rtn is None):- logger.warning("For INT8 RTN quantization, set `--disable_opt_rtn` as default.")+ logger.warning("`disable_opt_rtn` is turned on for W8A16 quantization to improve efficiency.")disable_opt_rtn = True- if disable_opt_rtn is None:- if self.iters == 0:- logger.info("For the most RTN cases, set `--disable_opt_rtn` to False as default.")- disable_otp_rtn = False+ if disable_opt_rtn is None and self.iters == 0:+ logger.info(+ "`enable_opt_rtn` is turned on, set `--disable_opt_rtn` for higher speed at the cost of accuracy."+ )+ disable_opt_rtn = False++ # Important Note! This is not very robust, do NOT rely on it to do high risky thing+ self.is_moe_model = is_moe_model(self.model)self.minmax_lr = minmax_lr or self.lrself.enable_alg_ext = enable_alg_ext@@ -1105,6 +1115,20 @@ class BaseCompressor(object):m.zp = Noneelse:try:+ disable_opt_rtn = self.disable_opt_rtn+ if (+ not disable_opt_rtn+ and self.orig_disable_opt_rtn is None+ and self.is_moe_model+ and "expert" in m.tmp_name+ and "shared_expert" not in m.tmp_name+ and self.super_bits is None # GGUF still uses the optimized RTN for MoE layers+ ):+ disable_opt_rtn = True+ logger.warning_once(+ "MoE layer detected: optimized RTN is disabled for efficiency. "+ "Use `--enable_opt_rtn` to force-enable it for MoE layers."+ )m = m.to(tuning_device)m = WrapperLinear(m,@@ -1113,7 +1137,7 @@ class BaseCompressor(object):enable_norm_bias_tuning=False,enable_round_tuning=False,enable_torch_compile=self.enable_torch_compile,- disable_opt_rtn=self.disable_opt_rtn,+ disable_opt_rtn=disable_opt_rtn,)m = m.unwrapper({})except torch.OutOfMemoryError:diff --git a/auto_round/compressors/config.py b/auto_round/compressors/config.pyindex f7e0d92f2..dd028b6cd 100644--- a/auto_round/compressors/config.py+++ b/auto_round/compressors/config.py@@ -247,7 +247,7 @@ class BaseExtraConfig:@dataclassclass TuningExtraConfig(BaseExtraConfig):amp: bool = True- disable_opt_rtn: bool | None = True+ disable_opt_rtn: bool | None = Noneenable_alg_ext: bool = Falseenable_minmax_tuning: bool = Trueenable_norm_bias_tuning: bool = Falsediff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex e19fbca68..8fbcc09fd 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -832,10 +832,10 @@ def get_moe_memory_ratio(block: torch.nn.Module) -> float:- Mixtral (2/8 experts): returns 0.25- Qwen2MoE (4/60 experts): returns ~0.067"""- from auto_round.utils.model import is_moe+ from auto_round.utils.model import is_moe_layerfor name, module in block.named_modules():- if not is_moe(module):+ if not is_moe_layer(module):continueconfig = getattr(block, "config", None)@@ -898,7 +898,7 @@ def estimate_tuning_block_mem(- additional_memory (float): Additional memory overhead (in GB) for operations like attention."""# Calculate all block parameters memory and build layer-wise memory dict- from auto_round.utils.model import get_layer_features, is_moe+ from auto_round.utils.model import get_layer_features, is_moe_layerlayer_memory_dict = {}diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 8b85bc26e..f4c6e7b75 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -598,9 +598,9 @@ def is_diffusion_model(model_or_path: Union[str, object]) -> bool:return False-def is_moe(module: torch.nn.Module) -> bool:+def is_moe_layer(module: torch.nn.Module) -> bool:"""Returns whether the module is an MOE layer."""- return any(+ return "moe" in type(module).__name__.lower() or any(key in type(module).__name__.lower()for key in ["MixtralSparseMoeBlock".lower(),@@ -1191,6 +1191,17 @@ def mv_module_from_gpu(module):return module.to("cpu")+def is_moe_model(model: torch.nn.Module) -> bool:+ if hasattr(model, "config"):+ for key in model.config.to_dict().keys():+ if "moe" in key or "expert" in key:+ return True+ for n, m in model.named_modules():+ if "expert" in n:+ return True+ return False++def to_dtype(input, dtype=torch.float32):"""Moves input data to the specified data type.@@ -1281,7 +1292,7 @@ def set_amax_for_all_moe_layers(model: torch.nn.Module, layer_name=None, attr_namodel = get_module(model, moe_name)# Handle input quantizers of experts that are not calibratedfor name, sub_module in model.named_modules():- if not (is_moe(sub_module) and hasattr(sub_module, "experts")):+ if not (is_moe_layer(sub_module) and hasattr(sub_module, "experts")):continueexpert_linear_names = get_expert_linear_names(sub_module)# Get input projection names for FP8 dispatch unification