PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support qwen3_5 moe (#1476)
Wenhua Cheng1b37c316987d
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex de58ec36a..8bdae36b2 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -27,9 +27,10 @@ import accelerateimport torchfrom accelerate.big_modeling import dispatch_model, infer_auto_device_mapfrom accelerate.utils import get_balanced_memory, get_max_memory+from packaging import versionfrom torch import autocastfrom tqdm import tqdm-from transformers import set_seed+from transformers import AutoConfig, set_seedfrom auto_round import envsfrom auto_round.auto_scheme.gen_auto_scheme import AutoScheme@@ -92,6 +93,7 @@ from auto_round.utils import (is_debug_mode,is_hpex_available,is_moe_model,+ is_moe_model_via_config,is_quantized_input_module,llm_load_model,memory_monitor,@@ -272,23 +274,35 @@ class BaseCompressor(object):self.quantized = Falseself.is_model_patched = Falseif isinstance(model, str):+ config: Optional[AutoConfig] = Nonetry:- # config = AutoConfig.from_pretrained(model)- self.is_model_patched = apply_model_monkey_patches(model_name=model)-- # TODO excluded ori_params_moe- # if (- # not self.is_model_patched- # and is_moe_model_via_config(config)- # and version.parse(transformers.__version__) >= version.parse("5.0.0")- # ):- # logger.warning(- # "This moe model is not optimized by AutoRound yet which may cause large ram usage, "- # "please submit an issue to https://github.com/intel/auto-round/issues"- # )-- except:- pass+ config = AutoConfig.from_pretrained(model, trust_remote_code=self.trust_remote_code)+ except (OSError, EnvironmentError) as e:+ logger.debug(+ "Failed to load config via AutoConfig.from_pretrained for %s: %s. "+ "Proceeding without config-based checks.",+ model,+ e,+ )++ self.is_model_patched = apply_model_monkey_patches(model_name=model)+ import transformers++ if (+ not self.is_model_patched+ and config is not None+ and is_moe_model_via_config(config)+ and version.parse(transformers.__version__) >= version.parse("5.0.0")+ ):+ from auto_round.modeling.fused_moe.replace_modules import BUILTIN_MODULES++ model_type = getattr(config, "model_type", None)+ if model_type is not None and model_type not in BUILTIN_MODULES:+ logger.warning(+ "This MoE model has not been optimized by AutoRound yet, which may result in high RAM usage, "+ "Please consider submitting an issue to https://github.com/intel/auto-round/issues"+ )+model, tokenizer = llm_load_model(model,platform=platform,@@ -1341,7 +1355,9 @@ class BaseCompressor(object):for block_name in block_names:pbar.set_description(f"Quantizing {block_name}")block = get_module(self.model, block_name)+materialize_model_(block)+for name, m in block.named_modules():if hasattr(m, "global_name") and m.global_name in all_to_quantized_module_names:self._quantize_layer_via_rtn(m.global_name, to_cpu=self.low_gpu_mem_usage)@@ -2159,79 +2175,82 @@ class BaseCompressor(object):all_inputs = self.cache_inter_data(block_names, nsamples, layer_names=[], last_cache_name=last_cache_name)else:try:- if not self.model.device.type == "meta":- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- self.model = dispatch_model(self.model, device_map=self.model.hf_device_map)- else:- # Change this if new device is supported- if str(self.model.device) == "cpu" and (not self.device.startswith("hpu")):- # type(self.model._no_split_modules) changes from list to set when transformers > 5.0- no_split_modules = list(getattr(self.model, "_no_split_modules", []))- devices = parse_available_devices(self.device_map)-- max_memory = get_max_memory()- new_max_memory = {}- if "cpu" not in devices:- devices.append("cpu")- for device in devices:- if ":" in device:- device = int(device.split(":")[-1])- elif device == "cpu":- device = "cpu"- elif isinstance(device, str):- device = 0- else:- raise ValueError(f"Unsupported device {device} in device_map: {self.device_map}")- if device not in max_memory:- # Skip devices that are not reported by accelerate's max_memory.- # This is expected when a device is unavailable or cannot provide memory info.- continue- # Use 90% of the reported max memory to leave headroom for activations,- # temporary tensors, other processes, and allocator fragmentation, reducing- # the chance of runtime OOM while still utilizing most available memory.- new_max_memory[device] = max_memory[device] * 0.9-- # If non-CPU devices were requested but none survived, fall back to CPU caching- # via the OOM handler below, avoiding unnecessary dispatch overhead.- requested_non_cpu = any((d != "cpu") for d in devices)- has_non_cpu_memory = any((k != "cpu") for k in new_max_memory)- if requested_non_cpu and not has_non_cpu_memory:- raise torch.OutOfMemoryError(- "No non-CPU device available in accelerate's reported memory. "- "Falling back to CPU caching."- )+ if any(p.device.type == "meta" for p in self.model.parameters()):+ materialize_model_(self.model)- new_max_memory = get_balanced_memory(- self.model,- max_memory=new_max_memory,- no_split_module_classes=no_split_modules,+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ self.model = dispatch_model(self.model, device_map=self.model.hf_device_map)+ else:+ # Change this if new device is supported+ if str(self.model.device) == "cpu" and (not self.device.startswith("hpu")):+ # type(self.model._no_split_modules) changes from list to set when transformers > 5.0+ no_split_modules = list(getattr(self.model, "_no_split_modules", []))+ devices = parse_available_devices(self.device_map)++ max_memory = get_max_memory()+ new_max_memory = {}+ if "cpu" not in devices:+ devices.append("cpu")+ for device in devices:+ if ":" in device:+ device = int(device.split(":")[-1])+ elif device == "cpu":+ device = "cpu"+ elif isinstance(device, str):+ device = 0+ else:+ raise ValueError(f"Unsupported device {device} in device_map: {self.device_map}")+ if device not in max_memory:+ # Skip devices that are not reported by accelerate's max_memory.+ # This is expected when a device is unavailable or cannot provide memory info.+ continue+ # Use 90% of the reported max memory to leave headroom for activations,+ # temporary tensors, other processes, and allocator fragmentation, reducing+ # the chance of runtime OOM while still utilizing most available memory.+ new_max_memory[device] = max_memory[device] * 0.9++ # If non-CPU devices were requested but none survived, fall back to CPU caching+ # via the OOM handler below, avoiding unnecessary dispatch overhead.+ requested_non_cpu = any((d != "cpu") for d in devices)+ has_non_cpu_memory = any((k != "cpu") for k in new_max_memory)+ if requested_non_cpu and not has_non_cpu_memory:+ raise torch.OutOfMemoryError(+ "No non-CPU device available in accelerate's reported memory. "+ "Falling back to CPU caching.")- self.model.tie_weights()- device_map = infer_auto_device_map(- self.model, max_memory=new_max_memory, no_split_module_classes=no_split_modules++ new_max_memory = get_balanced_memory(+ self.model,+ max_memory=new_max_memory,+ no_split_module_classes=no_split_modules,+ )+ self.model.tie_weights()+ device_map = infer_auto_device_map(+ self.model, max_memory=new_max_memory, no_split_module_classes=no_split_modules+ )+ if len(devices) > 1 and "cpu" in device_map.values():+ logger.warning(+ "Some layers are offloaded to cpu, which may severely impact calibration speed."+ " Please consider using more cards.")- if len(devices) > 1 and "cpu" in device_map.values():++ try:++ self.model = dispatch_model(self.model, device_map=device_map)+ except ValueError as e:+ if "offload_dir" in e.__str__():logger.warning(- "Some layers are offloaded to cpu, which may severely impact calibration speed."- " Please consider using more cards."+ f"Due to insufficient resources, disk is used to store the model."+ f" `offload_dir={envs.AR_WORK_SPACE}`")+ self.model = dispatch_model(+ self.model, device_map=device_map, offload_dir=envs.AR_WORK_SPACE+ )+ else:+ raise+ else:- try:- materialize_model_(self.model)- self.model = dispatch_model(self.model, device_map=device_map)- except ValueError as e:- if "offload_dir" in e.__str__():- logger.warning(- f"Due to insufficient resources, disk is used to store the model."- f" `offload_dir={envs.AR_WORK_SPACE}`"- )- self.model = dispatch_model(- self.model, device_map=device_map, offload_dir=envs.AR_WORK_SPACE- )- else:- raise- else:- self.model = self.model.to(self.device)+ self.model = self.model.to(self.device)all_inputs = self.cache_inter_data(block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_namediff --git a/auto_round/modeling/fused_moe/qwen3_5_moe.py b/auto_round/modeling/fused_moe/qwen3_5_moe.pynew file mode 100644index 000000000..b9fc560a7--- /dev/null+++ b/auto_round/modeling/fused_moe/qwen3_5_moe.py@@ -0,0 +1,113 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++import torch+import torch.nn.functional as F+from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import Qwen3_5MoeMLP+from transformers.utils.versions import require_version++from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase+from auto_round.utils import clear_memory, unsupported_meta_device++require_version("transformers>=5.2.0")++from auto_round.modeling.fused_moe.utils import _update_parameter+++class LinearQwen3_5MoeSparseMoeBlock(ReplacementModuleBase):+ def __init__(self, original, config):+ super().__init__(original)+ self.gate = original.gate+ text_config = config.get_text_config()+ self.shared_expert = original.shared_expert+ with torch.device("meta"):+ self.experts = SequentialQwen3_5MoeExperts(text_config, original.experts)+ self.shared_expert_gate = original.shared_expert_gate+ self.num_experts = text_config.num_experts++ @classmethod+ def original_module_class(cls) -> str:+ """Return the class name of the module this replaces."""+ return "Qwen3_5MoeSparseMoeBlock"++ def _materialize_weights(self) -> None:+ original = self._get_original_module()+ self.experts._materialize_weights(original.experts)+ clear_memory()++ def experts_forward(+ self,+ hidden_states: torch.Tensor,+ top_k_index: torch.Tensor,+ top_k_weights: torch.Tensor,+ ) -> torch.Tensor:+ final_hidden_states = torch.zeros_like(hidden_states)+ with torch.no_grad():+ expert_mask = torch.nn.functional.one_hot(top_k_index, num_classes=self.num_experts)+ expert_mask = expert_mask.permute(2, 1, 0)+ expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()++ for expert_idx in expert_hit:+ expert_idx = expert_idx[0]+ if expert_idx == self.num_experts:+ continue+ top_k_pos, token_idx = torch.where(expert_mask[expert_idx])+ current_state = hidden_states[token_idx]+ # gate, up = nn.functional.linear(current_state, self.gate_up_proj[expert_idx]).chunk(2, dim=-1)+ # current_hidden_states = self.act_fn(gate) * up+ # current_hidden_states = nn.functional.linear(current_hidden_states, self.down_proj[expert_idx])+ current_hidden_states = self.experts[expert_idx](current_state)+ current_hidden_states = current_hidden_states * top_k_weights[token_idx, top_k_pos, None]+ final_hidden_states.index_add_(0, token_idx, current_hidden_states.to(final_hidden_states.dtype))++ return final_hidden_states++ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:+ batch_size, sequence_length, hidden_dim = hidden_states.shape+ hidden_states_reshaped = hidden_states.view(-1, hidden_dim)+ shared_expert_output = self.shared_expert(hidden_states_reshaped)+ _, routing_weights, selected_experts = self.gate(hidden_states_reshaped)+ expert_output = self.experts_forward(hidden_states_reshaped, selected_experts, routing_weights)++ shared_expert_output = F.sigmoid(self.shared_expert_gate(hidden_states_reshaped)) * shared_expert_output++ expert_output += shared_expert_output+ expert_output = expert_output.reshape(batch_size, sequence_length, hidden_dim)+ return expert_output++ @classmethod+ def from_original(+ cls,+ original,+ config,+ **kwargs,+ ):+ """Create an instance from the original module."""+ return cls(original, config)+++class SequentialQwen3_5MoeExperts(torch.nn.ModuleList):+ def __init__(self, config, original):+ super().__init__()+ self.num_experts = original.gate_up_proj.shape[0]+ intermediate_size = config.moe_intermediate_size++ with torch.device("meta"):+ super().__init__([Qwen3_5MoeMLP(config, intermediate_size) for _ in range(self.num_experts)])++ def _materialize_weights(self, original) -> None:+ intermediate_size = original.down_proj.shape[-1]+ if not unsupported_meta_device(original):+ for i in range(self.num_experts):+ gate_up = original.gate_up_proj[i]+ down = original.down_proj[i]++ gate_proj = gate_up[:intermediate_size, :]+ up_proj = gate_up[intermediate_size:, :]++ _update_parameter(self[i].gate_proj, "weight", gate_proj.contiguous())+ _update_parameter(self[i].up_proj, "weight", up_proj.contiguous())+ _update_parameter(self[i].down_proj, "weight", down.contiguous())+ del gate_up, down, gate_proj, up_proj+ original.to_empty(device="meta") # release original experts parameters+ clear_memory()diff --git a/auto_round/modeling/fused_moe/qwen3_vl_moe.py b/auto_round/modeling/fused_moe/qwen3_vl_moe.pyindex 9864f2b10..102b5abd9 100644--- a/auto_round/modeling/fused_moe/qwen3_vl_moe.py+++ b/auto_round/modeling/fused_moe/qwen3_vl_moe.py@@ -158,9 +158,9 @@ class SequentialQwen3VLMoeTextExperts(torch.nn.ModuleList):gate_proj = gate_up[:, :intermediate_size]up_proj = gate_up[:, intermediate_size:]- _update_parameter(self[i].gate_proj, "weight", gate_proj.t())- _update_parameter(self[i].up_proj, "weight", up_proj.t())- _update_parameter(self[i].down_proj, "weight", down.t())+ _update_parameter(self[i].gate_proj, "weight", gate_proj.t().contiguous())+ _update_parameter(self[i].up_proj, "weight", up_proj.t().contiguous())+ _update_parameter(self[i].down_proj, "weight", down.t().contiguous())del gate_up, down, gate_proj, up_projoriginal.to_empty(device="meta") # release original experts parametersclear_memory()diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.pyindex 06845e93c..ac6bb8ee2 100644--- a/auto_round/modeling/fused_moe/replace_modules.py+++ b/auto_round/modeling/fused_moe/replace_modules.py@@ -25,10 +25,12 @@ from auto_round.modeling.fused_moe.moe_experts_interface import LINEAR_LOOP_IMPLfrom auto_round.utils import LazyImport, dump_mem_usage, dump_memory_usage_ctx, global_state, loggerBUILTIN_MODULES = {- "Llama4TextMoe": LazyImport("auto_round.modeling.fused_moe.llama4"),- "GptOssMLP": LazyImport("auto_round.modeling.fused_moe.gpt_oss"),- "Qwen3VLMoeTextSparseMoeBlock": LazyImport("auto_round.modeling.fused_moe.qwen3_vl_moe"),- "DeepseekV2Attention": LazyImport("auto_round.modeling.fused_moe.deepseek_v2"),+ "llama4": LazyImport("auto_round.modeling.fused_moe.llama4"),+ "gpt_oss": LazyImport("auto_round.modeling.fused_moe.gpt_oss"),+ "qwen3_vl_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_vl_moe"),+ "deepseek_v2": LazyImport("auto_round.modeling.fused_moe.deepseek_v2"),+ "qwen3_5_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),+ "qwen3_5_moe_text": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),}@@ -37,19 +39,11 @@ def _has_custom_replacement_only(model: torch.nn.Module) -> bool:If all MOE modules are covered by BUILTIN_MODULES, we can skip the linear_loop path entirely."""- for _, module in model.named_modules():- class_name = module.__class__.__name__- # Check if this looks like an MOE experts module (has fused 3D weights)- if (- hasattr(module, "gate_up_proj")- and isinstance(module.gate_up_proj, torch.nn.Parameter)- and module.gate_up_proj.dim() == 3- ):- # This is a fused experts module - check if it has a custom replacement- if class_name not in BUILTIN_MODULES and not ReplacementModuleBase.is_registered(class_name):- # Found a fused experts module without custom replacement - need linear_loop- return False- return True+ if hasattr(model, "config") and hasattr(model.config, "model_type"):+ model_type = model.config.model_type+ if model_type in BUILTIN_MODULES:+ return True+ return Falsedef _handle_moe_modules(model: torch.nn.Module) -> list[str]:@@ -84,15 +78,12 @@ def _handle_moe_modules(model: torch.nn.Module) -> list[str]:def _import_required_replacements(model: torch.nn.Module) -> None:"""Scan model and trigger lazy imports for registered replacement modules."""imported = set()-- for _, module in model.named_modules():- class_name = module.__class__.__name__-- if class_name in BUILTIN_MODULES and class_name not in imported:- # Trigger import by accessing the LazyImport object- _ = BUILTIN_MODULES[class_name].__name__ # or any attribute- imported.add(class_name)- logger.debug(f"Loaded replacement module for {class_name}")+ if hasattr(model, "config") and hasattr(model.config, "model_type"):+ model_type = model.config.model_type+ if model_type in BUILTIN_MODULES:+ _ = BUILTIN_MODULES[model_type].__name__ # or any attribute+ imported.add(model_type)+ logger.debug(f"Loaded replacement module for {model_type}")def _should_skip_moe_replacement(module: torch.nn.Module, model: torch.nn.Module) -> bool:@@ -129,6 +120,7 @@ def materialize_model_(model: torch.nn.Module) -> None:module.materialize_weights()model.apply(_materialize_module)+# check if any module on meta device remainsfound_meta = Falsefor name, param in model.named_parameters():@@ -332,7 +324,7 @@ def apply_replacements(if isinstance(module, ReplacementModuleBase):continueclass_name = module.__class__.__name__- if class_name in BUILTIN_MODULES and _should_skip_moe_replacement(module, model):+ if ReplacementModuleBase.is_registered(class_name) and _should_skip_moe_replacement(module, model):logger.debug(f"Skipping replacement for {name}: linear_loop experts already unfused")continueif ReplacementModuleBase.is_registered(class_name) and ReplacementModuleBase.get_replacement_class(diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 7627f0782..2371941f5 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -217,16 +217,34 @@ def register_ignore_layers(_PRE_DEFINED_IGNORE_LAYERS.append(rule)-# Qwen3MOEregister_ignore_layers(matchers=[- ArchitectureMatcher(r"Qwen3.*Moe", mode="regex"),+ ModelTypeMatcher(r"qwen3_vl_moe", mode="full"),],ignore_layers=["mlp.gate", # vllm inference issue],)+register_ignore_layers(+ matchers=[+ ModelTypeMatcher(r"qwen3_moe", mode="full"),+ ],+ ignore_layers=[+ "mlp.gate", # vllm inference issue+ ],+)++register_ignore_layers(+ matchers=[+ ModelTypeMatcher(r"qwen3_5_moe", mode="full"),+ ],+ ignore_layers=[+ "mlp.gate", # vllm inference issue+ ],+)++# longcatregister_ignore_layers(matchers=[@@ -266,17 +284,6 @@ register_ignore_layers()-# # qwen3_next-# register_ignore_layers(-# matchers=[-# ModelTypeMatcher(r"qwen3_next", mode="full"),-# ],-# ignore_layers=[-# "mlp.gate", # vllm issue-# ],-# )--def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:layers = []for rule in _PRE_DEFINED_IGNORE_LAYERS:@@ -291,5 +298,10 @@ def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:else:layers.extend(res)break+ if not layers:+ if hasattr(model, "config") and hasattr(model.config, "model_type"):+ model_type = model.config.model_type+ if "moe" in model_type: # Append gate which usually cause vllm issue+ layers.append("mlp.gate")return list(dict.fromkeys(layers))diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 1442512c9..3d010d150 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -1426,7 +1426,6 @@ def _get_reference_amax_from_experts(moe_module: torch.nn.Module, attr_name: str# 5b3ddff74cae9651f24bef15d3255c4ee053fc60/src/llmcompressor/pytorch/model_load/helpers.py#L144def copy_python_files_from_model_cache(model, save_path: str):config = model.config- cache_path = Noneif hasattr(config, "_name_or_path"):import osimport shutil@@ -1434,14 +1433,13 @@ def copy_python_files_from_model_cache(model, save_path: str):from huggingface_hub import hf_hub_downloadif version.parse(transformers.__version__) < version.parse("5.0.0"):- from transformers import TRANSFORMERS_CACHE+ from transformers.utils import TRANSFORMERS_CACHE- cache_dir = TRANSFORMERS_CACHE+ cache_dir = os.environ.get("HF_HOME", TRANSFORMERS_CACHE)+ else:from huggingface_hub.constants import HF_HUB_CACHE- cache_dir = os.environ.get("HF_HOME") or HF_HUB_CACHE-- cache_dir = os.environ.get("HF_HOME", None)+ cache_dir = os.environ.get("HF_HOME", HF_HUB_CACHE)from transformers.utils import http_user_agentcache_path = config._name_or_pathdiff --git a/test/helpers.py b/test/helpers.pyindex 8558fbd1b..2f98e4e55 100644--- a/test/helpers.py+++ b/test/helpers.py@@ -242,6 +242,7 @@ def get_output(model_name_or_path):return outputs.detach().cpu()[email protected]_mode()def is_model_outputs_similar(model_path_1, model_path_2, metric="cosine_similarity", threshold=0.98, k=5, verbose=True):"""Compare outputs from two models using specified metric and return pass/fail.diff --git a/test/test_cpu/schemes/test_scheme.py b/test/test_cpu/schemes/test_scheme.pyindex dcafb3ca8..77ab5fd1f 100644--- a/test/test_cpu/schemes/test_scheme.py+++ b/test/test_cpu/schemes/test_scheme.py@@ -61,7 +61,7 @@ class TestAutoRound:assert ar.model.model.layers[0].self_attn.q_proj.bits == 8assert ar.model.model.layers[0].self_attn.k_proj.bits == 16assert ar.model.model.layers[0].mlp.experts[0].up_proj.bits == 4- assert ar.model.model.layers[0].mlp.shared_expert.gate_proj.bits == 8+ # assert ar.model.model.layers[0].mlp.shared_expert.gate_proj.bits == 8 # gate has been added to ignore_layersshutil.rmtree(self.save_folder, ignore_errors=True)