PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support qwen3-moe/glm-4.7-flash monkey patch for transformers 5 (#1354)
Wenhua Cheng6bc1d344afcf
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex b1f46cc1e..a1c12631c 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -25,11 +25,13 @@ from typing import Any, Callable, Optional, Unionimport accelerateimport torch+import transformersfrom accelerate.big_modeling import dispatch_model, infer_auto_device_mapfrom accelerate.utils import get_balanced_memory, get_max_memory+from packaging import versionfrom torch import autocastfrom tqdm import tqdm-from transformers import set_seed+from transformers import AutoConfig, set_seedfrom auto_round import envsfrom auto_round.auto_scheme.gen_auto_scheme import AutoScheme@@ -52,10 +54,11 @@ from auto_round.compressors.utils import ()from auto_round.data_type import QUANT_FUNC_WITH_DTYPEfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size-from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG, ModelType+from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIGfrom auto_round.formats import OutputFormat, get_formatsfrom auto_round.logger import logger-from auto_round.modelling.replace_modules import materialize_model_, safe_to_cpu_+from auto_round.modeling.fused_moe import apply_model_monkey_patches+from auto_round.modeling.unfused_moe.replace_modules import materialize_model_, safe_to_cpu_from auto_round.schemes import (QuantizationScheme,_handle_special_schemes,@@ -93,6 +96,7 @@ from auto_round.utils import (is_fp8_model,is_hpex_available,is_moe_model,+ is_moe_model_via_config,llm_load_model,memory_monitor,mv_module_from_gpu,@@ -259,7 +263,25 @@ class BaseCompressor(object):self.trust_remote_code = kwargs.pop("trust_remote_code") if "trust_remote_code" in kwargs else Trueself.diffusion = kwargs.pop("diffusion") if "diffusion" in kwargs else Falseself.quantized = False+ self.is_model_patched = Falseif isinstance(model, str):+ try:+ # config = AutoConfig.from_pretrained(model)+ self.is_model_patched = apply_model_monkey_patches(model_name=model)++ # TODO excluded ori_params_moe+ # if (+ # not self.is_model_patched+ # and is_moe_model_via_config(config)+ # and version.parse(transformers.__version__) >= version.parse("5.0.0")+ # ):+ # logger.warning(+ # "This moe model is not optimized by AutoRound yet which may cause large ram usage, "+ # "please submit an issue to https://github.com/intel/auto-round/issues"+ # )++ except:+ passmodel, tokenizer = llm_load_model(model,platform=platform,@@ -469,7 +491,6 @@ class BaseCompressor(object):if is_hpex_available():logger.info("habana_frameworks is available, import htcore explicitly.")import habana_frameworks.torch.core as htcore # pylint: disable=E0401- import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401]self.attention_mask = []diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.pyindex 7641895b0..cc8b11a93 100644--- a/auto_round/compressors/shard_writer.py+++ b/auto_round/compressors/shard_writer.py@@ -38,7 +38,8 @@ class ShardWriter:# - final // 10 -> apply a safety margin so default shards are# smaller than the full model; this intentionally# underestimates size before clamping below.- model_size = int(total_params * rounder.bits // 1e9 // 8) // 10+ max_split_num = 10+ model_size = int(total_params * rounder.bits // 1e9 // 8 + max_split_num - 1) / max_split_nummodel_size = max(1, min(int(model_size), 5))# Configurationdiff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 3daa3c822..ced2d3f2b 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -30,6 +30,7 @@ from auto_round.inference.backend import ()from auto_round.inference.utils import _expand_regex_configfrom auto_round.logger import logger+from auto_round.modeling.fused_moe import apply_modeling_patchfrom auto_round.schemes import QuantizationSchemefrom auto_round.special_model_handler import update_modulefrom auto_round.utils import (@@ -591,9 +592,10 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.Mpacking_format = "auto_round:auto_awq"elif packing_format == "auto_round:gptq":packing_format = "auto_round:auto_gptq"-- # Preprocess model before replace layers- model = update_module(model, cleanup_original=True)+ is_applied = apply_modeling_patch(model)+ if not is_applied:+ # Preprocess model before replace layers+ model = update_module(model, cleanup_original=True)# Replace layers with quantized versionslayer_configs = get_layer_config(model, quantization_config)diff --git a/auto_round/modelling/__init__.py b/auto_round/modeling/__init__.pysimilarity index 93%rename from auto_round/modelling/__init__.pyrename to auto_round/modeling/__init__.pyindex 0d2740cdb..14a492441 100644--- a/auto_round/modelling/__init__.py+++ b/auto_round/modeling/__init__.py@@ -1,4 +1,4 @@-# Copyright (c) 2025 Intel Corporation+# Copyright (c) 2026 Intel Corporation## Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.diff --git a/auto_round/modeling/fused_moe/__init__.py b/auto_round/modeling/fused_moe/__init__.pynew file mode 100644index 000000000..1b50f0e41--- /dev/null+++ b/auto_round/modeling/fused_moe/__init__.py@@ -0,0 +1,193 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import os+from auto_round import envs+import transformers+import importlib+from packaging import version+from transformers import AutoConfig++from auto_round.logger import logger+import torch++MODEL_CONFIG = {+ "qwen3_moe": {+ "min_transformers_version": "5.0.0",+ "max_transformers_version": None,+ "checkpoint_mapping": [],+ "block_patch": [+ (+ "transformers.models.qwen3_moe.modeling_qwen3_moe.Qwen3MoeSparseMoeBlock",+ "auto_round.modeling.fused_moe.qwen3_moe.LinearQwen3MoeSparseMoeBlock",+ )+ ],+ },+ "glm4_moe_lite": {+ "min_transformers_version": "5.0.0",+ "max_transformers_version": None,+ "checkpoint_mapping": [],+ "block_patch": [+ (+ "transformers.models.glm4_moe_lite.modeling_glm4_moe_lite.Glm4MoeLiteMoE",+ "auto_round.modeling.fused_moe.glm_moe_light.LinearGlm4MoeLiteMoE",+ )+ ],+ },+}+++def get_checkpoint_conversion_mapping_ar(model_type):+ from transformers import conversion_mapping++ if not hasattr(conversion_mapping, "orig_get_checkpoint_conversion_mapping"):+ conversion_mapping.orig_get_checkpoint_conversion_mapping = conversion_mapping.get_checkpoint_conversion_mapping++ cfg = MODEL_CONFIG.get(model_type)+ if cfg:+ return cfg.get("checkpoint_mapping", [])++ from transformers import conversion_mapping++ return conversion_mapping.orig_get_checkpoint_conversion_mapping(model_type)+++def get_file_path_via_model_name(model_or_path: str, file_name):+ from huggingface_hub import hf_hub_download++ # 1) local folder+ if os.path.isdir(model_or_path):+ index_path = os.path.join(model_or_path, file_name)++ # 2) HF model name+ elif not envs.AR_USE_MODELSCOPE:+ index_path = hf_hub_download(+ repo_id=model_or_path,+ filename=file_name,+ repo_type="model",+ )+ elif envs.AR_USE_MODELSCOPE:+ from modelscope import snapshot_download # pylint: disable=E0401++ # ModelSCOPE is different, it returns the folder path+ folder = snapshot_download(model_or_path, allow_patterns=[file_name])+ index_path = os.path.join(folder, file_name)+ else:+ index_path = None++ return index_path+++def pre_check_config(model_name: str | torch.nn.Module):+ if isinstance(model_name, str):+ config = AutoConfig.from_pretrained(model_name)+ elif isinstance(model_name, torch.nn.Module):+ config = getattr(model_name, "config", None)+ if config is None:+ return False++ model_type = getattr(config, "model_type", None)+ if model_type is None or model_type not in MODEL_CONFIG:+ return False++ cfg = MODEL_CONFIG[model_type]++ min_ver = cfg.get("min_transformers_version")+ max_ver = cfg.get("max_transformers_version")+ tf_ver = version.parse(transformers.__version__)+ if min_ver and tf_ver < version.parse(min_ver):+ return False+ if max_ver and tf_ver > version.parse(max_ver):+ return False+ try:+ file_path = get_file_path_via_model_name(model_name, "model.safetensors.index.json")+ if os.path.exists(file_path):+ import json++ with open(file_path, "r") as f:+ index_data = json.load(f)+ model_keys = list(index_data.get("weight_map", {}).keys())+ for key in model_keys:+ if "gate_up_proj" in key:+ return False+ except:+ return True+ return True+++# This is for model checkpoint with linear definition+def apply_model_monkey_patches(model_name: str) -> bool:+ res = pre_check_config(model_name)+ if not res:+ return False+ # patch blocks+ config = AutoConfig.from_pretrained(model_name)+ model_type = getattr(config, "model_type")++ cfg = MODEL_CONFIG[model_type]+ for orig_path, custom_path in cfg.get("block_patch", []):+ orig_module_path, orig_class_name = orig_path.rsplit(".", 1)+ custom_module_path, custom_class_name = custom_path.rsplit(".", 1)++ try:+ orig_module = importlib.import_module(orig_module_path)+ custom_module = importlib.import_module(custom_module_path)+ custom_class = getattr(custom_module, custom_class_name)+ setattr(orig_module, orig_class_name, custom_class)++ if version.parse(transformers.__version__) >= version.parse("5.0.0"):+ from transformers import conversion_mapping++ if not hasattr(conversion_mapping, "orig_get_checkpoint_conversion_mapping"):+ conversion_mapping.orig_get_checkpoint_conversion_mapping = (+ conversion_mapping.get_checkpoint_conversion_mapping+ )++ conversion_mapping.get_checkpoint_conversion_mapping = get_checkpoint_conversion_mapping_ar+ transformers.modeling_utils.get_checkpoint_conversion_mapping = get_checkpoint_conversion_mapping_ar+ logger.info(f"Patched {orig_path} -> {custom_path}")+ return True++ except Exception as e:+ logger.warning(f"Failed to patch {orig_path}: {e}")+ return False+ return False+++def apply_modeling_patch(model: torch.nn.Module) -> bool:+ res = pre_check_config(model)+ if not res:+ return False+ model_type = getattr(model.config, "model_type")+ cfg = MODEL_CONFIG[model_type]+ # patch blocks+ for orig_path, custom_path in cfg.get("block_patch", []):+ orig_module_path, orig_class_name = orig_path.rsplit(".", 1)+ custom_module_path, custom_class_name = custom_path.rsplit(".", 1)+ try:+ orig_module = importlib.import_module(orig_module_path)+ custom_module = importlib.import_module(custom_module_path)+ custom_class = getattr(custom_module, custom_class_name)+ orig_class = getattr(orig_module, orig_class_name)+ names = []+ for n, m in model.named_modules():+ if isinstance(m, orig_class):+ names.append(n)+ for n in names:+ model.set_submodule(n, custom_class(model.config), True)+ logger.info(f"Patched module: {orig_path} -> {custom_path}")+ return True+ except Exception as e:+ logger.warning(f"Failed to patch {orig_path}: {e}")+ return False+ return Falsediff --git a/auto_round/modeling/fused_moe/glm_moe_light.py b/auto_round/modeling/fused_moe/glm_moe_light.pynew file mode 100644index 000000000..c9506ecb4--- /dev/null+++ b/auto_round/modeling/fused_moe/glm_moe_light.py@@ -0,0 +1,107 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch+import torch.nn as nn+++class LinearGlm4MoeLiteMoE(nn.Module):+ """+ A mixed expert module containing shared experts.+ """++ def __init__(self, config):+ super().__init__()+ self.config = config+ self.num_experts = config.num_local_experts+ from transformers.models.glm4_moe_lite.modeling_glm4_moe_lite import Glm4MoeLiteMLP, Glm4MoeLiteTopkRouter++ self.experts = nn.ModuleList(+ [Glm4MoeLiteMLP(config, intermediate_size=config.moe_intermediate_size) for _ in range(self.num_experts)]+ )++ self.gate = Glm4MoeLiteTopkRouter(config)+ self.shared_experts = Glm4MoeLiteMLP(+ config=config, intermediate_size=config.moe_intermediate_size * config.n_shared_experts+ )+ self.n_routed_experts = config.n_routed_experts+ self.n_group = config.n_group+ self.topk_group = config.topk_group+ self.norm_topk_prob = config.norm_topk_prob+ self.routed_scaling_factor = config.routed_scaling_factor+ self.top_k = config.num_experts_per_tok++ def experts_forward(+ self,+ hidden_states: torch.Tensor,+ top_k_index: torch.Tensor,+ top_k_weights: torch.Tensor,+ ) -> torch.Tensor:+ """ """+ final_hidden_states = torch.zeros_like(hidden_states)+ with torch.no_grad():+ expert_mask = torch.nn.functional.one_hot(top_k_index, num_classes=self.num_experts)+ expert_mask = expert_mask.permute(2, 1, 0)+ expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()++ for expert_idx in expert_hit:+ expert_idx = expert_idx[0]+ if expert_idx == self.num_experts:+ continue+ top_k_pos, token_idx = torch.where(expert_mask[expert_idx])+ current_state = hidden_states[token_idx]+ expert_layer = self.experts[expert_idx]+ current_hidden_states = expert_layer(current_state)+ # gate, up = nn.functional.linear(current_state, self.gate_up_proj[expert_idx]).chunk(2, dim=-1)+ # current_hidden_states = self.act_fn(gate) * up+ # current_hidden_states = nn.functional.linear(current_hidden_states, self.down_proj[expert_idx])+ current_hidden_states = current_hidden_states * top_k_weights[token_idx, top_k_pos, None]+ final_hidden_states.index_add_(0, token_idx, current_hidden_states.to(final_hidden_states.dtype))++ return final_hidden_states++ def route_tokens_to_experts(self, router_logits):+ router_logits = router_logits.sigmoid()+ router_logits_for_choice = router_logits + self.gate.e_score_correction_bias+ group_scores = (+ router_logits_for_choice.view(-1, self.n_group, self.n_routed_experts // self.n_group)+ .topk(2, dim=-1)[0]+ .sum(dim=-1)+ )+ group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]+ group_mask = torch.zeros_like(group_scores)+ group_mask.scatter_(1, group_idx, 1)+ score_mask = (+ group_mask.unsqueeze(-1)+ .expand(-1, self.n_group, self.n_routed_experts // self.n_group)+ .reshape(-1, self.n_routed_experts)+ )+ scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), 0.0)+ topk_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1]+ topk_weights = router_logits.gather(1, topk_indices)+ if self.norm_topk_prob:+ denominator = topk_weights.sum(dim=-1, keepdim=True) + 1e-20+ topk_weights /= denominator+ topk_weights = topk_weights * self.routed_scaling_factor+ return topk_indices, topk_weights++ def forward(self, hidden_states):+ residuals = hidden_states+ orig_shape = hidden_states.shape+ router_logits = self.gate(hidden_states)+ topk_indices, topk_weights = self.route_tokens_to_experts(router_logits)+ hidden_states = hidden_states.view(-1, hidden_states.shape[-1])+ hidden_states = self.experts_forward(hidden_states, topk_indices, topk_weights).view(*orig_shape)+ hidden_states = hidden_states + self.shared_experts(residuals)+ return hidden_statesdiff --git a/auto_round/modeling/fused_moe/qwen3_moe.py b/auto_round/modeling/fused_moe/qwen3_moe.pynew file mode 100644index 000000000..02e1bc62a--- /dev/null+++ b/auto_round/modeling/fused_moe/qwen3_moe.py@@ -0,0 +1,73 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch+import torch.nn as nn+from torch.nn import functional as F+++class LinearQwen3MoeSparseMoeBlock(nn.Module):+ def __init__(self, config):+ super().__init__()+ from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeMLP++ self.num_experts = config.num_experts+ self.top_k = config.num_experts_per_tok+ self.norm_topk_prob = config.norm_topk_prob++ # gating+ self.gate = nn.Linear(config.hidden_size, config.num_experts, bias=False)+ self.experts = nn.ModuleList(+ [Qwen3MoeMLP(config, intermediate_size=config.moe_intermediate_size) for _ in range(self.num_experts)]+ )++ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:+ """ """+ batch_size, sequence_length, hidden_dim = hidden_states.shape+ hidden_states = hidden_states.view(-1, hidden_dim)+ # router_logits: (batch * sequence_length, n_experts)+ router_logits = self.gate(hidden_states)++ routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)+ routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)+ if self.norm_topk_prob: # only diff with mixtral sparse moe block!+ routing_weights /= routing_weights.sum(dim=-1, keepdim=True)+ # we cast back to the input dtype+ routing_weights = routing_weights.to(hidden_states.dtype)++ final_hidden_states = torch.zeros(+ (batch_size * sequence_length, hidden_dim), dtype=hidden_states.dtype, device=hidden_states.device+ )++ # One hot encode the selected experts to create an expert mask+ # this will be used to easily index which expert is going to be solicited+ expert_mask = torch.nn.functional.one_hot(selected_experts, num_classes=self.num_experts).permute(2, 1, 0)++ # Loop over all available experts in the model and perform the computation on each expert+ expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()+ for expert_idx in expert_hit:+ expert_layer = self.experts[expert_idx]+ idx, top_x = torch.where(expert_mask[expert_idx].squeeze(0))++ # Index the correct hidden states and compute the expert hidden state for+ # the current expert. We need to make sure to multiply the output hidden+ # states by `routing_weights` on the corresponding tokens (top-1 and top-2)+ current_state = hidden_states[None, top_x].reshape(-1, hidden_dim)+ current_hidden_states = expert_layer(current_state) * routing_weights[top_x, idx, None]++ # However `index_add_` only support torch tensors for indexing so we'll use+ # the `top_x` tensor here.+ final_hidden_states.index_add_(0, top_x, current_hidden_states.to(hidden_states.dtype))+ final_hidden_states = final_hidden_states.reshape(batch_size, sequence_length, hidden_dim)+ return final_hidden_statesdiff --git a/auto_round/modeling/unfused_moe/__init__.py b/auto_round/modeling/unfused_moe/__init__.pynew file mode 100644index 000000000..14a492441--- /dev/null+++ b/auto_round/modeling/unfused_moe/__init__.py@@ -0,0 +1,13 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.diff --git a/auto_round/modelling/deepseek_v2.py b/auto_round/modeling/unfused_moe/deepseek_v2.pysimilarity index 98%rename from auto_round/modelling/deepseek_v2.pyrename to auto_round/modeling/unfused_moe/deepseek_v2.pyindex 7caff2696..0f026aac4 100644--- a/auto_round/modelling/deepseek_v2.py+++ b/auto_round/modeling/unfused_moe/deepseek_v2.py@@ -18,7 +18,6 @@ from functools import partialfrom typing import Callable, Optionalimport torch-from transformers.cache_utils import Cachefrom transformers.modeling_rope_utils import dynamic_rope_updatefrom transformers.modeling_utils import ALL_ATTENTION_FUNCTIONSfrom transformers.models.deepseek_v2.configuration_deepseek_v2 import DeepseekV2Config@@ -26,7 +25,7 @@ from transformers.models.deepseek_v2.modeling_deepseek_v2 import (eager_attention_forward,)-from auto_round.modelling.replace_modules import ReplacementModuleBase+from auto_round.modeling.unfused_moe.replace_modules import ReplacementModuleBasefrom auto_round.utils import is_hpex_availablediff --git a/auto_round/modelling/gpt_oss.py b/auto_round/modeling/unfused_moe/gpt_oss.pysimilarity index 96%rename from auto_round/modelling/gpt_oss.pyrename to auto_round/modeling/unfused_moe/gpt_oss.pyindex a71b5b610..96caeb71d 100644--- a/auto_round/modelling/gpt_oss.py+++ b/auto_round/modeling/unfused_moe/gpt_oss.py@@ -27,9 +27,9 @@ else:from transformers.models.gpt_oss.configuration_gpt_oss import GptOssConfigfrom transformers.models.gpt_oss.modeling_gpt_oss import GptOssMLP-from auto_round.modelling.replace_modules import ReplacementModuleBase-from auto_round.modelling.utils import _update_parameter-from auto_round.utils import LazyImport, clear_memory, logger, unsupported_meta_device+from auto_round.modeling.unfused_moe.replace_modules import ReplacementModuleBase+from auto_round.modeling.unfused_moe.utils import _update_parameter+from auto_round.utils import clear_memory, unsupported_meta_deviceclass GPTOssSingleExpert(nn.Module):diff --git a/auto_round/modelling/llama4.py b/auto_round/modeling/unfused_moe/llama4.pysimilarity index 96%rename from auto_round/modelling/llama4.pyrename to auto_round/modeling/unfused_moe/llama4.pyindex 7057f777d..5e6adc274 100644--- a/auto_round/modelling/llama4.py+++ b/auto_round/modeling/unfused_moe/llama4.py@@ -21,10 +21,10 @@ if transformers_version < version.parse("5.0.0"):from transformers.modeling_utils import no_init_weightselse:from transformers.initialization import no_init_weights-from transformers.models.llama4.modeling_llama4 import Llama4Config, Llama4TextMLP, Llama4TextMoe+from transformers.models.llama4.modeling_llama4 import Llama4Config, Llama4TextMLP-from auto_round.modelling.replace_modules import ReplacementModuleBase-from auto_round.modelling.utils import _update_parameter+from auto_round.modeling.unfused_moe.replace_modules import ReplacementModuleBase+from auto_round.modeling.unfused_moe.utils import _update_parameterfrom auto_round.utils import clear_memory, unsupported_meta_devicediff --git a/auto_round/modelling/qwen3_vl_moe.py b/auto_round/modeling/unfused_moe/qwen3_vl_moe.pysimilarity index 96%rename from auto_round/modelling/qwen3_vl_moe.pyrename to auto_round/modeling/unfused_moe/qwen3_vl_moe.pyindex 609b1df74..1c0d62a29 100644--- a/auto_round/modelling/qwen3_vl_moe.py+++ b/auto_round/modeling/unfused_moe/qwen3_vl_moe.py@@ -15,16 +15,14 @@import torchimport transformersfrom packaging import version-from torch import nn-from transformers.activations import ACT2FN-from auto_round.modelling.replace_modules import ReplacementModuleBase-from auto_round.utils import clear_memory, logger, unsupported_meta_device+from auto_round.modeling.unfused_moe.replace_modules import ReplacementModuleBase+from auto_round.utils import clear_memory, unsupported_meta_devicetransformers_version = version.parse(transformers.__version__)from typing import TYPE_CHECKING-from auto_round.modelling.utils import _update_parameter+from auto_round.modeling.unfused_moe.utils import _update_parameterif TYPE_CHECKING:from transformers import Qwen3VLMoeConfig, Qwen3VLMoeTextConfigdiff --git a/auto_round/modelling/replace_modules.py b/auto_round/modeling/unfused_moe/replace_modules.pysimilarity index 98%rename from auto_round/modelling/replace_modules.pyrename to auto_round/modeling/unfused_moe/replace_modules.pyindex d77ffca2d..c62345fa1 100644--- a/auto_round/modelling/replace_modules.py+++ b/auto_round/modeling/unfused_moe/replace_modules.py@@ -23,10 +23,10 @@ from transformers import PreTrainedModelfrom auto_round.utils import LazyImport, dump_mem_usage, dump_memory_usage_ctx, global_state, loggerBUILTIN_MODULES = {- "Llama4TextMoe": LazyImport("auto_round.modelling.llama4"),- "GptOssMLP": LazyImport("auto_round.modelling.gpt_oss"),- "Qwen3VLMoeTextSparseMoeBlock": LazyImport("auto_round.modelling.qwen3_vl_moe"),- "DeepseekV2Attention": LazyImport("auto_round.modelling.deepseek_v2"),+ "Llama4TextMoe": LazyImport("auto_round.modeling.unfused_moe.llama4"),+ "GptOssMLP": LazyImport("auto_round.modeling.unfused_moe.gpt_oss"),+ "Qwen3VLMoeTextSparseMoeBlock": LazyImport("auto_round.modeling.unfused_moe.qwen3_vl_moe"),+ "DeepseekV2Attention": LazyImport("auto_round.modeling.unfused_moe.deepseek_v2"),}diff --git a/auto_round/modelling/utils.py b/auto_round/modeling/unfused_moe/utils.pysimilarity index 100%rename from auto_round/modelling/utils.pyrename to auto_round/modeling/unfused_moe/utils.pydiff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex f3621f913..da02b43b0 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -17,10 +17,9 @@ from typing import Any, Callableimport torch-import auto_round.modelling as auto_round_modellingfrom auto_round.formats import OutputFormat-from auto_round.modelling.replace_modules import apply_replacements, materialize_model_, release_original_module_-from auto_round.utils import LazyImport, logger, unsupported_meta_device+from auto_round.modeling.unfused_moe.replace_modules import apply_replacements, release_original_module_+from auto_round.utils import loggermllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_sizediff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 012415507..8f18f7cad 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -1390,6 +1390,14 @@ def is_moe_model(model: torch.nn.Module) -> bool:return False+def is_moe_model_via_config(config) -> bool:+ if hasattr(config, "to_dict"):+ for key in config.to_dict().keys():+ if "moe" in key or "expert" in key:+ return True+ return False++def to_dtype(input, dtype=torch.float32):"""Moves input data to the specified data type.diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.pyindex 8872edccb..f98a5b958 100644--- a/test/test_cpu/models/test_moe_model.py+++ b/test/test_cpu/models/test_moe_model.py@@ -99,7 +99,7 @@ def test_gptoss(setup_gpt_oss, scheme):# Ensure the quantized model is not Noneassert quantized_model is not None, "Quantized model should not be None."from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear- from auto_round.modelling.gpt_oss import GPTOssSingleExpert+ from auto_round.modeling.unfused_moe.gpt_oss import GPTOssSingleExpertsingle_expert_cnt = count_modules_by_type(quantized_model, GPTOssSingleExpert)quant_linear_cnt = count_modules_by_type(quantized_model, QuantLinear)diff --git a/test/test_cuda/models/test_moe_model.py b/test/test_cuda/models/test_moe_model.pyindex 0cd2bd5e9..442d559ea 100644--- a/test/test_cuda/models/test_moe_model.py+++ b/test/test_cuda/models/test_moe_model.py@@ -9,8 +9,7 @@ from transformers.models.qwen3.modeling_qwen3 import Qwen3Config, Qwen3ForCausalfrom transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeForConditionalGenerationfrom auto_round import AutoRound-from auto_round.modelling.replace_modules import ReplacementModuleBase-from auto_round.utils import LazyImport, logger, unsupported_meta_device+from auto_round.modeling.unfused_moe.replace_modules import ReplacementModuleBase@pytest.fixture@@ -211,7 +210,7 @@ def has_module(model: torch.nn.Module, module_name: str) -> bool:def test_register_module_out_of_tree_base():- from auto_round.modelling.replace_modules import ReplacementModuleBase+ from auto_round.modeling.unfused_moe.replace_modules import ReplacementModuleBasefor name, subclass in ReplacementModuleBase._replacement_registry.items():if name == "Qwen3MLP":