Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support qwen3-moe/glm-4.7-flash monkey patch for transformers 5 (#1354)

Wenhua Cheng6bc1d344afcf
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index b1f46cc1e..a1c12631c 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -25,11 +25,13 @@ from typing import Any, Callable, Optional, Union
import accelerate
import torch
+import transformers
from accelerate.big_modeling import dispatch_model, infer_auto_device_map
from accelerate.utils import get_balanced_memory, get_max_memory
+from packaging import version
from torch import autocast
from tqdm import tqdm
-from transformers import set_seed
+from transformers import AutoConfig, set_seed
from auto_round import envs
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
@@ -52,10 +54,11 @@ from auto_round.compressors.utils import (
)
from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
-from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG, ModelType
+from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG
from auto_round.formats import OutputFormat, get_formats
from auto_round.logger import logger
-from auto_round.modelling.replace_modules import materialize_model_, safe_to_cpu_
+from auto_round.modeling.fused_moe import apply_model_monkey_patches
+from auto_round.modeling.unfused_moe.replace_modules import materialize_model_, safe_to_cpu_
from auto_round.schemes import (
QuantizationScheme,
_handle_special_schemes,
@@ -93,6 +96,7 @@ from auto_round.utils import (
is_fp8_model,
is_hpex_available,
is_moe_model,
+ is_moe_model_via_config,
llm_load_model,
memory_monitor,
mv_module_from_gpu,
@@ -259,7 +263,25 @@ class BaseCompressor(object):
self.trust_remote_code = kwargs.pop("trust_remote_code") if "trust_remote_code" in kwargs else True
self.diffusion = kwargs.pop("diffusion") if "diffusion" in kwargs else False
self.quantized = False
+ self.is_model_patched = False
if isinstance(model, str):
+ try:
+ # config = AutoConfig.from_pretrained(model)
+ self.is_model_patched = apply_model_monkey_patches(model_name=model)
+
+ # TODO excluded ori_params_moe
+ # if (
+ # not self.is_model_patched
+ # and is_moe_model_via_config(config)
+ # and version.parse(transformers.__version__) >= version.parse("5.0.0")
+ # ):
+ # logger.warning(
+ # "This moe model is not optimized by AutoRound yet which may cause large ram usage, "
+ # "please submit an issue to https://github.com/intel/auto-round/issues"
+ # )
+
+ except:
+ pass
model, tokenizer = llm_load_model(
model,
platform=platform,
@@ -469,7 +491,6 @@ class BaseCompressor(object):
if is_hpex_available():
logger.info("habana_frameworks is available, import htcore explicitly.")
import habana_frameworks.torch.core as htcore # pylint: disable=E0401
- import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401]
self.attention_mask = []
diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.py
index 7641895b0..cc8b11a93 100644
--- a/auto_round/compressors/shard_writer.py
+++ b/auto_round/compressors/shard_writer.py
@@ -38,7 +38,8 @@ class ShardWriter:
# - final // 10 -> apply a safety margin so default shards are
# smaller than the full model; this intentionally
# underestimates size before clamping below.
- model_size = int(total_params * rounder.bits // 1e9 // 8) // 10
+ max_split_num = 10
+ model_size = int(total_params * rounder.bits // 1e9 // 8 + max_split_num - 1) / max_split_num
model_size = max(1, min(int(model_size), 5))
# Configuration
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 3daa3c822..ced2d3f2b 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -30,6 +30,7 @@ from auto_round.inference.backend import (
)
from auto_round.inference.utils import _expand_regex_config
from auto_round.logger import logger
+from auto_round.modeling.fused_moe import apply_modeling_patch
from auto_round.schemes import QuantizationScheme
from auto_round.special_model_handler import update_module
from auto_round.utils import (
@@ -591,9 +592,10 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.M
packing_format = "auto_round:auto_awq"
elif packing_format == "auto_round:gptq":
packing_format = "auto_round:auto_gptq"
-
- # Preprocess model before replace layers
- model = update_module(model, cleanup_original=True)
+ is_applied = apply_modeling_patch(model)
+ if not is_applied:
+ # Preprocess model before replace layers
+ model = update_module(model, cleanup_original=True)
# Replace layers with quantized versions
layer_configs = get_layer_config(model, quantization_config)
diff --git a/auto_round/modelling/__init__.py b/auto_round/modeling/__init__.py
similarity index 93%
rename from auto_round/modelling/__init__.py
rename to auto_round/modeling/__init__.py
index 0d2740cdb..14a492441 100644
--- a/auto_round/modelling/__init__.py
+++ b/auto_round/modeling/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025 Intel Corporation
+# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/auto_round/modeling/fused_moe/__init__.py b/auto_round/modeling/fused_moe/__init__.py
new file mode 100644
index 000000000..1b50f0e41
--- /dev/null
+++ b/auto_round/modeling/fused_moe/__init__.py
@@ -0,0 +1,193 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+from auto_round import envs
+import transformers
+import importlib
+from packaging import version
+from transformers import AutoConfig
+
+from auto_round.logger import logger
+import torch
+
+MODEL_CONFIG = {
+ "qwen3_moe": {
+ "min_transformers_version": "5.0.0",
+ "max_transformers_version": None,
+ "checkpoint_mapping": [],
+ "block_patch": [
+ (
+ "transformers.models.qwen3_moe.modeling_qwen3_moe.Qwen3MoeSparseMoeBlock",
+ "auto_round.modeling.fused_moe.qwen3_moe.LinearQwen3MoeSparseMoeBlock",
+ )
+ ],
+ },
+ "glm4_moe_lite": {
+ "min_transformers_version": "5.0.0",
+ "max_transformers_version": None,
+ "checkpoint_mapping": [],
+ "block_patch": [
+ (
+ "transformers.models.glm4_moe_lite.modeling_glm4_moe_lite.Glm4MoeLiteMoE",
+ "auto_round.modeling.fused_moe.glm_moe_light.LinearGlm4MoeLiteMoE",
+ )
+ ],
+ },
+}
+
+
+def get_checkpoint_conversion_mapping_ar(model_type):
+ from transformers import conversion_mapping
+
+ if not hasattr(conversion_mapping, "orig_get_checkpoint_conversion_mapping"):
+ conversion_mapping.orig_get_checkpoint_conversion_mapping = conversion_mapping.get_checkpoint_conversion_mapping
+
+ cfg = MODEL_CONFIG.get(model_type)
+ if cfg:
+ return cfg.get("checkpoint_mapping", [])
+
+ from transformers import conversion_mapping
+
+ return conversion_mapping.orig_get_checkpoint_conversion_mapping(model_type)
+
+
+def get_file_path_via_model_name(model_or_path: str, file_name):
+ from huggingface_hub import hf_hub_download
+
+ # 1) local folder
+ if os.path.isdir(model_or_path):
+ index_path = os.path.join(model_or_path, file_name)
+
+ # 2) HF model name
+ elif not envs.AR_USE_MODELSCOPE:
+ index_path = hf_hub_download(
+ repo_id=model_or_path,
+ filename=file_name,
+ repo_type="model",
+ )
+ elif envs.AR_USE_MODELSCOPE:
+ from modelscope import snapshot_download # pylint: disable=E0401
+
+ # ModelSCOPE is different, it returns the folder path
+ folder = snapshot_download(model_or_path, allow_patterns=[file_name])
+ index_path = os.path.join(folder, file_name)
+ else:
+ index_path = None
+
+ return index_path
+
+
+def pre_check_config(model_name: str | torch.nn.Module):
+ if isinstance(model_name, str):
+ config = AutoConfig.from_pretrained(model_name)
+ elif isinstance(model_name, torch.nn.Module):
+ config = getattr(model_name, "config", None)
+ if config is None:
+ return False
+
+ model_type = getattr(config, "model_type", None)
+ if model_type is None or model_type not in MODEL_CONFIG:
+ return False
+
+ cfg = MODEL_CONFIG[model_type]
+
+ min_ver = cfg.get("min_transformers_version")
+ max_ver = cfg.get("max_transformers_version")
+ tf_ver = version.parse(transformers.__version__)
+ if min_ver and tf_ver < version.parse(min_ver):
+ return False
+ if max_ver and tf_ver > version.parse(max_ver):
+ return False
+ try:
+ file_path = get_file_path_via_model_name(model_name, "model.safetensors.index.json")
+ if os.path.exists(file_path):
+ import json
+
+ with open(file_path, "r") as f:
+ index_data = json.load(f)
+ model_keys = list(index_data.get("weight_map", {}).keys())
+ for key in model_keys:
+ if "gate_up_proj" in key:
+ return False
+ except:
+ return True
+ return True
+
+
+# This is for model checkpoint with linear definition
+def apply_model_monkey_patches(model_name: str) -> bool:
+ res = pre_check_config(model_name)
+ if not res:
+ return False
+ # patch blocks
+ config = AutoConfig.from_pretrained(model_name)
+ model_type = getattr(config, "model_type")
+
+ cfg = MODEL_CONFIG[model_type]
+ for orig_path, custom_path in cfg.get("block_patch", []):
+ orig_module_path, orig_class_name = orig_path.rsplit(".", 1)
+ custom_module_path, custom_class_name = custom_path.rsplit(".", 1)
+
+ try:
+ orig_module = importlib.import_module(orig_module_path)
+ custom_module = importlib.import_module(custom_module_path)
+ custom_class = getattr(custom_module, custom_class_name)
+ setattr(orig_module, orig_class_name, custom_class)
+
+ if version.parse(transformers.__version__) >= version.parse("5.0.0"):
+ from transformers import conversion_mapping
+
+ if not hasattr(conversion_mapping, "orig_get_checkpoint_conversion_mapping"):
+ conversion_mapping.orig_get_checkpoint_conversion_mapping = (
+ conversion_mapping.get_checkpoint_conversion_mapping
+ )
+
+ conversion_mapping.get_checkpoint_conversion_mapping = get_checkpoint_conversion_mapping_ar
+ transformers.modeling_utils.get_checkpoint_conversion_mapping = get_checkpoint_conversion_mapping_ar
+ logger.info(f"Patched {orig_path} -> {custom_path}")
+ return True
+
+ except Exception as e:
+ logger.warning(f"Failed to patch {orig_path}: {e}")
+ return False
+ return False
+
+
+def apply_modeling_patch(model: torch.nn.Module) -> bool:
+ res = pre_check_config(model)
+ if not res:
+ return False
+ model_type = getattr(model.config, "model_type")
+ cfg = MODEL_CONFIG[model_type]
+ # patch blocks
+ for orig_path, custom_path in cfg.get("block_patch", []):
+ orig_module_path, orig_class_name = orig_path.rsplit(".", 1)
+ custom_module_path, custom_class_name = custom_path.rsplit(".", 1)
+ try:
+ orig_module = importlib.import_module(orig_module_path)
+ custom_module = importlib.import_module(custom_module_path)
+ custom_class = getattr(custom_module, custom_class_name)
+ orig_class = getattr(orig_module, orig_class_name)
+ names = []
+ for n, m in model.named_modules():
+ if isinstance(m, orig_class):
+ names.append(n)
+ for n in names:
+ model.set_submodule(n, custom_class(model.config), True)
+ logger.info(f"Patched module: {orig_path} -> {custom_path}")
+ return True
+ except Exception as e:
+ logger.warning(f"Failed to patch {orig_path}: {e}")
+ return False
+ return False
diff --git a/auto_round/modeling/fused_moe/glm_moe_light.py b/auto_round/modeling/fused_moe/glm_moe_light.py
new file mode 100644
index 000000000..c9506ecb4
--- /dev/null
+++ b/auto_round/modeling/fused_moe/glm_moe_light.py
@@ -0,0 +1,107 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+import torch.nn as nn
+
+
+class LinearGlm4MoeLiteMoE(nn.Module):
+ """
+ A mixed expert module containing shared experts.
+ """
+
+ def __init__(self, config):
+ super().__init__()
+ self.config = config
+ self.num_experts = config.num_local_experts
+ from transformers.models.glm4_moe_lite.modeling_glm4_moe_lite import Glm4MoeLiteMLP, Glm4MoeLiteTopkRouter
+
+ self.experts = nn.ModuleList(
+ [Glm4MoeLiteMLP(config, intermediate_size=config.moe_intermediate_size) for _ in range(self.num_experts)]
+ )
+
+ self.gate = Glm4MoeLiteTopkRouter(config)
+ self.shared_experts = Glm4MoeLiteMLP(
+ config=config, intermediate_size=config.moe_intermediate_size * config.n_shared_experts
+ )
+ self.n_routed_experts = config.n_routed_experts
+ self.n_group = config.n_group
+ self.topk_group = config.topk_group
+ self.norm_topk_prob = config.norm_topk_prob
+ self.routed_scaling_factor = config.routed_scaling_factor
+ self.top_k = config.num_experts_per_tok
+
+ def experts_forward(
+ self,
+ hidden_states: torch.Tensor,
+ top_k_index: torch.Tensor,
+ top_k_weights: torch.Tensor,
+ ) -> torch.Tensor:
+ """ """
+ final_hidden_states = torch.zeros_like(hidden_states)
+ with torch.no_grad():
+ expert_mask = torch.nn.functional.one_hot(top_k_index, num_classes=self.num_experts)
+ expert_mask = expert_mask.permute(2, 1, 0)
+ expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()
+
+ for expert_idx in expert_hit:
+ expert_idx = expert_idx[0]
+ if expert_idx == self.num_experts:
+ continue
+ top_k_pos, token_idx = torch.where(expert_mask[expert_idx])
+ current_state = hidden_states[token_idx]
+ expert_layer = self.experts[expert_idx]
+ current_hidden_states = expert_layer(current_state)
+ # gate, up = nn.functional.linear(current_state, self.gate_up_proj[expert_idx]).chunk(2, dim=-1)
+ # current_hidden_states = self.act_fn(gate) * up
+ # current_hidden_states = nn.functional.linear(current_hidden_states, self.down_proj[expert_idx])
+ current_hidden_states = current_hidden_states * top_k_weights[token_idx, top_k_pos, None]
+ final_hidden_states.index_add_(0, token_idx, current_hidden_states.to(final_hidden_states.dtype))
+
+ return final_hidden_states
+
+ def route_tokens_to_experts(self, router_logits):
+ router_logits = router_logits.sigmoid()
+ router_logits_for_choice = router_logits + self.gate.e_score_correction_bias
+ group_scores = (
+ router_logits_for_choice.view(-1, self.n_group, self.n_routed_experts // self.n_group)
+ .topk(2, dim=-1)[0]
+ .sum(dim=-1)
+ )
+ group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]
+ group_mask = torch.zeros_like(group_scores)
+ group_mask.scatter_(1, group_idx, 1)
+ score_mask = (
+ group_mask.unsqueeze(-1)
+ .expand(-1, self.n_group, self.n_routed_experts // self.n_group)
+ .reshape(-1, self.n_routed_experts)
+ )
+ scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), 0.0)
+ topk_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1]
+ topk_weights = router_logits.gather(1, topk_indices)
+ if self.norm_topk_prob:
+ denominator = topk_weights.sum(dim=-1, keepdim=True) + 1e-20
+ topk_weights /= denominator
+ topk_weights = topk_weights * self.routed_scaling_factor
+ return topk_indices, topk_weights
+
+ def forward(self, hidden_states):
+ residuals = hidden_states
+ orig_shape = hidden_states.shape
+ router_logits = self.gate(hidden_states)
+ topk_indices, topk_weights = self.route_tokens_to_experts(router_logits)
+ hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
+ hidden_states = self.experts_forward(hidden_states, topk_indices, topk_weights).view(*orig_shape)
+ hidden_states = hidden_states + self.shared_experts(residuals)
+ return hidden_states
diff --git a/auto_round/modeling/fused_moe/qwen3_moe.py b/auto_round/modeling/fused_moe/qwen3_moe.py
new file mode 100644
index 000000000..02e1bc62a
--- /dev/null
+++ b/auto_round/modeling/fused_moe/qwen3_moe.py
@@ -0,0 +1,73 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+
+
+class LinearQwen3MoeSparseMoeBlock(nn.Module):
+ def __init__(self, config):
+ super().__init__()
+ from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeMLP
+
+ self.num_experts = config.num_experts
+ self.top_k = config.num_experts_per_tok
+ self.norm_topk_prob = config.norm_topk_prob
+
+ # gating
+ self.gate = nn.Linear(config.hidden_size, config.num_experts, bias=False)
+ self.experts = nn.ModuleList(
+ [Qwen3MoeMLP(config, intermediate_size=config.moe_intermediate_size) for _ in range(self.num_experts)]
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ """ """
+ batch_size, sequence_length, hidden_dim = hidden_states.shape
+ hidden_states = hidden_states.view(-1, hidden_dim)
+ # router_logits: (batch * sequence_length, n_experts)
+ router_logits = self.gate(hidden_states)
+
+ routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
+ routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)
+ if self.norm_topk_prob: # only diff with mixtral sparse moe block!
+ routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
+ # we cast back to the input dtype
+ routing_weights = routing_weights.to(hidden_states.dtype)
+
+ final_hidden_states = torch.zeros(
+ (batch_size * sequence_length, hidden_dim), dtype=hidden_states.dtype, device=hidden_states.device
+ )
+
+ # One hot encode the selected experts to create an expert mask
+ # this will be used to easily index which expert is going to be solicited
+ expert_mask = torch.nn.functional.one_hot(selected_experts, num_classes=self.num_experts).permute(2, 1, 0)
+
+ # Loop over all available experts in the model and perform the computation on each expert
+ expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()
+ for expert_idx in expert_hit:
+ expert_layer = self.experts[expert_idx]
+ idx, top_x = torch.where(expert_mask[expert_idx].squeeze(0))
+
+ # Index the correct hidden states and compute the expert hidden state for
+ # the current expert. We need to make sure to multiply the output hidden
+ # states by `routing_weights` on the corresponding tokens (top-1 and top-2)
+ current_state = hidden_states[None, top_x].reshape(-1, hidden_dim)
+ current_hidden_states = expert_layer(current_state) * routing_weights[top_x, idx, None]
+
+ # However `index_add_` only support torch tensors for indexing so we'll use
+ # the `top_x` tensor here.
+ final_hidden_states.index_add_(0, top_x, current_hidden_states.to(hidden_states.dtype))
+ final_hidden_states = final_hidden_states.reshape(batch_size, sequence_length, hidden_dim)
+ return final_hidden_states
diff --git a/auto_round/modeling/unfused_moe/__init__.py b/auto_round/modeling/unfused_moe/__init__.py
new file mode 100644
index 000000000..14a492441
--- /dev/null
+++ b/auto_round/modeling/unfused_moe/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/auto_round/modelling/deepseek_v2.py b/auto_round/modeling/unfused_moe/deepseek_v2.py
similarity index 98%
rename from auto_round/modelling/deepseek_v2.py
rename to auto_round/modeling/unfused_moe/deepseek_v2.py
index 7caff2696..0f026aac4 100644
--- a/auto_round/modelling/deepseek_v2.py
+++ b/auto_round/modeling/unfused_moe/deepseek_v2.py
@@ -18,7 +18,6 @@ from functools import partial
from typing import Callable, Optional
import torch
-from transformers.cache_utils import Cache
from transformers.modeling_rope_utils import dynamic_rope_update
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
from transformers.models.deepseek_v2.configuration_deepseek_v2 import DeepseekV2Config
@@ -26,7 +25,7 @@ from transformers.models.deepseek_v2.modeling_deepseek_v2 import (
eager_attention_forward,
)
-from auto_round.modelling.replace_modules import ReplacementModuleBase
+from auto_round.modeling.unfused_moe.replace_modules import ReplacementModuleBase
from auto_round.utils import is_hpex_available
diff --git a/auto_round/modelling/gpt_oss.py b/auto_round/modeling/unfused_moe/gpt_oss.py
similarity index 96%
rename from auto_round/modelling/gpt_oss.py
rename to auto_round/modeling/unfused_moe/gpt_oss.py
index a71b5b610..96caeb71d 100644
--- a/auto_round/modelling/gpt_oss.py
+++ b/auto_round/modeling/unfused_moe/gpt_oss.py
@@ -27,9 +27,9 @@ else:
from transformers.models.gpt_oss.configuration_gpt_oss import GptOssConfig
from transformers.models.gpt_oss.modeling_gpt_oss import GptOssMLP
-from auto_round.modelling.replace_modules import ReplacementModuleBase
-from auto_round.modelling.utils import _update_parameter
-from auto_round.utils import LazyImport, clear_memory, logger, unsupported_meta_device
+from auto_round.modeling.unfused_moe.replace_modules import ReplacementModuleBase
+from auto_round.modeling.unfused_moe.utils import _update_parameter
+from auto_round.utils import clear_memory, unsupported_meta_device
class GPTOssSingleExpert(nn.Module):
diff --git a/auto_round/modelling/llama4.py b/auto_round/modeling/unfused_moe/llama4.py
similarity index 96%
rename from auto_round/modelling/llama4.py
rename to auto_round/modeling/unfused_moe/llama4.py
index 7057f777d..5e6adc274 100644
--- a/auto_round/modelling/llama4.py
+++ b/auto_round/modeling/unfused_moe/llama4.py
@@ -21,10 +21,10 @@ if transformers_version < version.parse("5.0.0"):
from transformers.modeling_utils import no_init_weights
else:
from transformers.initialization import no_init_weights
-from transformers.models.llama4.modeling_llama4 import Llama4Config, Llama4TextMLP, Llama4TextMoe
+from transformers.models.llama4.modeling_llama4 import Llama4Config, Llama4TextMLP
-from auto_round.modelling.replace_modules import ReplacementModuleBase
-from auto_round.modelling.utils import _update_parameter
+from auto_round.modeling.unfused_moe.replace_modules import ReplacementModuleBase
+from auto_round.modeling.unfused_moe.utils import _update_parameter
from auto_round.utils import clear_memory, unsupported_meta_device
diff --git a/auto_round/modelling/qwen3_vl_moe.py b/auto_round/modeling/unfused_moe/qwen3_vl_moe.py
similarity index 96%
rename from auto_round/modelling/qwen3_vl_moe.py
rename to auto_round/modeling/unfused_moe/qwen3_vl_moe.py
index 609b1df74..1c0d62a29 100644
--- a/auto_round/modelling/qwen3_vl_moe.py
+++ b/auto_round/modeling/unfused_moe/qwen3_vl_moe.py
@@ -15,16 +15,14 @@
import torch
import transformers
from packaging import version
-from torch import nn
-from transformers.activations import ACT2FN
-from auto_round.modelling.replace_modules import ReplacementModuleBase
-from auto_round.utils import clear_memory, logger, unsupported_meta_device
+from auto_round.modeling.unfused_moe.replace_modules import ReplacementModuleBase
+from auto_round.utils import clear_memory, unsupported_meta_device
transformers_version = version.parse(transformers.__version__)
from typing import TYPE_CHECKING
-from auto_round.modelling.utils import _update_parameter
+from auto_round.modeling.unfused_moe.utils import _update_parameter
if TYPE_CHECKING:
from transformers import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
diff --git a/auto_round/modelling/replace_modules.py b/auto_round/modeling/unfused_moe/replace_modules.py
similarity index 98%
rename from auto_round/modelling/replace_modules.py
rename to auto_round/modeling/unfused_moe/replace_modules.py
index d77ffca2d..c62345fa1 100644
--- a/auto_round/modelling/replace_modules.py
+++ b/auto_round/modeling/unfused_moe/replace_modules.py
@@ -23,10 +23,10 @@ from transformers import PreTrainedModel
from auto_round.utils import LazyImport, dump_mem_usage, dump_memory_usage_ctx, global_state, logger
BUILTIN_MODULES = {
- "Llama4TextMoe": LazyImport("auto_round.modelling.llama4"),
- "GptOssMLP": LazyImport("auto_round.modelling.gpt_oss"),
- "Qwen3VLMoeTextSparseMoeBlock": LazyImport("auto_round.modelling.qwen3_vl_moe"),
- "DeepseekV2Attention": LazyImport("auto_round.modelling.deepseek_v2"),
+ "Llama4TextMoe": LazyImport("auto_round.modeling.unfused_moe.llama4"),
+ "GptOssMLP": LazyImport("auto_round.modeling.unfused_moe.gpt_oss"),
+ "Qwen3VLMoeTextSparseMoeBlock": LazyImport("auto_round.modeling.unfused_moe.qwen3_vl_moe"),
+ "DeepseekV2Attention": LazyImport("auto_round.modeling.unfused_moe.deepseek_v2"),
}
diff --git a/auto_round/modelling/utils.py b/auto_round/modeling/unfused_moe/utils.py
similarity index 100%
rename from auto_round/modelling/utils.py
rename to auto_round/modeling/unfused_moe/utils.py
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index f3621f913..da02b43b0 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -17,10 +17,9 @@ from typing import Any, Callable
import torch
-import auto_round.modelling as auto_round_modelling
from auto_round.formats import OutputFormat
-from auto_round.modelling.replace_modules import apply_replacements, materialize_model_, release_original_module_
-from auto_round.utils import LazyImport, logger, unsupported_meta_device
+from auto_round.modeling.unfused_moe.replace_modules import apply_replacements, release_original_module_
+from auto_round.utils import logger
mllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_size
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 012415507..8f18f7cad 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -1390,6 +1390,14 @@ def is_moe_model(model: torch.nn.Module) -> bool:
return False
+def is_moe_model_via_config(config) -> bool:
+ if hasattr(config, "to_dict"):
+ for key in config.to_dict().keys():
+ if "moe" in key or "expert" in key:
+ return True
+ return False
+
+
def to_dtype(input, dtype=torch.float32):
"""Moves input data to the specified data type.
diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.py
index 8872edccb..f98a5b958 100644
--- a/test/test_cpu/models/test_moe_model.py
+++ b/test/test_cpu/models/test_moe_model.py
@@ -99,7 +99,7 @@ def test_gptoss(setup_gpt_oss, scheme):
# Ensure the quantized model is not None
assert quantized_model is not None, "Quantized model should not be None."
from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear
- from auto_round.modelling.gpt_oss import GPTOssSingleExpert
+ from auto_round.modeling.unfused_moe.gpt_oss import GPTOssSingleExpert
single_expert_cnt = count_modules_by_type(quantized_model, GPTOssSingleExpert)
quant_linear_cnt = count_modules_by_type(quantized_model, QuantLinear)
diff --git a/test/test_cuda/models/test_moe_model.py b/test/test_cuda/models/test_moe_model.py
index 0cd2bd5e9..442d559ea 100644
--- a/test/test_cuda/models/test_moe_model.py
+++ b/test/test_cuda/models/test_moe_model.py
@@ -9,8 +9,7 @@ from transformers.models.qwen3.modeling_qwen3 import Qwen3Config, Qwen3ForCausal
from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeForConditionalGeneration
from auto_round import AutoRound
-from auto_round.modelling.replace_modules import ReplacementModuleBase
-from auto_round.utils import LazyImport, logger, unsupported_meta_device
+from auto_round.modeling.unfused_moe.replace_modules import ReplacementModuleBase
@pytest.fixture
@@ -211,7 +210,7 @@ def has_module(model: torch.nn.Module, module_name: str) -> bool:
def test_register_module_out_of_tree_base():
- from auto_round.modelling.replace_modules import ReplacementModuleBase
+ from auto_round.modeling.unfused_moe.replace_modules import ReplacementModuleBase
for name, subclass in ReplacementModuleBase._replacement_registry.items():
if name == "Qwen3MLP":