Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Refactor module replacement (#1153)

Yi Liu87d5580aeca4
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 7bdb684ce..d9f1e6054 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -64,7 +64,7 @@ from auto_round.schemes import (
preset_name_to_scheme,
)
from auto_round.sign_sgd import SignSGD
-from auto_round.special_model_handler import _handle_moe_model
+from auto_round.special_model_handler import update_module
from auto_round.utils import (
INNER_SUPPORTED_LAYER_TYPES,
SUPPORTED_DTYPES,
@@ -1348,7 +1348,6 @@ class BaseCompressor(object):
memory_monitor.log_summary()
pbar.update(1)
-
pbar.close()
# Process remaining layers not in blocks
for name in all_to_quantized_module_names:
@@ -1400,7 +1399,7 @@ class BaseCompressor(object):
formats = self.formats if hasattr(self, "formats") else None
# It is best to modify the model structure in the quantize function and check the format,
# because it may cause the gguf format to not be exported normally.
- self.model = _handle_moe_model(self.model, formats=formats)
+ self.model = update_module(self.model, formats=formats)
# Temporary names must be assigned after handle_moe_model;
# placing them earlier would cause them to be removed when the module is replaced.
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 450182c92..fc5d5e7ba 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -31,7 +31,7 @@ from auto_round.inference.backend import (
from auto_round.inference.utils import _expand_regex_config
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
-from auto_round.special_model_handler import _handle_moe_model
+from auto_round.special_model_handler import update_module
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
check_start_with_block_name,
@@ -593,7 +593,7 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.M
packing_format = "auto_round:auto_gptq"
# Preprocess model before replace layers
- model = _handle_moe_model(model)
+ model = update_module(model)
# Replace layers with quantized versions
layer_configs = get_layer_config(model, quantization_config)
diff --git a/auto_round/modelling/gpt_oss.py b/auto_round/modelling/gpt_oss.py
index 9fe183779..06506ef27 100644
--- a/auto_round/modelling/gpt_oss.py
+++ b/auto_round/modelling/gpt_oss.py
@@ -19,9 +19,8 @@ from transformers.modeling_utils import no_init_weights as skip_weights_initiali
from transformers.models.gpt_oss.configuration_gpt_oss import GptOssConfig
from transformers.models.gpt_oss.modeling_gpt_oss import GptOssMLP
-from auto_round.utils import unsupported_meta_device
-
-__all__ = ["get_replacement_info"]
+from auto_round.modelling.replace_modules import ReplacementModuleBase
+from auto_round.utils import LazyImport, logger, unsupported_meta_device
def _update_parameter(
@@ -54,13 +53,13 @@ class GPTOssSingleExpert(nn.Module):
return self.down_proj(act)
-class SequentialGPTOSSMoE(nn.Module):
+class SequentialGPTOSSMoE(ReplacementModuleBase):
"""
Replaces GPT-OSS fused-expert MoE with per-expert `GPTOssSingleExpert` modules.
Copies weights from fused tensors and reuses the original router and optional shared_expert.
"""
- def __init__(self, config: GptOssConfig, original: GptOssMLP):
+ def __init__(self, original: GptOssMLP, config: GptOssConfig):
super().__init__()
hidden_size = config.hidden_size
intermediate_size = config.intermediate_size
@@ -123,10 +122,17 @@ class SequentialGPTOSSMoE(nn.Module):
final_hidden_states += expert_output
return final_hidden_states.view(B, T, H), router_scores.view(B * T, -1)
-
-def get_replacement_info(config):
- return (
- SequentialGPTOSSMoE,
- config.get_text_config(),
- GptOssMLP.__name__,
- )
+ @classmethod
+ def original_module_class(cls) -> str:
+ """Return the class name of the module this replaces."""
+ return "GptOssMLP"
+
+ @classmethod
+ def from_original(
+ cls,
+ original: GptOssMLP,
+ config: GptOssConfig,
+ **kwargs,
+ ) -> "SequentialGPTOSSMoE":
+ """Create an instance from the original module."""
+ return cls(original, config)
diff --git a/auto_round/modelling/llama4.py b/auto_round/modelling/llama4.py
index cb64fdc68..811488709 100644
--- a/auto_round/modelling/llama4.py
+++ b/auto_round/modelling/llama4.py
@@ -12,14 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Note: adapted from # https://github.com/vllm-project/llm-compressor/blob/main/src/llmcompressor/modeling/llama4.py
-
-__all__ = ["get_replacement_info"]
-
-
import torch
from transformers.modeling_utils import no_init_weights
-from transformers.models.llama4.modeling_llama4 import Llama4TextMLP
+from transformers.models.llama4.modeling_llama4 import Llama4Config, Llama4TextMLP, Llama4TextMoe
+from auto_round.modelling.replace_modules import ReplacementModuleBase
from auto_round.utils import unsupported_meta_device
@@ -43,9 +40,10 @@ class SequentialLlama4TextExperts(torch.nn.ModuleList):
self[i].down_proj.weight.data.copy_(down.t())
-class SequentialLlama4TextMoe(torch.nn.Module):
- def __init__(self, config, original):
+class SequentialLlama4TextMoe(ReplacementModuleBase):
+ def __init__(self, original, config):
super().__init__()
+ config = config.text_config
self.top_k = config.num_experts_per_tok
self.hidden_dim = config.hidden_size
self.num_experts = config.num_local_experts
@@ -76,6 +74,17 @@ class SequentialLlama4TextMoe(torch.nn.Module):
return out, router_logits
-
-def get_replacement_info(config):
- return SequentialLlama4TextMoe, config.get_text_config(), "Llama4TextMoe"
+ @classmethod
+ def original_module_class(cls) -> str:
+ """Return the class name of the module this replaces."""
+ return "Llama4TextMoe"
+
+ @classmethod
+ def from_original(
+ cls,
+ original: torch.nn.Module,
+ config: Llama4Config,
+ **kwargs,
+ ) -> "SequentialLlama4TextMoe":
+ """Create an instance from the original module."""
+ return cls(original, config)
diff --git a/auto_round/modelling/qwen3_vl_moe.py b/auto_round/modelling/qwen3_vl_moe.py
index 9a47871e5..dbed42b1a 100644
--- a/auto_round/modelling/qwen3_vl_moe.py
+++ b/auto_round/modelling/qwen3_vl_moe.py
@@ -18,6 +18,7 @@ from packaging import version
from torch import nn
from transformers.activations import ACT2FN
+from auto_round.modelling.replace_modules import ReplacementModuleBase
from auto_round.utils import logger, unsupported_meta_device
transformers_version = version.parse(transformers.__version__)
@@ -41,7 +42,7 @@ def _update_parameter(
# Adapted from https://github.com/vllm-project/llm-compressor/blob/main/src/llmcompressor/modeling/qwen3_vl_moe.py
-class LinearQwen3VLMoeTextSparseMoeBlock(torch.nn.Module):
+class LinearQwen3VLMoeTextSparseMoeBlock(ReplacementModuleBase):
"""
Calibration version of Qwen3VLMoeTextSparseMoeBlock that sends all tokens to all
experts.
@@ -117,6 +118,21 @@ class LinearQwen3VLMoeTextSparseMoeBlock(torch.nn.Module):
else:
return next_states
+ @classmethod
+ def original_module_class(cls) -> str:
+ """Return the class name of the module this replaces."""
+ return "Qwen3VLMoeTextSparseMoeBlock"
+
+ @classmethod
+ def from_original(
+ cls,
+ original: "Qwen3VLMoeTextSparseMoeBlock",
+ config: "Qwen3VLMoeConfig",
+ **kwargs,
+ ) -> "LinearQwen3VLMoeTextSparseMoeBlock":
+ """Create an instance from the original module."""
+ return cls(original, config)
+
class SequentialQwen3VLMoeTextExperts(torch.nn.ModuleList):
def __init__(self, config, original):
@@ -140,7 +156,3 @@ class SequentialQwen3VLMoeTextExperts(torch.nn.ModuleList):
_update_parameter(self[i].gate_proj, "weight", gate_proj.t().contiguous())
_update_parameter(self[i].up_proj, "weight", up_proj.t().contiguous())
_update_parameter(self[i].down_proj, "weight", down.t().contiguous())
-
-
-def get_replacement_info(config):
- return (LinearQwen3VLMoeTextSparseMoeBlock, config, "Qwen3VLMoeTextSparseMoeBlock")
diff --git a/auto_round/modelling/replace_modules.py b/auto_round/modelling/replace_modules.py
new file mode 100644
index 000000000..c8e34954f
--- /dev/null
+++ b/auto_round/modelling/replace_modules.py
@@ -0,0 +1,180 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from typing import Dict, Type
+
+import torch
+from tqdm import tqdm
+from transformers import PreTrainedModel
+
+from auto_round.utils import LazyImport, logger
+
+BUILTIN_MODULES = {
+ "Llama4TextMoe": LazyImport("auto_round.modelling.llama4"),
+ "GptOssMLP": LazyImport("auto_round.modelling.gpt_oss"),
+ "Qwen3VLMoeTextSparseMoeBlock": LazyImport("auto_round.modelling.qwen3_vl_moe"),
+}
+
+
+def _import_required_replacements(model: torch.nn.Module) -> None:
+ """Scan model and trigger lazy imports for registered replacement modules."""
+ imported = set()
+
+ for _, module in model.named_modules():
+ class_name = module.__class__.__name__
+
+ if class_name in BUILTIN_MODULES and class_name not in imported:
+ # Trigger import by accessing the LazyImport object
+ _ = BUILTIN_MODULES[class_name].__name__ # or any attribute
+ imported.add(class_name)
+ logger.debug(f"Loaded replacement module for {class_name}")
+
+
+class ReplacementModuleBase(ABC, torch.nn.Module):
+ """
+ Abstract base class for module replacement during calibration phase.
+
+ Replacement modules replace original modules to ensure all components
+ receive data for proper quantization statistics.
+
+ Subclasses must:
+ 1. Implement `original_module_class()` to return the target module class name
+ 2. Implement `__init__()` with signature:
+ (self, original, config)
+ """
+
+ # Registry: module class name -> replacement module class
+ _replacement_registry: Dict[str, Type["ReplacementModuleBase"]] = {}
+
+ def __init_subclass__(cls, **kwargs):
+ """Automatically register subclasses in the replacement registry."""
+ super().__init_subclass__(**kwargs)
+
+ # Only register if it's a concrete implementation (not ABC)
+ if not getattr(cls, "__abstractmethods__", None):
+ if cls.original_module_class() is None:
+ raise TypeError(
+ f"{cls.__name__} must implement 'original_module_class()' class method "
+ "to return the name of the module class it replaces"
+ )
+
+ if cls.original_module_class() in cls._replacement_registry:
+ existing = cls._replacement_registry[cls.original_module_class()]
+ raise ValueError(
+ f"Module '{cls.original_module_class()}' already registered to "
+ f"{existing.__name__}. Cannot register {cls.__name__}."
+ )
+
+ cls._replacement_registry[cls.original_module_class()] = cls
+ logger.trace(f"Registered {cls.__name__} for replacing {cls.original_module_class()}")
+
+ @classmethod
+ def get_replacement_class(cls, module_class_name: str) -> Type["ReplacementModuleBase"]:
+ """Get replacement class for a given module class name."""
+ return cls._replacement_registry.get(module_class_name)
+
+ @classmethod
+ def is_registered(cls, module_class_name: str) -> bool:
+ """Check if a module class has a replacement implementation."""
+ return module_class_name in cls._replacement_registry
+
+ @classmethod
+ def is_to_be_replaced(
+ cls,
+ original: torch.nn.Module,
+ module_class_name: str,
+ ) -> bool:
+ """Determine if the given module should be replaced.
+
+ Users can extend this method to add custom logic for replacement.
+ """
+ return cls.is_registered(module_class_name)
+
+ @classmethod
+ def get_registered_modules(cls) -> list:
+ """Get list of all registered module class names."""
+ return list(cls._replacement_registry.keys())
+
+ @classmethod
+ @abstractmethod
+ def original_module_class(cls) -> str:
+ """Return the class name of the module this replaces."""
+ pass
+
+ @classmethod
+ @abstractmethod
+ def from_original(
+ cls,
+ original: torch.nn.Module,
+ config,
+ ) -> "ReplacementModuleBase":
+ """Create replacement module from original module."""
+ pass
+
+
+# Note: adapted from llm-compressor
+# https://github.com/vllm-project/llm-compressor/blob/main/src/llmcompressor/modeling/moe_context.py
+
+
+def apply_replacements(
+ model: torch.nn.Module,
+) -> torch.nn.Module:
+ """
+ Function to apply module replacements to a model.
+
+ This scans all modules in the model and replaces any registered modules with their
+ replacement equivalents. Non-permanent modules are tracked for later restoration.
+
+ The model is modified in-place, so the same model object should be used.
+
+ Args:
+ model: The model to apply module replacement to (modified in-place).
+
+ Returns:
+ The model with modules replaced.
+ """
+ _import_required_replacements(model)
+ replaced = {}
+
+ # Step 1: Collect all modules that need replacement
+ logger.debug("Scanning for modules to replace")
+ modules_to_replace = []
+ for name, module in model.named_modules():
+ # skip replaced modules
+ if isinstance(module, ReplacementModuleBase):
+ continue
+ class_name = module.__class__.__name__
+ if ReplacementModuleBase.is_to_be_replaced(module, class_name):
+ modules_to_replace.append((name, module, class_name))
+
+ # Step 2: Replace modules
+ if modules_to_replace:
+ logger.info(f"Found {len(modules_to_replace)} modules to replace")
+ for name, module, class_name in tqdm(modules_to_replace, desc="Replacing modules"):
+ replacement_cls = ReplacementModuleBase.get_replacement_class(class_name)
+ replacement = replacement_cls.from_original(
+ module,
+ model.config,
+ )
+ model.set_submodule(name, replacement)
+ replaced[name] = (module, replacement)
+ else:
+ logger.debug("No modules found for replacement")
+
+ # Log what was replaced
+ if replaced:
+ logger.info(f"Replaced {len(replaced)} modules")
+
+ return model
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index eb8fa3e26..3bb60bfb0 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -15,6 +15,7 @@ import torch
import auto_round.modelling as auto_round_modelling
from auto_round.formats import OutputFormat
+from auto_round.modelling.replace_modules import apply_replacements
from auto_round.utils import LazyImport, logger, unsupported_meta_device
mllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_size
@@ -40,30 +41,9 @@ SPECIAL_SHARED_CACHE_KEYS = {
"Gemma3ForConditionalGeneration": ("position_embeddings_global", "position_embeddings_local")
}
SPECIAL_SHARED_CACHE_KEYS["MiniMaxText01ForCausalLM"] = ("slope_rate",)
-
-CONVERT_EXPERT_TO_LINEAR_MODELS = ["llama4", "gpt_oss", "qwen3_vl_moe"]
-
MISTRAL_3_2_MODELS = ["Mistral-Small-3.2", "Magistral-Small", "Devstral-Small"]
-def _get_moe_converter(config):
- # Dispatch table for model_type to replacement_info functions
- moe_converters = {
- "gpt_oss": LazyImport("auto_round.modelling.gpt_oss.get_replacement_info"),
- "llama4": LazyImport("auto_round.modelling.llama4.get_replacement_info"),
- "qwen3_vl_moe": LazyImport("auto_round.modelling.qwen3_vl_moe.get_replacement_info"),
- }
-
- # Retrieve the appropriate function based on model_type
- if config.model_type in moe_converters:
- return moe_converters[config.model_type](config)
- else:
- raise ValueError(
- f"Unsupported model_type '{config.model_type}'. "
- f"Currently, MoE converter only supports: {', '.join(moe_converters.keys())}."
- )
-
-
def _handle_special_model(model):
if hasattr(model, "config") and model.config.model_type == "deepseek_vl_v2":
from functools import partial
@@ -72,28 +52,11 @@ def _handle_special_model(model):
return model
-def _handle_moe_model(model, formats: list[OutputFormat] = None):
+def update_module(model, formats: list[OutputFormat] = None):
if formats is not None and any([format_.is_gguf() for format_ in formats]):
return model
- if hasattr(model.config, "model_type") and model.config.model_type in CONVERT_EXPERT_TO_LINEAR_MODELS:
- from tqdm import tqdm
- from auto_round.utils import clear_memory
-
- new_moe_class, convert_config, orig_cls_name = _get_moe_converter(model.config)
- if not unsupported_meta_device(model):
- model = model.to("cpu")
- clear_memory()
-
- for name, module in tqdm(model.named_modules(), desc="Converting model"):
- cls_name = module.__class__.__name__
- if cls_name == orig_cls_name:
- new_module = new_moe_class(config=convert_config, original=module)
- parent, child = name.rsplit(".", maxsplit=1)
- parent = model.get_submodule(parent)
- setattr(parent, child, new_module)
-
- return model
+ return apply_replacements(model)
def _get_deepseek_vl2_multimodal_block(model, quant_vision=False):
diff --git a/test/test_cpu/test_moe_model.py b/test/test_cpu/test_moe_model.py
index 79e54c4d1..f70238f68 100644
--- a/test/test_cpu/test_moe_model.py
+++ b/test/test_cpu/test_moe_model.py
@@ -26,8 +26,8 @@ def setup_llama4():
model_name = "/tf_dataset/auto_round/models/meta-llama/Llama-4-Scout-17B-16E-Instruct"
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
- config.vision_config.num_hidden_layers = 2 # Reduce layers for testing
- config.text_config.num_hidden_layers = 2
+ config.vision_config.num_hidden_layers = 1 # Reduce layers for testing
+ config.text_config.num_hidden_layers = 1
model = Llama4ForConditionalGeneration(config)
output_dir = "/tmp/test_quantized_llama4"
return model, tokenizer, output_dir, config
@@ -56,6 +56,7 @@ def quantize_model(model, tokenizer, output_dir, scheme, iters=0):
scheme=scheme,
nsamples=2,
iters=iters,
+ seqlen=32,
fp_layers="self_attn,router,lm_head,mlp.gate",
)
quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)
diff --git a/test/test_cuda/test_moe_model.py b/test/test_cuda/test_moe_model.py
index 1865519c9..8c21abbdf 100644
--- a/test/test_cuda/test_moe_model.py
+++ b/test/test_cuda/test_moe_model.py
@@ -2,11 +2,15 @@ import shutil
import pytest
import torch
+import torch.nn as nn
from transformers import AutoConfig, AutoProcessor, AutoTokenizer, Llama4ForConditionalGeneration
from transformers.models.gpt_oss.modeling_gpt_oss import GptOssForCausalLM
+from transformers.models.qwen3.modeling_qwen3 import Qwen3Config, Qwen3ForCausalLM, Qwen3MLP
from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeForConditionalGeneration
from auto_round import AutoRound
+from auto_round.modelling.replace_modules import ReplacementModuleBase
+from auto_round.utils import LazyImport, logger, unsupported_meta_device
@pytest.fixture
@@ -27,8 +31,8 @@ def setup_llama4():
model_name = "/dataset/Llama-4-Scout-17B-16E-Instruct"
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
- config.vision_config.num_hidden_layers = 2 # Reduce layers for testing
- config.text_config.num_hidden_layers = 2
+ config.vision_config.num_hidden_layers = 1 # Reduce layers for testing
+ config.text_config.num_hidden_layers = 1
model = Llama4ForConditionalGeneration(config)
output_dir = "test_quantized_llama4"
return model, tokenizer, output_dir, config
@@ -45,7 +49,7 @@ def setup_qwen3_vl_moe():
config.num_hidden_layers = 1 # Reduce layers for testing
processor = AutoProcessor.from_pretrained(model_name)
model = Qwen3VLMoeForConditionalGeneration(config)
- output_dir = "/tmp/test_quantized_qwen3_vl_moe"
+ output_dir = "test_quantized_qwen3_vl_moe"
return model, tokenizer, processor, output_dir, config
@@ -57,6 +61,7 @@ def quantize_model(model, tokenizer, output_dir, scheme, iters=0):
scheme=scheme,
nsamples=2,
iters=iters,
+ seqlen=32,
fp_layers="self_attn,router,lm_head,mlp.gate",
)
quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)
@@ -121,6 +126,47 @@ def test_llama4(setup_llama4):
shutil.rmtree(output_dir, ignore_errors=True)
+class NewQwen3MLP(ReplacementModuleBase):
+ def __init__(self, original: Qwen3MLP, config: Qwen3Config):
+ super().__init__()
+ self.config = config
+ self.hidden_size = config.hidden_size
+ self.intermediate_size = config.intermediate_size
+ self.new_gate_proj = original.gate_proj
+ self.new_up_proj = original.up_proj
+ self.new_down_proj = original.down_proj
+ self.act_fn = nn.GELU()
+
+ def forward(self, x):
+ gate = self.new_gate_proj(x)
+ up = self.new_up_proj(x)
+ gate_act = self.act_fn(gate)
+ act = gate_act * up
+ out = self.new_down_proj(act)
+ return out
+
+ @classmethod
+ def original_module_class(cls) -> str:
+ return "Qwen3MLP"
+
+ @classmethod
+ def from_original(cls, original: Qwen3MLP, config: Qwen3Config):
+ return cls(original, config)
+
+
+def setup_qwen3():
+ """Fixture to set up the qwen3 model and tokenizer."""
+ model_name = "/models/Qwen3-0.6B"
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ config = AutoConfig.from_pretrained(model_name)
+ config.num_hidden_layers = 2 # Reduce layers for testing
+ config.layer_types = config.layer_types[:2] # Reduce layers for testing
+ model = Qwen3ForCausalLM(config)
+ output_dir = "test_quantized_qwen3"
+ return model, tokenizer, output_dir, config
+
+
def test_qwen3_vl_moe_mxfp(setup_qwen3_vl_moe):
model, tokenizer, processor, output_dir, config = setup_qwen3_vl_moe
autoround = AutoRound(
@@ -142,6 +188,11 @@ def test_qwen3_vl_moe_mxfp(setup_qwen3_vl_moe):
if m.__class__.__name__ == "QuantLinear":
loaded_m = loaded_model.get_submodule(n)
assert (loaded_m.weight_packed == m.weight_packed).all()
+
+ inp = torch.randint(0, 100, (1, 64)).to("cuda")
+ with torch.inference_mode():
+ loaded_out = loaded_model(inp)
+
# test generation
tokenizer = AutoTokenizer.from_pretrained(output_dir)
text = "There is a girl who likes adventure,"
@@ -149,3 +200,34 @@ def test_qwen3_vl_moe_mxfp(setup_qwen3_vl_moe):
print(tokenizer.decode(loaded_model.generate(**inputs, max_new_tokens=50)[0]))
# clean the output directory after test
shutil.rmtree(output_dir, ignore_errors=True)
+
+
+def has_module(model: torch.nn.Module, module_name: str) -> bool:
+ for n, m in model.named_modules():
+ if module_name in n:
+ return True
+ return False
+
+
+def test_register_module_out_of_tree_base():
+ from auto_round.modelling.replace_modules import ReplacementModuleBase
+
+ for name, subclass in ReplacementModuleBase._replacement_registry.items():
+ if name == "Qwen3MLP":
+ assert subclass == NewQwen3MLP, "Qwen3MLP not registered correctly."
+
+
+def test_register_module_out_of_tree_model(setup_qwen3):
+ model, tokenizer, output_dir, config = setup_qwen3
+ quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4")
+ # Ensure the quantized model is not None
+ assert quantized_model is not None, "Quantized model should not be None."
+ check_module_names = ["new_gate_proj", "new_up_proj", "new_down_proj"]
+ for name in check_module_names:
+ assert has_module(quantized_model, name), f"Module {name} not found in quantized model."
+ loaded_model = Qwen3ForCausalLM.from_pretrained(output_dir)
+ quantized_model.to("cuda")
+
+ for name in check_module_names:
+ assert has_module(loaded_model, name), f"Module {name} not found in loaded model."
+ loaded_model.to("cuda")