PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Refactor module replacement (#1153)
Yi Liu87d5580aeca4
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 7bdb684ce..d9f1e6054 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -64,7 +64,7 @@ from auto_round.schemes import (preset_name_to_scheme,)from auto_round.sign_sgd import SignSGD-from auto_round.special_model_handler import _handle_moe_model+from auto_round.special_model_handler import update_modulefrom auto_round.utils import (INNER_SUPPORTED_LAYER_TYPES,SUPPORTED_DTYPES,@@ -1348,7 +1348,6 @@ class BaseCompressor(object):memory_monitor.log_summary()pbar.update(1)-pbar.close()# Process remaining layers not in blocksfor name in all_to_quantized_module_names:@@ -1400,7 +1399,7 @@ class BaseCompressor(object):formats = self.formats if hasattr(self, "formats") else None# It is best to modify the model structure in the quantize function and check the format,# because it may cause the gguf format to not be exported normally.- self.model = _handle_moe_model(self.model, formats=formats)+ self.model = update_module(self.model, formats=formats)# Temporary names must be assigned after handle_moe_model;# placing them earlier would cause them to be removed when the module is replaced.diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 450182c92..fc5d5e7ba 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -31,7 +31,7 @@ from auto_round.inference.backend import (from auto_round.inference.utils import _expand_regex_configfrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme-from auto_round.special_model_handler import _handle_moe_model+from auto_round.special_model_handler import update_modulefrom auto_round.utils import (SUPPORTED_LAYER_TYPES,check_start_with_block_name,@@ -593,7 +593,7 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.Mpacking_format = "auto_round:auto_gptq"# Preprocess model before replace layers- model = _handle_moe_model(model)+ model = update_module(model)# Replace layers with quantized versionslayer_configs = get_layer_config(model, quantization_config)diff --git a/auto_round/modelling/gpt_oss.py b/auto_round/modelling/gpt_oss.pyindex 9fe183779..06506ef27 100644--- a/auto_round/modelling/gpt_oss.py+++ b/auto_round/modelling/gpt_oss.py@@ -19,9 +19,8 @@ from transformers.modeling_utils import no_init_weights as skip_weights_initialifrom transformers.models.gpt_oss.configuration_gpt_oss import GptOssConfigfrom transformers.models.gpt_oss.modeling_gpt_oss import GptOssMLP-from auto_round.utils import unsupported_meta_device--__all__ = ["get_replacement_info"]+from auto_round.modelling.replace_modules import ReplacementModuleBase+from auto_round.utils import LazyImport, logger, unsupported_meta_devicedef _update_parameter(@@ -54,13 +53,13 @@ class GPTOssSingleExpert(nn.Module):return self.down_proj(act)-class SequentialGPTOSSMoE(nn.Module):+class SequentialGPTOSSMoE(ReplacementModuleBase):"""Replaces GPT-OSS fused-expert MoE with per-expert `GPTOssSingleExpert` modules.Copies weights from fused tensors and reuses the original router and optional shared_expert."""- def __init__(self, config: GptOssConfig, original: GptOssMLP):+ def __init__(self, original: GptOssMLP, config: GptOssConfig):super().__init__()hidden_size = config.hidden_sizeintermediate_size = config.intermediate_size@@ -123,10 +122,17 @@ class SequentialGPTOSSMoE(nn.Module):final_hidden_states += expert_outputreturn final_hidden_states.view(B, T, H), router_scores.view(B * T, -1)--def get_replacement_info(config):- return (- SequentialGPTOSSMoE,- config.get_text_config(),- GptOssMLP.__name__,- )+ @classmethod+ def original_module_class(cls) -> str:+ """Return the class name of the module this replaces."""+ return "GptOssMLP"++ @classmethod+ def from_original(+ cls,+ original: GptOssMLP,+ config: GptOssConfig,+ **kwargs,+ ) -> "SequentialGPTOSSMoE":+ """Create an instance from the original module."""+ return cls(original, config)diff --git a/auto_round/modelling/llama4.py b/auto_round/modelling/llama4.pyindex cb64fdc68..811488709 100644--- a/auto_round/modelling/llama4.py+++ b/auto_round/modelling/llama4.py@@ -12,14 +12,11 @@# See the License for the specific language governing permissions and# limitations under the License.# Note: adapted from # https://github.com/vllm-project/llm-compressor/blob/main/src/llmcompressor/modeling/llama4.py--__all__ = ["get_replacement_info"]--import torchfrom transformers.modeling_utils import no_init_weights-from transformers.models.llama4.modeling_llama4 import Llama4TextMLP+from transformers.models.llama4.modeling_llama4 import Llama4Config, Llama4TextMLP, Llama4TextMoe+from auto_round.modelling.replace_modules import ReplacementModuleBasefrom auto_round.utils import unsupported_meta_device@@ -43,9 +40,10 @@ class SequentialLlama4TextExperts(torch.nn.ModuleList):self[i].down_proj.weight.data.copy_(down.t())-class SequentialLlama4TextMoe(torch.nn.Module):- def __init__(self, config, original):+class SequentialLlama4TextMoe(ReplacementModuleBase):+ def __init__(self, original, config):super().__init__()+ config = config.text_configself.top_k = config.num_experts_per_tokself.hidden_dim = config.hidden_sizeself.num_experts = config.num_local_experts@@ -76,6 +74,17 @@ class SequentialLlama4TextMoe(torch.nn.Module):return out, router_logits--def get_replacement_info(config):- return SequentialLlama4TextMoe, config.get_text_config(), "Llama4TextMoe"+ @classmethod+ def original_module_class(cls) -> str:+ """Return the class name of the module this replaces."""+ return "Llama4TextMoe"++ @classmethod+ def from_original(+ cls,+ original: torch.nn.Module,+ config: Llama4Config,+ **kwargs,+ ) -> "SequentialLlama4TextMoe":+ """Create an instance from the original module."""+ return cls(original, config)diff --git a/auto_round/modelling/qwen3_vl_moe.py b/auto_round/modelling/qwen3_vl_moe.pyindex 9a47871e5..dbed42b1a 100644--- a/auto_round/modelling/qwen3_vl_moe.py+++ b/auto_round/modelling/qwen3_vl_moe.py@@ -18,6 +18,7 @@ from packaging import versionfrom torch import nnfrom transformers.activations import ACT2FN+from auto_round.modelling.replace_modules import ReplacementModuleBasefrom auto_round.utils import logger, unsupported_meta_devicetransformers_version = version.parse(transformers.__version__)@@ -41,7 +42,7 @@ def _update_parameter(# Adapted from https://github.com/vllm-project/llm-compressor/blob/main/src/llmcompressor/modeling/qwen3_vl_moe.py-class LinearQwen3VLMoeTextSparseMoeBlock(torch.nn.Module):+class LinearQwen3VLMoeTextSparseMoeBlock(ReplacementModuleBase):"""Calibration version of Qwen3VLMoeTextSparseMoeBlock that sends all tokens to allexperts.@@ -117,6 +118,21 @@ class LinearQwen3VLMoeTextSparseMoeBlock(torch.nn.Module):else:return next_states+ @classmethod+ def original_module_class(cls) -> str:+ """Return the class name of the module this replaces."""+ return "Qwen3VLMoeTextSparseMoeBlock"++ @classmethod+ def from_original(+ cls,+ original: "Qwen3VLMoeTextSparseMoeBlock",+ config: "Qwen3VLMoeConfig",+ **kwargs,+ ) -> "LinearQwen3VLMoeTextSparseMoeBlock":+ """Create an instance from the original module."""+ return cls(original, config)+class SequentialQwen3VLMoeTextExperts(torch.nn.ModuleList):def __init__(self, config, original):@@ -140,7 +156,3 @@ class SequentialQwen3VLMoeTextExperts(torch.nn.ModuleList):_update_parameter(self[i].gate_proj, "weight", gate_proj.t().contiguous())_update_parameter(self[i].up_proj, "weight", up_proj.t().contiguous())_update_parameter(self[i].down_proj, "weight", down.t().contiguous())---def get_replacement_info(config):- return (LinearQwen3VLMoeTextSparseMoeBlock, config, "Qwen3VLMoeTextSparseMoeBlock")diff --git a/auto_round/modelling/replace_modules.py b/auto_round/modelling/replace_modules.pynew file mode 100644index 000000000..c8e34954f--- /dev/null+++ b/auto_round/modelling/replace_modules.py@@ -0,0 +1,180 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from abc import ABC, abstractmethod+from typing import Dict, Type++import torch+from tqdm import tqdm+from transformers import PreTrainedModel++from auto_round.utils import LazyImport, logger++BUILTIN_MODULES = {+ "Llama4TextMoe": LazyImport("auto_round.modelling.llama4"),+ "GptOssMLP": LazyImport("auto_round.modelling.gpt_oss"),+ "Qwen3VLMoeTextSparseMoeBlock": LazyImport("auto_round.modelling.qwen3_vl_moe"),+}+++def _import_required_replacements(model: torch.nn.Module) -> None:+ """Scan model and trigger lazy imports for registered replacement modules."""+ imported = set()++ for _, module in model.named_modules():+ class_name = module.__class__.__name__++ if class_name in BUILTIN_MODULES and class_name not in imported:+ # Trigger import by accessing the LazyImport object+ _ = BUILTIN_MODULES[class_name].__name__ # or any attribute+ imported.add(class_name)+ logger.debug(f"Loaded replacement module for {class_name}")+++class ReplacementModuleBase(ABC, torch.nn.Module):+ """+ Abstract base class for module replacement during calibration phase.++ Replacement modules replace original modules to ensure all components+ receive data for proper quantization statistics.++ Subclasses must:+ 1. Implement `original_module_class()` to return the target module class name+ 2. Implement `__init__()` with signature:+ (self, original, config)+ """++ # Registry: module class name -> replacement module class+ _replacement_registry: Dict[str, Type["ReplacementModuleBase"]] = {}++ def __init_subclass__(cls, **kwargs):+ """Automatically register subclasses in the replacement registry."""+ super().__init_subclass__(**kwargs)++ # Only register if it's a concrete implementation (not ABC)+ if not getattr(cls, "__abstractmethods__", None):+ if cls.original_module_class() is None:+ raise TypeError(+ f"{cls.__name__} must implement 'original_module_class()' class method "+ "to return the name of the module class it replaces"+ )++ if cls.original_module_class() in cls._replacement_registry:+ existing = cls._replacement_registry[cls.original_module_class()]+ raise ValueError(+ f"Module '{cls.original_module_class()}' already registered to "+ f"{existing.__name__}. Cannot register {cls.__name__}."+ )++ cls._replacement_registry[cls.original_module_class()] = cls+ logger.trace(f"Registered {cls.__name__} for replacing {cls.original_module_class()}")++ @classmethod+ def get_replacement_class(cls, module_class_name: str) -> Type["ReplacementModuleBase"]:+ """Get replacement class for a given module class name."""+ return cls._replacement_registry.get(module_class_name)++ @classmethod+ def is_registered(cls, module_class_name: str) -> bool:+ """Check if a module class has a replacement implementation."""+ return module_class_name in cls._replacement_registry++ @classmethod+ def is_to_be_replaced(+ cls,+ original: torch.nn.Module,+ module_class_name: str,+ ) -> bool:+ """Determine if the given module should be replaced.++ Users can extend this method to add custom logic for replacement.+ """+ return cls.is_registered(module_class_name)++ @classmethod+ def get_registered_modules(cls) -> list:+ """Get list of all registered module class names."""+ return list(cls._replacement_registry.keys())++ @classmethod+ @abstractmethod+ def original_module_class(cls) -> str:+ """Return the class name of the module this replaces."""+ pass++ @classmethod+ @abstractmethod+ def from_original(+ cls,+ original: torch.nn.Module,+ config,+ ) -> "ReplacementModuleBase":+ """Create replacement module from original module."""+ pass+++# Note: adapted from llm-compressor+# https://github.com/vllm-project/llm-compressor/blob/main/src/llmcompressor/modeling/moe_context.py+++def apply_replacements(+ model: torch.nn.Module,+) -> torch.nn.Module:+ """+ Function to apply module replacements to a model.++ This scans all modules in the model and replaces any registered modules with their+ replacement equivalents. Non-permanent modules are tracked for later restoration.++ The model is modified in-place, so the same model object should be used.++ Args:+ model: The model to apply module replacement to (modified in-place).++ Returns:+ The model with modules replaced.+ """+ _import_required_replacements(model)+ replaced = {}++ # Step 1: Collect all modules that need replacement+ logger.debug("Scanning for modules to replace")+ modules_to_replace = []+ for name, module in model.named_modules():+ # skip replaced modules+ if isinstance(module, ReplacementModuleBase):+ continue+ class_name = module.__class__.__name__+ if ReplacementModuleBase.is_to_be_replaced(module, class_name):+ modules_to_replace.append((name, module, class_name))++ # Step 2: Replace modules+ if modules_to_replace:+ logger.info(f"Found {len(modules_to_replace)} modules to replace")+ for name, module, class_name in tqdm(modules_to_replace, desc="Replacing modules"):+ replacement_cls = ReplacementModuleBase.get_replacement_class(class_name)+ replacement = replacement_cls.from_original(+ module,+ model.config,+ )+ model.set_submodule(name, replacement)+ replaced[name] = (module, replacement)+ else:+ logger.debug("No modules found for replacement")++ # Log what was replaced+ if replaced:+ logger.info(f"Replaced {len(replaced)} modules")++ return modeldiff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex eb8fa3e26..3bb60bfb0 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -15,6 +15,7 @@ import torchimport auto_round.modelling as auto_round_modellingfrom auto_round.formats import OutputFormat+from auto_round.modelling.replace_modules import apply_replacementsfrom auto_round.utils import LazyImport, logger, unsupported_meta_devicemllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_size@@ -40,30 +41,9 @@ SPECIAL_SHARED_CACHE_KEYS = {"Gemma3ForConditionalGeneration": ("position_embeddings_global", "position_embeddings_local")}SPECIAL_SHARED_CACHE_KEYS["MiniMaxText01ForCausalLM"] = ("slope_rate",)--CONVERT_EXPERT_TO_LINEAR_MODELS = ["llama4", "gpt_oss", "qwen3_vl_moe"]-MISTRAL_3_2_MODELS = ["Mistral-Small-3.2", "Magistral-Small", "Devstral-Small"]-def _get_moe_converter(config):- # Dispatch table for model_type to replacement_info functions- moe_converters = {- "gpt_oss": LazyImport("auto_round.modelling.gpt_oss.get_replacement_info"),- "llama4": LazyImport("auto_round.modelling.llama4.get_replacement_info"),- "qwen3_vl_moe": LazyImport("auto_round.modelling.qwen3_vl_moe.get_replacement_info"),- }-- # Retrieve the appropriate function based on model_type- if config.model_type in moe_converters:- return moe_converters[config.model_type](config)- else:- raise ValueError(- f"Unsupported model_type '{config.model_type}'. "- f"Currently, MoE converter only supports: {', '.join(moe_converters.keys())}."- )--def _handle_special_model(model):if hasattr(model, "config") and model.config.model_type == "deepseek_vl_v2":from functools import partial@@ -72,28 +52,11 @@ def _handle_special_model(model):return model-def _handle_moe_model(model, formats: list[OutputFormat] = None):+def update_module(model, formats: list[OutputFormat] = None):if formats is not None and any([format_.is_gguf() for format_ in formats]):return model- if hasattr(model.config, "model_type") and model.config.model_type in CONVERT_EXPERT_TO_LINEAR_MODELS:- from tqdm import tqdm- from auto_round.utils import clear_memory-- new_moe_class, convert_config, orig_cls_name = _get_moe_converter(model.config)- if not unsupported_meta_device(model):- model = model.to("cpu")- clear_memory()-- for name, module in tqdm(model.named_modules(), desc="Converting model"):- cls_name = module.__class__.__name__- if cls_name == orig_cls_name:- new_module = new_moe_class(config=convert_config, original=module)- parent, child = name.rsplit(".", maxsplit=1)- parent = model.get_submodule(parent)- setattr(parent, child, new_module)-- return model+ return apply_replacements(model)def _get_deepseek_vl2_multimodal_block(model, quant_vision=False):diff --git a/test/test_cpu/test_moe_model.py b/test/test_cpu/test_moe_model.pyindex 79e54c4d1..f70238f68 100644--- a/test/test_cpu/test_moe_model.py+++ b/test/test_cpu/test_moe_model.py@@ -26,8 +26,8 @@ def setup_llama4():model_name = "/tf_dataset/auto_round/models/meta-llama/Llama-4-Scout-17B-16E-Instruct"tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)- config.vision_config.num_hidden_layers = 2 # Reduce layers for testing- config.text_config.num_hidden_layers = 2+ config.vision_config.num_hidden_layers = 1 # Reduce layers for testing+ config.text_config.num_hidden_layers = 1model = Llama4ForConditionalGeneration(config)output_dir = "/tmp/test_quantized_llama4"return model, tokenizer, output_dir, config@@ -56,6 +56,7 @@ def quantize_model(model, tokenizer, output_dir, scheme, iters=0):scheme=scheme,nsamples=2,iters=iters,+ seqlen=32,fp_layers="self_attn,router,lm_head,mlp.gate",)quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)diff --git a/test/test_cuda/test_moe_model.py b/test/test_cuda/test_moe_model.pyindex 1865519c9..8c21abbdf 100644--- a/test/test_cuda/test_moe_model.py+++ b/test/test_cuda/test_moe_model.py@@ -2,11 +2,15 @@ import shutilimport pytestimport torch+import torch.nn as nnfrom transformers import AutoConfig, AutoProcessor, AutoTokenizer, Llama4ForConditionalGenerationfrom transformers.models.gpt_oss.modeling_gpt_oss import GptOssForCausalLM+from transformers.models.qwen3.modeling_qwen3 import Qwen3Config, Qwen3ForCausalLM, Qwen3MLPfrom transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeForConditionalGenerationfrom auto_round import AutoRound+from auto_round.modelling.replace_modules import ReplacementModuleBase+from auto_round.utils import LazyImport, logger, unsupported_meta_device@pytest.fixture@@ -27,8 +31,8 @@ def setup_llama4():model_name = "/dataset/Llama-4-Scout-17B-16E-Instruct"tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)- config.vision_config.num_hidden_layers = 2 # Reduce layers for testing- config.text_config.num_hidden_layers = 2+ config.vision_config.num_hidden_layers = 1 # Reduce layers for testing+ config.text_config.num_hidden_layers = 1model = Llama4ForConditionalGeneration(config)output_dir = "test_quantized_llama4"return model, tokenizer, output_dir, config@@ -45,7 +49,7 @@ def setup_qwen3_vl_moe():config.num_hidden_layers = 1 # Reduce layers for testingprocessor = AutoProcessor.from_pretrained(model_name)model = Qwen3VLMoeForConditionalGeneration(config)- output_dir = "/tmp/test_quantized_qwen3_vl_moe"+ output_dir = "test_quantized_qwen3_vl_moe"return model, tokenizer, processor, output_dir, config@@ -57,6 +61,7 @@ def quantize_model(model, tokenizer, output_dir, scheme, iters=0):scheme=scheme,nsamples=2,iters=iters,+ seqlen=32,fp_layers="self_attn,router,lm_head,mlp.gate",)quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)@@ -121,6 +126,47 @@ def test_llama4(setup_llama4):shutil.rmtree(output_dir, ignore_errors=True)+class NewQwen3MLP(ReplacementModuleBase):+ def __init__(self, original: Qwen3MLP, config: Qwen3Config):+ super().__init__()+ self.config = config+ self.hidden_size = config.hidden_size+ self.intermediate_size = config.intermediate_size+ self.new_gate_proj = original.gate_proj+ self.new_up_proj = original.up_proj+ self.new_down_proj = original.down_proj+ self.act_fn = nn.GELU()++ def forward(self, x):+ gate = self.new_gate_proj(x)+ up = self.new_up_proj(x)+ gate_act = self.act_fn(gate)+ act = gate_act * up+ out = self.new_down_proj(act)+ return out++ @classmethod+ def original_module_class(cls) -> str:+ return "Qwen3MLP"++ @classmethod+ def from_original(cls, original: Qwen3MLP, config: Qwen3Config):+ return cls(original, config)+++def setup_qwen3():+ """Fixture to set up the qwen3 model and tokenizer."""+ model_name = "/models/Qwen3-0.6B"+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ config = AutoConfig.from_pretrained(model_name)+ config.num_hidden_layers = 2 # Reduce layers for testing+ config.layer_types = config.layer_types[:2] # Reduce layers for testing+ model = Qwen3ForCausalLM(config)+ output_dir = "test_quantized_qwen3"+ return model, tokenizer, output_dir, config++def test_qwen3_vl_moe_mxfp(setup_qwen3_vl_moe):model, tokenizer, processor, output_dir, config = setup_qwen3_vl_moeautoround = AutoRound(@@ -142,6 +188,11 @@ def test_qwen3_vl_moe_mxfp(setup_qwen3_vl_moe):if m.__class__.__name__ == "QuantLinear":loaded_m = loaded_model.get_submodule(n)assert (loaded_m.weight_packed == m.weight_packed).all()++ inp = torch.randint(0, 100, (1, 64)).to("cuda")+ with torch.inference_mode():+ loaded_out = loaded_model(inp)+# test generationtokenizer = AutoTokenizer.from_pretrained(output_dir)text = "There is a girl who likes adventure,"@@ -149,3 +200,34 @@ def test_qwen3_vl_moe_mxfp(setup_qwen3_vl_moe):print(tokenizer.decode(loaded_model.generate(**inputs, max_new_tokens=50)[0]))# clean the output directory after testshutil.rmtree(output_dir, ignore_errors=True)+++def has_module(model: torch.nn.Module, module_name: str) -> bool:+ for n, m in model.named_modules():+ if module_name in n:+ return True+ return False+++def test_register_module_out_of_tree_base():+ from auto_round.modelling.replace_modules import ReplacementModuleBase++ for name, subclass in ReplacementModuleBase._replacement_registry.items():+ if name == "Qwen3MLP":+ assert subclass == NewQwen3MLP, "Qwen3MLP not registered correctly."+++def test_register_module_out_of_tree_model(setup_qwen3):+ model, tokenizer, output_dir, config = setup_qwen3+ quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4")+ # Ensure the quantized model is not None+ assert quantized_model is not None, "Quantized model should not be None."+ check_module_names = ["new_gate_proj", "new_up_proj", "new_down_proj"]+ for name in check_module_names:+ assert has_module(quantized_model, name), f"Module {name} not found in quantized model."+ loaded_model = Qwen3ForCausalLM.from_pretrained(output_dir)+ quantized_model.to("cuda")++ for name in check_module_names:+ assert has_module(loaded_model, name), f"Module {name} not found in loaded model."+ loaded_model.to("cuda")