PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine moe modellings to release orginal expert module's ram (#1265)
Weiweie383eea36471
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 96d80f7ea..f6714db16 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -146,9 +146,9 @@ def get_pile_dataset(# Check for proxy or SSL errorif "proxy" in error_message.lower() or isinstance(e, ssl.SSLError) or "SSL" in error_message.upper():logger.error(- "Network error detected, please checking proxy settings."- "Error: {error_message}. Or consider using a backup dataset by `pip install modelscope`"- " and set '--dataset swift/pile-val-backup' in AutoRound API."+ f"Network error detected, please check proxy settings. "+ f"Error: {error_message}. Or consider using a backup dataset by `pip install modelscope` "+ f"and set '--dataset swift/pile-val-backup' in AutoRound API.")else:logger.error(f"Failed to load the dataset: {error_message}")diff --git a/auto_round/modelling/gpt_oss.py b/auto_round/modelling/gpt_oss.pyindex 06506ef27..7514b77b2 100644--- a/auto_round/modelling/gpt_oss.py+++ b/auto_round/modelling/gpt_oss.py@@ -20,7 +20,7 @@ from transformers.models.gpt_oss.configuration_gpt_oss import GptOssConfigfrom transformers.models.gpt_oss.modeling_gpt_oss import GptOssMLPfrom auto_round.modelling.replace_modules import ReplacementModuleBase-from auto_round.utils import LazyImport, logger, unsupported_meta_device+from auto_round.utils import LazyImport, clear_memory, logger, unsupported_meta_devicedef _update_parameter(@@ -92,6 +92,8 @@ class SequentialGPTOSSMoE(ReplacementModuleBase):_update_parameter(mlp.gate_proj, "bias", original.experts.gate_up_proj_bias[i, ::2])_update_parameter(mlp.up_proj, "bias", original.experts.gate_up_proj_bias[i, 1::2])_update_parameter(mlp.down_proj, "bias", original.experts.down_proj_bias[i]) # [H]+ original.experts.to_empty(device="meta") # release original experts parameters+ clear_memory()def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:B, T, H = hidden_states.shapediff --git a/auto_round/modelling/llama4.py b/auto_round/modelling/llama4.pyindex 811488709..ddca377b8 100644--- a/auto_round/modelling/llama4.py+++ b/auto_round/modelling/llama4.py@@ -17,13 +17,14 @@ from transformers.modeling_utils import no_init_weightsfrom transformers.models.llama4.modeling_llama4 import Llama4Config, Llama4TextMLP, Llama4TextMoefrom auto_round.modelling.replace_modules import ReplacementModuleBase-from auto_round.utils import unsupported_meta_device+from auto_round.utils import clear_memory, unsupported_meta_deviceclass SequentialLlama4TextExperts(torch.nn.ModuleList):def __init__(self, config, original):self.num_experts = original.gate_up_proj.shape[0]- with no_init_weights():+ target_device = next(original.parameters()).device+ with no_init_weights(), torch.device(target_device):super().__init__([Llama4TextMLP(config) for _ in range(self.num_experts)])if not unsupported_meta_device(original):@@ -38,6 +39,8 @@ class SequentialLlama4TextExperts(torch.nn.ModuleList):self[i].gate_proj.weight.data.copy_(gate_proj.t())self[i].up_proj.weight.data.copy_(up_proj.t())self[i].down_proj.weight.data.copy_(down.t())+ del gate_up, down, gate_proj, up_proj+ original.to_empty(device="meta") # release original experts parametersclass SequentialLlama4TextMoe(ReplacementModuleBase):diff --git a/auto_round/modelling/qwen3_vl_moe.py b/auto_round/modelling/qwen3_vl_moe.pyindex 2e38bb740..345855fa2 100644--- a/auto_round/modelling/qwen3_vl_moe.py+++ b/auto_round/modelling/qwen3_vl_moe.py@@ -19,7 +19,7 @@ from torch import nnfrom transformers.activations import ACT2FNfrom auto_round.modelling.replace_modules import ReplacementModuleBase-from auto_round.utils import logger, unsupported_meta_device+from auto_round.utils import clear_memory, logger, unsupported_meta_devicetransformers_version = version.parse(transformers.__version__)@@ -113,7 +113,7 @@ class LinearQwen3VLMoeTextSparseMoeBlock(ReplacementModuleBase):next_states.index_add_(0, token_idx, weighted_output.to(hidden_states.dtype))next_states = next_states.reshape(batch_size, sequence_length, hidden_dim)- if transformers_version <= version.parse("4.57.3"):+ if transformers_version < version.parse("5.0"):return next_states, router_logitselse:return next_states@@ -143,7 +143,9 @@ class SequentialQwen3VLMoeTextExperts(torch.nn.ModuleList):Qwen3VLMoeTextMLP,)- super().__init__([Qwen3VLMoeTextMLP(config, intermediate_size) for _ in range(self.num_experts)])+ target_device = next(original.parameters()).device+ with torch.device(target_device):+ super().__init__([Qwen3VLMoeTextMLP(config, intermediate_size) for _ in range(self.num_experts)])if not unsupported_meta_device(original):for i in range(self.num_experts):@@ -153,6 +155,9 @@ class SequentialQwen3VLMoeTextExperts(torch.nn.ModuleList):gate_proj = gate_up[:, :intermediate_size]up_proj = gate_up[:, intermediate_size:]- _update_parameter(self[i].gate_proj, "weight", gate_proj.t().contiguous())- _update_parameter(self[i].up_proj, "weight", up_proj.t().contiguous())- _update_parameter(self[i].down_proj, "weight", down.t().contiguous())+ _update_parameter(self[i].gate_proj, "weight", gate_proj.t())+ _update_parameter(self[i].up_proj, "weight", up_proj.t())+ _update_parameter(self[i].down_proj, "weight", down.t())+ del gate_up, down, gate_proj, up_proj+ original.to_empty(device="meta") # release original experts parameters+ clear_memory()diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.pyindex 89a6c8b13..4934d619e 100644--- a/test/test_cpu/models/test_moe_model.py+++ b/test/test_cpu/models/test_moe_model.py@@ -11,7 +11,7 @@ from ...helpers import get_model_pathgpt_oss_name_or_path = get_model_path("unsloth/gpt-oss-20b-BF16")llama4_name_or_path = get_model_path("meta-llama/Llama-4-Scout-17B-16E-Instruct")-+qwen3_vl_moe_name_or_path = get_model_path("Qwen/Qwen3-VL-30B-A3B-Instruct")# local path for debug# llama4_name_or_path = get_model_path("/dataset/Llama-4-Scout-17B-16E-Instruct")@@ -44,7 +44,7 @@ def setup_llama4():@pytest.fixturedef setup_qwen3_vl_moe():"""Fixture to set up the qwen3_vl_moe model and tokenizer."""- model_name = "/tf_dataset/auto_round/models/Qwen/Qwen3-VL-30B-A3B-Instruct"+ model_name = qwen3_vl_moe_name_or_pathtokenizer = AutoTokenizer.from_pretrained(model_name)config = AutoConfig.from_pretrained(model_name)config.vision_config.num_hidden_layers = 1