Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine moe modellings to release orginal expert module's ram (#1265)

Weiweie383eea36471
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index 96d80f7ea..f6714db16 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -146,9 +146,9 @@ def get_pile_dataset(
# Check for proxy or SSL error
if "proxy" in error_message.lower() or isinstance(e, ssl.SSLError) or "SSL" in error_message.upper():
logger.error(
- "Network error detected, please checking proxy settings."
- "Error: {error_message}. Or consider using a backup dataset by `pip install modelscope`"
- " and set '--dataset swift/pile-val-backup' in AutoRound API."
+ f"Network error detected, please check proxy settings. "
+ f"Error: {error_message}. Or consider using a backup dataset by `pip install modelscope` "
+ f"and set '--dataset swift/pile-val-backup' in AutoRound API."
)
else:
logger.error(f"Failed to load the dataset: {error_message}")
diff --git a/auto_round/modelling/gpt_oss.py b/auto_round/modelling/gpt_oss.py
index 06506ef27..7514b77b2 100644
--- a/auto_round/modelling/gpt_oss.py
+++ b/auto_round/modelling/gpt_oss.py
@@ -20,7 +20,7 @@ from transformers.models.gpt_oss.configuration_gpt_oss import GptOssConfig
from transformers.models.gpt_oss.modeling_gpt_oss import GptOssMLP
from auto_round.modelling.replace_modules import ReplacementModuleBase
-from auto_round.utils import LazyImport, logger, unsupported_meta_device
+from auto_round.utils import LazyImport, clear_memory, logger, unsupported_meta_device
def _update_parameter(
@@ -92,6 +92,8 @@ class SequentialGPTOSSMoE(ReplacementModuleBase):
_update_parameter(mlp.gate_proj, "bias", original.experts.gate_up_proj_bias[i, ::2])
_update_parameter(mlp.up_proj, "bias", original.experts.gate_up_proj_bias[i, 1::2])
_update_parameter(mlp.down_proj, "bias", original.experts.down_proj_bias[i]) # [H]
+ original.experts.to_empty(device="meta") # release original experts parameters
+ clear_memory()
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
B, T, H = hidden_states.shape
diff --git a/auto_round/modelling/llama4.py b/auto_round/modelling/llama4.py
index 811488709..ddca377b8 100644
--- a/auto_round/modelling/llama4.py
+++ b/auto_round/modelling/llama4.py
@@ -17,13 +17,14 @@ from transformers.modeling_utils import no_init_weights
from transformers.models.llama4.modeling_llama4 import Llama4Config, Llama4TextMLP, Llama4TextMoe
from auto_round.modelling.replace_modules import ReplacementModuleBase
-from auto_round.utils import unsupported_meta_device
+from auto_round.utils import clear_memory, unsupported_meta_device
class SequentialLlama4TextExperts(torch.nn.ModuleList):
def __init__(self, config, original):
self.num_experts = original.gate_up_proj.shape[0]
- with no_init_weights():
+ target_device = next(original.parameters()).device
+ with no_init_weights(), torch.device(target_device):
super().__init__([Llama4TextMLP(config) for _ in range(self.num_experts)])
if not unsupported_meta_device(original):
@@ -38,6 +39,8 @@ class SequentialLlama4TextExperts(torch.nn.ModuleList):
self[i].gate_proj.weight.data.copy_(gate_proj.t())
self[i].up_proj.weight.data.copy_(up_proj.t())
self[i].down_proj.weight.data.copy_(down.t())
+ del gate_up, down, gate_proj, up_proj
+ original.to_empty(device="meta") # release original experts parameters
class SequentialLlama4TextMoe(ReplacementModuleBase):
diff --git a/auto_round/modelling/qwen3_vl_moe.py b/auto_round/modelling/qwen3_vl_moe.py
index 2e38bb740..345855fa2 100644
--- a/auto_round/modelling/qwen3_vl_moe.py
+++ b/auto_round/modelling/qwen3_vl_moe.py
@@ -19,7 +19,7 @@ from torch import nn
from transformers.activations import ACT2FN
from auto_round.modelling.replace_modules import ReplacementModuleBase
-from auto_round.utils import logger, unsupported_meta_device
+from auto_round.utils import clear_memory, logger, unsupported_meta_device
transformers_version = version.parse(transformers.__version__)
@@ -113,7 +113,7 @@ class LinearQwen3VLMoeTextSparseMoeBlock(ReplacementModuleBase):
next_states.index_add_(0, token_idx, weighted_output.to(hidden_states.dtype))
next_states = next_states.reshape(batch_size, sequence_length, hidden_dim)
- if transformers_version <= version.parse("4.57.3"):
+ if transformers_version < version.parse("5.0"):
return next_states, router_logits
else:
return next_states
@@ -143,7 +143,9 @@ class SequentialQwen3VLMoeTextExperts(torch.nn.ModuleList):
Qwen3VLMoeTextMLP,
)
- super().__init__([Qwen3VLMoeTextMLP(config, intermediate_size) for _ in range(self.num_experts)])
+ target_device = next(original.parameters()).device
+ with torch.device(target_device):
+ super().__init__([Qwen3VLMoeTextMLP(config, intermediate_size) for _ in range(self.num_experts)])
if not unsupported_meta_device(original):
for i in range(self.num_experts):
@@ -153,6 +155,9 @@ class SequentialQwen3VLMoeTextExperts(torch.nn.ModuleList):
gate_proj = gate_up[:, :intermediate_size]
up_proj = gate_up[:, intermediate_size:]
- _update_parameter(self[i].gate_proj, "weight", gate_proj.t().contiguous())
- _update_parameter(self[i].up_proj, "weight", up_proj.t().contiguous())
- _update_parameter(self[i].down_proj, "weight", down.t().contiguous())
+ _update_parameter(self[i].gate_proj, "weight", gate_proj.t())
+ _update_parameter(self[i].up_proj, "weight", up_proj.t())
+ _update_parameter(self[i].down_proj, "weight", down.t())
+ del gate_up, down, gate_proj, up_proj
+ original.to_empty(device="meta") # release original experts parameters
+ clear_memory()
diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.py
index 89a6c8b13..4934d619e 100644
--- a/test/test_cpu/models/test_moe_model.py
+++ b/test/test_cpu/models/test_moe_model.py
@@ -11,7 +11,7 @@ from ...helpers import get_model_path
gpt_oss_name_or_path = get_model_path("unsloth/gpt-oss-20b-BF16")
llama4_name_or_path = get_model_path("meta-llama/Llama-4-Scout-17B-16E-Instruct")
-
+qwen3_vl_moe_name_or_path = get_model_path("Qwen/Qwen3-VL-30B-A3B-Instruct")
# local path for debug
# llama4_name_or_path = get_model_path("/dataset/Llama-4-Scout-17B-16E-Instruct")
@@ -44,7 +44,7 @@ def setup_llama4():
@pytest.fixture
def setup_qwen3_vl_moe():
"""Fixture to set up the qwen3_vl_moe model and tokenizer."""
- model_name = "/tf_dataset/auto_round/models/Qwen/Qwen3-VL-30B-A3B-Instruct"
+ model_name = qwen3_vl_moe_name_or_path
tokenizer = AutoTokenizer.from_pretrained(model_name)
config = AutoConfig.from_pretrained(model_name)
config.vision_config.num_hidden_layers = 1