Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support glm5 (#1466)

Wenhua Cheng62c9808d5773
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index bc19dd32b..75e9b0f3d 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -212,8 +212,8 @@ def save_quantized_as_autogptq(
# --- Save metadata (tokenizer, processor, etc.) ---
if output_dir:
- if os.path.exists(output_dir):
- logger.warning(f"{output_dir} already exists, may cause overwrite conflicts.")
+ # if os.path.exists(output_dir):
+ # logger.info(f"{output_dir} already exists, may cause overwrite conflicts.")
for comp in (tokenizer, processor, image_processor):
if comp is not None and hasattr(comp, "save_pretrained"):
comp.save_pretrained(output_dir)
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index ac4701dd3..c57bf452c 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -332,8 +332,8 @@ def save_quantized_as_autoround(
if output_dir is None:
model.tokenizer = tokenizer
return model
- if os.path.exists(output_dir):
- logger.warning(f"{output_dir} already exists, this may cause model conflict")
+ # if os.path.exists(output_dir):
+ # logger.info(f"{output_dir} already exists, this may cause model conflict")
if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):
tokenizer.save_pretrained(output_dir)
diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.py
index 82fcdcf20..70c7c71b1 100644
--- a/auto_round/export/export_to_autoround/export_to_fp8.py
+++ b/auto_round/export/export_to_autoround/export_to_fp8.py
@@ -232,8 +232,8 @@ def save_quantized_as_autoround(
if output_dir is None:
model.tokenizer = tokenizer
return model
- if os.path.exists(output_dir):
- logger.warning(f"{output_dir} already exists, this may cause model conflict")
+ # if os.path.exists(output_dir):
+ # logger.warning(f"{output_dir} already exists, this may cause model conflict")
if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.py
index 3c39d06e4..47e545ff9 100644
--- a/auto_round/export/export_to_llmcompressor/export.py
+++ b/auto_round/export/export_to_llmcompressor/export.py
@@ -141,8 +141,8 @@ def save_quantized_as_llmcompressor(
safe_serialization = kwargs.get("safe_serialization", True)
processor = kwargs.get("processor", None)
- if output_dir is not None and os.path.exists(output_dir):
- logger.warning(f"{output_dir} already exists, this may cause model conflict")
+ # if output_dir is not None and os.path.exists(output_dir):
+ # logger.warning(f"{output_dir} already exists, this may cause model conflict")
if not inplace:
model = copy.deepcopy(model.to("cpu"))
diff --git a/auto_round/export/utils.py b/auto_round/export/utils.py
index 42e6b4f9e..3bdacf096 100644
--- a/auto_round/export/utils.py
+++ b/auto_round/export/utils.py
@@ -110,6 +110,8 @@ def save_model(
model.config.save_pretrained(save_dir)
if hasattr(model, "generation_config") and model.generation_config is not None:
+ if hasattr(model, "generation_config"):
+ setattr(model.generation_config, "do_sample", True)
model.generation_config.save_pretrained(save_dir)
elif has_unfused_experts:
# For models with unfused MOE experts, save state_dict directly to avoid
diff --git a/auto_round/modeling/unfused_moe/__init__.py b/auto_round/modeling/unfused_moe/__init__.py
index a30e2ae45..d00c21279 100644
--- a/auto_round/modeling/unfused_moe/__init__.py
+++ b/auto_round/modeling/unfused_moe/__init__.py
@@ -55,6 +55,17 @@ MODEL_CONFIG = {
)
],
},
+ "glm_moe_dsa": {
+ "min_transformers_version": "5.0.0",
+ "max_transformers_version": "6.0.0",
+ "checkpoint_mapping": [],
+ "block_patch": [
+ (
+ "transformers.models.glm_moe_dsa.modeling_glm_moe_dsa.GlmMoeDsaMoE",
+ "auto_round.modeling.unfused_moe.glm_moe_dsa.LinearGlmMoeDsaMoE",
+ )
+ ],
+ },
"qwen3_next": {
"min_transformers_version": "5.0.0",
"max_transformers_version": "6.0.0",
diff --git a/auto_round/modeling/unfused_moe/glm_moe_dsa.py b/auto_round/modeling/unfused_moe/glm_moe_dsa.py
new file mode 100644
index 000000000..2fb34de0f
--- /dev/null
+++ b/auto_round/modeling/unfused_moe/glm_moe_dsa.py
@@ -0,0 +1,95 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import torch
+import torch.nn as nn
+
+
+class LinearGlmMoeDsaMoE(nn.Module):
+ """
+ A mixed expert module containing shared experts.
+ """
+
+ def __init__(self, config):
+ super().__init__()
+ self.config = config
+ self.num_experts = config.num_local_experts
+ from transformers.models.glm_moe_dsa.modeling_glm_moe_dsa import GlmMoeDsaMLP, GlmMoeDsaTopkRouter
+
+ self.experts = nn.ModuleList(
+ [GlmMoeDsaMLP(config, intermediate_size=config.moe_intermediate_size) for _ in range(self.num_experts)]
+ )
+
+ self.gate = GlmMoeDsaTopkRouter(config)
+ self.shared_experts = GlmMoeDsaMLP(
+ config=config, intermediate_size=config.moe_intermediate_size * config.n_shared_experts
+ )
+ self.n_routed_experts = config.n_routed_experts
+ self.n_group = config.n_group
+ self.topk_group = config.topk_group
+ self.norm_topk_prob = config.norm_topk_prob
+ self.routed_scaling_factor = config.routed_scaling_factor
+ self.top_k = config.num_experts_per_tok
+
+ def experts_forward(
+ self,
+ hidden_states: torch.Tensor,
+ top_k_index: torch.Tensor,
+ top_k_weights: torch.Tensor,
+ ) -> torch.Tensor:
+ final_hidden_states = torch.zeros_like(hidden_states)
+ with torch.no_grad():
+ expert_mask = torch.nn.functional.one_hot(top_k_index, num_classes=self.num_experts)
+ expert_mask = expert_mask.permute(2, 1, 0)
+ expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()
+
+ for expert_idx in expert_hit:
+ expert_idx = expert_idx[0]
+ if expert_idx == self.num_experts:
+ continue
+ top_k_pos, token_idx = torch.where(expert_mask[expert_idx])
+ current_state = hidden_states[token_idx]
+ # gate, up = nn.functional.linear(current_state, self.gate_up_proj[expert_idx]).chunk(2, dim=-1)
+ # current_hidden_states = self.act_fn(gate) * up
+ # current_hidden_states = nn.functional.linear(current_hidden_states, self.down_proj[expert_idx])
+ expert_layer = self.experts[expert_idx]
+ current_hidden_states = expert_layer(current_state)
+ current_hidden_states = current_hidden_states * top_k_weights[token_idx, top_k_pos, None]
+ final_hidden_states.index_add_(0, token_idx, current_hidden_states.to(final_hidden_states.dtype))
+
+ return final_hidden_states
+
+ def route_tokens_to_experts(self, router_logits):
+ router_logits = router_logits.sigmoid()
+ router_logits_for_choice = router_logits + self.gate.e_score_correction_bias
+ group_scores = (
+ router_logits_for_choice.view(-1, self.n_group, self.n_routed_experts // self.n_group)
+ .topk(2, dim=-1)[0]
+ .sum(dim=-1)
+ )
+ group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]
+ group_mask = torch.zeros_like(group_scores)
+ group_mask.scatter_(1, group_idx, 1)
+ score_mask = (
+ group_mask.unsqueeze(-1)
+ .expand(-1, self.n_group, self.n_routed_experts // self.n_group)
+ .reshape(-1, self.n_routed_experts)
+ )
+ scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), 0.0)
+ topk_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1]
+ topk_weights = router_logits.gather(1, topk_indices)
+ if self.norm_topk_prob:
+ denominator = topk_weights.sum(dim=-1, keepdim=True) + 1e-20
+ topk_weights /= denominator
+ topk_weights = topk_weights * self.routed_scaling_factor
+ return topk_indices, topk_weights
+
+ def forward(self, hidden_states):
+ residuals = hidden_states
+ orig_shape = hidden_states.shape
+ router_logits = self.gate(hidden_states)
+ topk_indices, topk_weights = self.route_tokens_to_experts(router_logits)
+ hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
+ hidden_states = self.experts_forward(hidden_states, topk_indices, topk_weights).view(*orig_shape)
+ hidden_states = hidden_states + self.shared_experts(residuals)
+ return hidden_states
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 5a0b16bb7..7627f0782 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -256,6 +256,16 @@ register_ignore_layers(
],
)
+
+# glm5
+register_ignore_layers(
+ matchers=[
+ ModelTypeMatcher(r"glm_moe_dsa", mode="full"),
+ ],
+ ignore_layers=[get_glm_flash_ignore_layers, "weights_proj"], # vllm issue
+)
+
+
# # qwen3_next
# register_ignore_layers(
# matchers=[
diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.py
index e501066b3..efb5ebc52 100644
--- a/auto_round_extension/torch/qlinear_torch_zp.py
+++ b/auto_round_extension/torch/qlinear_torch_zp.py
@@ -33,7 +33,7 @@ class QuantLinear(nn.Module):
QUANT_TYPE = "torch"
- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):
+ def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, g_idx=False, **kwargs):
super().__init__()
if bits not in [2, 3, 4, 8]:
raise NotImplementedError("Only 2,3,4,8 bits are supported.")
@@ -47,6 +47,11 @@ class QuantLinear(nn.Module):
"qweight",
torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
)
+ if g_idx:
+ self.register_buffer(
+ "g_idx", torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32)
+ )
+
self.register_buffer(
"qzeros",
torch.zeros(