PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support glm5 (#1466)
Wenhua Cheng62c9808d5773
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex bc19dd32b..75e9b0f3d 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -212,8 +212,8 @@ def save_quantized_as_autogptq(# --- Save metadata (tokenizer, processor, etc.) ---if output_dir:- if os.path.exists(output_dir):- logger.warning(f"{output_dir} already exists, may cause overwrite conflicts.")+ # if os.path.exists(output_dir):+ # logger.info(f"{output_dir} already exists, may cause overwrite conflicts.")for comp in (tokenizer, processor, image_processor):if comp is not None and hasattr(comp, "save_pretrained"):comp.save_pretrained(output_dir)diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex ac4701dd3..c57bf452c 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -332,8 +332,8 @@ def save_quantized_as_autoround(if output_dir is None:model.tokenizer = tokenizerreturn model- if os.path.exists(output_dir):- logger.warning(f"{output_dir} already exists, this may cause model conflict")+ # if os.path.exists(output_dir):+ # logger.info(f"{output_dir} already exists, this may cause model conflict")if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):tokenizer.save_pretrained(output_dir)diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.pyindex 82fcdcf20..70c7c71b1 100644--- a/auto_round/export/export_to_autoround/export_to_fp8.py+++ b/auto_round/export/export_to_autoround/export_to_fp8.py@@ -232,8 +232,8 @@ def save_quantized_as_autoround(if output_dir is None:model.tokenizer = tokenizerreturn model- if os.path.exists(output_dir):- logger.warning(f"{output_dir} already exists, this may cause model conflict")+ # if os.path.exists(output_dir):+ # logger.warning(f"{output_dir} already exists, this may cause model conflict")if tokenizer is not None:tokenizer.save_pretrained(output_dir)diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex 3c39d06e4..47e545ff9 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -141,8 +141,8 @@ def save_quantized_as_llmcompressor(safe_serialization = kwargs.get("safe_serialization", True)processor = kwargs.get("processor", None)- if output_dir is not None and os.path.exists(output_dir):- logger.warning(f"{output_dir} already exists, this may cause model conflict")+ # if output_dir is not None and os.path.exists(output_dir):+ # logger.warning(f"{output_dir} already exists, this may cause model conflict")if not inplace:model = copy.deepcopy(model.to("cpu"))diff --git a/auto_round/export/utils.py b/auto_round/export/utils.pyindex 42e6b4f9e..3bdacf096 100644--- a/auto_round/export/utils.py+++ b/auto_round/export/utils.py@@ -110,6 +110,8 @@ def save_model(model.config.save_pretrained(save_dir)if hasattr(model, "generation_config") and model.generation_config is not None:+ if hasattr(model, "generation_config"):+ setattr(model.generation_config, "do_sample", True)model.generation_config.save_pretrained(save_dir)elif has_unfused_experts:# For models with unfused MOE experts, save state_dict directly to avoiddiff --git a/auto_round/modeling/unfused_moe/__init__.py b/auto_round/modeling/unfused_moe/__init__.pyindex a30e2ae45..d00c21279 100644--- a/auto_round/modeling/unfused_moe/__init__.py+++ b/auto_round/modeling/unfused_moe/__init__.py@@ -55,6 +55,17 @@ MODEL_CONFIG = {)],},+ "glm_moe_dsa": {+ "min_transformers_version": "5.0.0",+ "max_transformers_version": "6.0.0",+ "checkpoint_mapping": [],+ "block_patch": [+ (+ "transformers.models.glm_moe_dsa.modeling_glm_moe_dsa.GlmMoeDsaMoE",+ "auto_round.modeling.unfused_moe.glm_moe_dsa.LinearGlmMoeDsaMoE",+ )+ ],+ },"qwen3_next": {"min_transformers_version": "5.0.0","max_transformers_version": "6.0.0",diff --git a/auto_round/modeling/unfused_moe/glm_moe_dsa.py b/auto_round/modeling/unfused_moe/glm_moe_dsa.pynew file mode 100644index 000000000..2fb34de0f--- /dev/null+++ b/auto_round/modeling/unfused_moe/glm_moe_dsa.py@@ -0,0 +1,95 @@+# Copyright (C) 2026 Intel Corporation+# SPDX-License-Identifier: Apache-2.0++import torch+import torch.nn as nn+++class LinearGlmMoeDsaMoE(nn.Module):+ """+ A mixed expert module containing shared experts.+ """++ def __init__(self, config):+ super().__init__()+ self.config = config+ self.num_experts = config.num_local_experts+ from transformers.models.glm_moe_dsa.modeling_glm_moe_dsa import GlmMoeDsaMLP, GlmMoeDsaTopkRouter++ self.experts = nn.ModuleList(+ [GlmMoeDsaMLP(config, intermediate_size=config.moe_intermediate_size) for _ in range(self.num_experts)]+ )++ self.gate = GlmMoeDsaTopkRouter(config)+ self.shared_experts = GlmMoeDsaMLP(+ config=config, intermediate_size=config.moe_intermediate_size * config.n_shared_experts+ )+ self.n_routed_experts = config.n_routed_experts+ self.n_group = config.n_group+ self.topk_group = config.topk_group+ self.norm_topk_prob = config.norm_topk_prob+ self.routed_scaling_factor = config.routed_scaling_factor+ self.top_k = config.num_experts_per_tok++ def experts_forward(+ self,+ hidden_states: torch.Tensor,+ top_k_index: torch.Tensor,+ top_k_weights: torch.Tensor,+ ) -> torch.Tensor:+ final_hidden_states = torch.zeros_like(hidden_states)+ with torch.no_grad():+ expert_mask = torch.nn.functional.one_hot(top_k_index, num_classes=self.num_experts)+ expert_mask = expert_mask.permute(2, 1, 0)+ expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()++ for expert_idx in expert_hit:+ expert_idx = expert_idx[0]+ if expert_idx == self.num_experts:+ continue+ top_k_pos, token_idx = torch.where(expert_mask[expert_idx])+ current_state = hidden_states[token_idx]+ # gate, up = nn.functional.linear(current_state, self.gate_up_proj[expert_idx]).chunk(2, dim=-1)+ # current_hidden_states = self.act_fn(gate) * up+ # current_hidden_states = nn.functional.linear(current_hidden_states, self.down_proj[expert_idx])+ expert_layer = self.experts[expert_idx]+ current_hidden_states = expert_layer(current_state)+ current_hidden_states = current_hidden_states * top_k_weights[token_idx, top_k_pos, None]+ final_hidden_states.index_add_(0, token_idx, current_hidden_states.to(final_hidden_states.dtype))++ return final_hidden_states++ def route_tokens_to_experts(self, router_logits):+ router_logits = router_logits.sigmoid()+ router_logits_for_choice = router_logits + self.gate.e_score_correction_bias+ group_scores = (+ router_logits_for_choice.view(-1, self.n_group, self.n_routed_experts // self.n_group)+ .topk(2, dim=-1)[0]+ .sum(dim=-1)+ )+ group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]+ group_mask = torch.zeros_like(group_scores)+ group_mask.scatter_(1, group_idx, 1)+ score_mask = (+ group_mask.unsqueeze(-1)+ .expand(-1, self.n_group, self.n_routed_experts // self.n_group)+ .reshape(-1, self.n_routed_experts)+ )+ scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), 0.0)+ topk_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1]+ topk_weights = router_logits.gather(1, topk_indices)+ if self.norm_topk_prob:+ denominator = topk_weights.sum(dim=-1, keepdim=True) + 1e-20+ topk_weights /= denominator+ topk_weights = topk_weights * self.routed_scaling_factor+ return topk_indices, topk_weights++ def forward(self, hidden_states):+ residuals = hidden_states+ orig_shape = hidden_states.shape+ router_logits = self.gate(hidden_states)+ topk_indices, topk_weights = self.route_tokens_to_experts(router_logits)+ hidden_states = hidden_states.view(-1, hidden_states.shape[-1])+ hidden_states = self.experts_forward(hidden_states, topk_indices, topk_weights).view(*orig_shape)+ hidden_states = hidden_states + self.shared_experts(residuals)+ return hidden_statesdiff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 5a0b16bb7..7627f0782 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -256,6 +256,16 @@ register_ignore_layers(],)++# glm5+register_ignore_layers(+ matchers=[+ ModelTypeMatcher(r"glm_moe_dsa", mode="full"),+ ],+ ignore_layers=[get_glm_flash_ignore_layers, "weights_proj"], # vllm issue+)++# # qwen3_next# register_ignore_layers(# matchers=[diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.pyindex e501066b3..efb5ebc52 100644--- a/auto_round_extension/torch/qlinear_torch_zp.py+++ b/auto_round_extension/torch/qlinear_torch_zp.py@@ -33,7 +33,7 @@ class QuantLinear(nn.Module):QUANT_TYPE = "torch"- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):+ def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, g_idx=False, **kwargs):super().__init__()if bits not in [2, 3, 4, 8]:raise NotImplementedError("Only 2,3,4,8 bits are supported.")@@ -47,6 +47,11 @@ class QuantLinear(nn.Module):"qweight",torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),)+ if g_idx:+ self.register_buffer(+ "g_idx", torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32)+ )+self.register_buffer("qzeros",torch.zeros(