PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix nemoron gate inference issue (#2325)
Wenhua Cheng0ea7c93c3450
diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.pyindex 2a9bc7f1c..cf0954b1b 100644--- a/auto_round/modeling/fused_moe/moe_experts_interface.py+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py@@ -276,15 +276,22 @@ def linear_loop_experts_forward(# Get this expert's container with its projection layersexpert = getattr(self, str(expert_idx))- gate_out = expert.gate_proj(expert_input) # (num_samples, intermediate_dim)up_out = expert.up_proj(expert_input) # (num_samples, intermediate_dim)- # Apply gating- if hasattr(self, "_apply_gate"):- gate_up_out = torch.cat([gate_out, up_out], dim=-1)- gated_out = self._apply_gate(gate_up_out) # (num_samples, intermediate_dim)+ # Non-gated MLP experts (e.g. NemotronH's @use_experts_implementation(has_gate=False))+ # expose only up_proj/down_proj: out = down_proj(act_fn(up_proj(x))). Gated experts+ # (Mixtral/Qwen3-MoE/...) additionally own gate_proj. Mirror the grouped path+ # (grouped_linear_experts_forward) which keys the same branch off gate_proj presence.+ if hasattr(expert, "gate_proj"):+ gate_out = expert.gate_proj(expert_input) # (num_samples, intermediate_dim)+ # Apply gating+ if hasattr(self, "_apply_gate"):+ gate_up_out = torch.cat([gate_out, up_out], dim=-1)+ gated_out = self._apply_gate(gate_up_out) # (num_samples, intermediate_dim)+ else:+ gated_out = self.act_fn(gate_out) * up_out # (num_samples, intermediate_dim)else:- gated_out = self.act_fn(gate_out) * up_out # (num_samples, intermediate_dim)+ gated_out = self.act_fn(up_out) # (num_samples, intermediate_dim)# Down projectionexpert_out = expert.down_proj(gated_out) # (num_samples, hidden_dim)diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex c8f09f2a9..010b3669e 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -2877,11 +2877,6 @@ def place_ngram_embeddings_for_tuning_(module: torch.nn.Module, gpu_devices: lislen(fallback_ngram_names),", ".join(fallback_ngram_names[:3]),)- else:- logger.info_once(- "AR_NGRAM_DEVICE=%s is set, but no ngram modules were found under this block.",- setting or "auto",- )return []total_ngram_nbytes = sum(_module_storage_nbytes(sub) for _, sub in ngram_modules)logger.info_once(